252 lines
7.8 KiB
TypeScript
252 lines
7.8 KiB
TypeScript
import "server-only";
|
|
|
|
import sharp from "sharp";
|
|
import { uploadFile } from "@/lib/storage/service";
|
|
import type { NineGridCropBox, NineGridDetectionResult, NineGridTile } from "@/lib/video-agent/types";
|
|
|
|
type ProjectionPeak = {
|
|
index: number;
|
|
value: number;
|
|
};
|
|
|
|
function clamp01(value: number) {
|
|
if (!Number.isFinite(value)) return 0;
|
|
return Math.max(0, Math.min(1, value));
|
|
}
|
|
|
|
function sampleStddevFromUint8(data: Uint8Array, limit = 20000) {
|
|
const total = Math.min(data.length, Math.max(128, limit));
|
|
if (total <= 0) return 0;
|
|
const stride = Math.max(1, Math.floor(data.length / total));
|
|
const sampled: number[] = [];
|
|
for (let index = 0; index < data.length && sampled.length < total; index += stride) {
|
|
sampled.push(data[index] || 0);
|
|
}
|
|
return stddev(sampled);
|
|
}
|
|
|
|
function toArrayBuffer(buffer: Buffer): ArrayBuffer {
|
|
return buffer.buffer.slice(
|
|
buffer.byteOffset,
|
|
buffer.byteOffset + buffer.byteLength
|
|
) as ArrayBuffer;
|
|
}
|
|
|
|
function movingAverage(values: number[], radius = 3): number[] {
|
|
if (values.length === 0) return values;
|
|
const result = new Array(values.length).fill(0);
|
|
for (let index = 0; index < values.length; index += 1) {
|
|
let sum = 0;
|
|
let count = 0;
|
|
const start = Math.max(0, index - radius);
|
|
const end = Math.min(values.length - 1, index + radius);
|
|
for (let cursor = start; cursor <= end; cursor += 1) {
|
|
sum += values[cursor] || 0;
|
|
count += 1;
|
|
}
|
|
result[index] = count > 0 ? sum / count : 0;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function mean(values: number[]) {
|
|
if (values.length === 0) return 0;
|
|
return values.reduce((total, item) => total + item, 0) / values.length;
|
|
}
|
|
|
|
function stddev(values: number[]) {
|
|
if (values.length === 0) return 0;
|
|
const avg = mean(values);
|
|
const variance =
|
|
values.reduce((total, item) => total + Math.pow(item - avg, 2), 0) / values.length;
|
|
return Math.sqrt(Math.max(variance, 0));
|
|
}
|
|
|
|
function findPeakNear(
|
|
scores: number[],
|
|
targetIndex: number,
|
|
windowRadius: number
|
|
): ProjectionPeak {
|
|
const start = Math.max(0, Math.round(targetIndex - windowRadius));
|
|
const end = Math.min(scores.length - 1, Math.round(targetIndex + windowRadius));
|
|
let bestIndex = Math.round(targetIndex);
|
|
let bestValue = -Infinity;
|
|
for (let index = start; index <= end; index += 1) {
|
|
const value = scores[index] || 0;
|
|
if (value > bestValue) {
|
|
bestValue = value;
|
|
bestIndex = index;
|
|
}
|
|
}
|
|
return { index: bestIndex, value: Number.isFinite(bestValue) ? bestValue : 0 };
|
|
}
|
|
|
|
function scoreGridAxis(scores: number[], firstTarget: number, secondTarget: number) {
|
|
const radius = Math.max(8, Math.round(scores.length * 0.12));
|
|
const firstPeak = findPeakNear(scores, firstTarget, radius);
|
|
const secondPeak = findPeakNear(scores, secondTarget, radius);
|
|
const avg = mean(scores);
|
|
const sigma = stddev(scores);
|
|
const dynamicBase = avg + sigma;
|
|
|
|
const closenessA = clamp01(1 - Math.abs(firstPeak.index - firstTarget) / radius);
|
|
const closenessB = clamp01(1 - Math.abs(secondPeak.index - secondTarget) / radius);
|
|
const closeness = (closenessA + closenessB) / 2;
|
|
|
|
const denom = Math.max(dynamicBase, 1);
|
|
const strengthA = clamp01(firstPeak.value / (denom * 2.2));
|
|
const strengthB = clamp01(secondPeak.value / (denom * 2.2));
|
|
const strength = (strengthA + strengthB) / 2;
|
|
|
|
return {
|
|
score: clamp01(closeness * 0.55 + strength * 0.45),
|
|
peaks: [firstPeak, secondPeak],
|
|
};
|
|
}
|
|
|
|
function buildNineGridCropBoxes(width: number, height: number): NineGridCropBox[] {
|
|
const baseW = Math.floor(width / 3);
|
|
const baseH = Math.floor(height / 3);
|
|
const boxes: NineGridCropBox[] = [];
|
|
let index = 1;
|
|
for (let row = 0; row < 3; row += 1) {
|
|
for (let col = 0; col < 3; col += 1) {
|
|
const x = col * baseW;
|
|
const y = row * baseH;
|
|
const w = col === 2 ? width - x : baseW;
|
|
const h = row === 2 ? height - y : baseH;
|
|
boxes.push({ index, x, y, w, h });
|
|
index += 1;
|
|
}
|
|
}
|
|
return boxes;
|
|
}
|
|
|
|
async function fetchImageBuffer(imageUrl: string) {
|
|
const response = await fetch(imageUrl);
|
|
if (!response.ok) {
|
|
throw new Error(`图片下载失败 (HTTP ${response.status})`);
|
|
}
|
|
const contentType = response.headers.get("content-type") || "";
|
|
if (!/^image\//i.test(contentType)) {
|
|
throw new Error(`输入不是图片类型: ${contentType || "unknown"}`);
|
|
}
|
|
return Buffer.from(await response.arrayBuffer());
|
|
}
|
|
|
|
async function sliceAndUploadTiles(params: {
|
|
sourceBuffer: Buffer;
|
|
cropBoxes: NineGridCropBox[];
|
|
}): Promise<NineGridTile[]> {
|
|
const { sourceBuffer, cropBoxes } = params;
|
|
const timestamp = Date.now();
|
|
const tiles: NineGridTile[] = [];
|
|
for (const cropBox of cropBoxes) {
|
|
const tileBuffer = await sharp(sourceBuffer)
|
|
.extract({
|
|
left: cropBox.x,
|
|
top: cropBox.y,
|
|
width: cropBox.w,
|
|
height: cropBox.h,
|
|
})
|
|
.jpeg({ quality: 92 })
|
|
.toBuffer();
|
|
|
|
const filename = `video-storyboard-tile-${timestamp}-${cropBox.index}.jpg`;
|
|
const uploaded = await uploadFile(filename, toArrayBuffer(tileBuffer));
|
|
tiles.push({
|
|
index: cropBox.index,
|
|
imageUrl: uploaded.url,
|
|
width: cropBox.w,
|
|
height: cropBox.h,
|
|
});
|
|
}
|
|
return tiles;
|
|
}
|
|
|
|
export async function runNineGridDetectorSkill(params: {
|
|
imageUrl: string;
|
|
minConfidence: number;
|
|
}): Promise<NineGridDetectionResult> {
|
|
const sourceImageUrl = String(params.imageUrl || "").trim();
|
|
if (!sourceImageUrl) {
|
|
throw new Error("缺少九宫格输入图片");
|
|
}
|
|
|
|
const sourceBuffer = await fetchImageBuffer(sourceImageUrl);
|
|
const baseImage = sharp(sourceBuffer, { failOn: "none" }).rotate();
|
|
const metadata = await baseImage.metadata();
|
|
const width = Number(metadata.width || 0);
|
|
const height = Number(metadata.height || 0);
|
|
|
|
if (width < 240 || height < 240) {
|
|
return {
|
|
isNineGrid: false,
|
|
confidence: 0,
|
|
qualityScore: 0,
|
|
cropBoxes: [],
|
|
width,
|
|
height,
|
|
sourceImageUrl,
|
|
tiles: [],
|
|
};
|
|
}
|
|
|
|
const grayscale = await baseImage.greyscale().raw().toBuffer({
|
|
resolveWithObject: true,
|
|
});
|
|
const raw = grayscale.data;
|
|
const channels = grayscale.info.channels || 1;
|
|
const imgW = grayscale.info.width;
|
|
const imgH = grayscale.info.height;
|
|
const step = Math.max(1, Math.round(Math.min(imgW, imgH) / 900));
|
|
|
|
const colScores = new Array(imgW).fill(0);
|
|
const rowScores = new Array(imgH).fill(0);
|
|
|
|
for (let y = 0; y < imgH; y += step) {
|
|
const rowOffset = y * imgW * channels;
|
|
for (let x = step; x < imgW; x += step) {
|
|
const current = raw[rowOffset + x * channels] || 0;
|
|
const prev = raw[rowOffset + (x - step) * channels] || 0;
|
|
colScores[x] += Math.abs(current - prev);
|
|
}
|
|
}
|
|
|
|
for (let x = 0; x < imgW; x += step) {
|
|
for (let y = step; y < imgH; y += step) {
|
|
const current = raw[(y * imgW + x) * channels] || 0;
|
|
const prev = raw[((y - step) * imgW + x) * channels] || 0;
|
|
rowScores[y] += Math.abs(current - prev);
|
|
}
|
|
}
|
|
|
|
const smoothCols = movingAverage(colScores, 5);
|
|
const smoothRows = movingAverage(rowScores, 5);
|
|
const colAxis = scoreGridAxis(smoothCols, imgW / 3, (imgW * 2) / 3);
|
|
const rowAxis = scoreGridAxis(smoothRows, imgH / 3, (imgH * 2) / 3);
|
|
|
|
const ratio = width / Math.max(height, 1);
|
|
const aspectScore = clamp01(1 - Math.min(Math.abs(ratio - 1), 1));
|
|
const gridScore = clamp01((colAxis.score + rowAxis.score) / 2);
|
|
const confidence = clamp01(aspectScore * 0.3 + gridScore * 0.7);
|
|
const qualityScore = clamp01((sampleStddevFromUint8(raw, 20000) || 0) / 72);
|
|
|
|
const isNineGrid = confidence >= params.minConfidence && aspectScore >= 0.62;
|
|
const cropBoxes = isNineGrid ? buildNineGridCropBoxes(width, height) : [];
|
|
const tiles = isNineGrid
|
|
? await sliceAndUploadTiles({ sourceBuffer: await baseImage.toBuffer(), cropBoxes })
|
|
: [];
|
|
|
|
return {
|
|
isNineGrid,
|
|
confidence,
|
|
qualityScore,
|
|
cropBoxes,
|
|
width,
|
|
height,
|
|
sourceImageUrl,
|
|
tiles,
|
|
};
|
|
}
|