feat: add subject-aware detail overlay layout
This commit is contained in:
parent
139b14b3c6
commit
409770d386
233
app/api/canvas/detail-subject-analyze/route.ts
Normal file
233
app/api/canvas/detail-subject-analyze/route.ts
Normal file
@ -0,0 +1,233 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import sharp from "sharp";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import {
|
||||
clampNormalized,
|
||||
type DetailUiSubjectAnalysis,
|
||||
} from "@/lib/detail-ui-subject-analysis";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 30;
|
||||
|
||||
const ANALYZE_MAX_SIDE = 128;
|
||||
const ANALYZE_FETCH_TIMEOUT_MS = 12000;
|
||||
|
||||
function percentile(values: number[], ratio: number) {
|
||||
if (values.length === 0) return 0;
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const index = Math.max(0, Math.min(sorted.length - 1, Math.floor(sorted.length * ratio)));
|
||||
return sorted[index] || 0;
|
||||
}
|
||||
|
||||
function buildSafeZones(params: {
|
||||
subjectBox: { x: number; y: number; width: number; height: number };
|
||||
}) {
|
||||
const gap = 0.035;
|
||||
const leftWidth = Math.max(0, params.subjectBox.x - gap);
|
||||
const rightX = clampNormalized(params.subjectBox.x + params.subjectBox.width + gap);
|
||||
const rightWidth = Math.max(0, 1 - rightX);
|
||||
const topHeight = Math.max(0, params.subjectBox.y - gap);
|
||||
const bottomY = clampNormalized(params.subjectBox.y + params.subjectBox.height + gap);
|
||||
const bottomHeight = Math.max(0, 1 - bottomY);
|
||||
|
||||
return {
|
||||
left:
|
||||
leftWidth > 0.16
|
||||
? {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: leftWidth,
|
||||
height: 1,
|
||||
}
|
||||
: null,
|
||||
right:
|
||||
rightWidth > 0.16
|
||||
? {
|
||||
x: rightX,
|
||||
y: 0,
|
||||
width: rightWidth,
|
||||
height: 1,
|
||||
}
|
||||
: null,
|
||||
top:
|
||||
topHeight > 0.16
|
||||
? {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 1,
|
||||
height: topHeight,
|
||||
}
|
||||
: null,
|
||||
bottom:
|
||||
bottomHeight > 0.14
|
||||
? {
|
||||
x: 0,
|
||||
y: bottomY,
|
||||
width: 1,
|
||||
height: bottomHeight,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function analyzeSubjectRegion(params: {
|
||||
grayscale: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
}): DetailUiSubjectAnalysis {
|
||||
const scores: number[] = [];
|
||||
const scoreGrid = new Float32Array(params.width * params.height);
|
||||
|
||||
const pixelAt = (x: number, y: number) =>
|
||||
params.grayscale[Math.max(0, Math.min(params.grayscale.length - 1, y * params.width + x))] || 0;
|
||||
|
||||
for (let y = 1; y < params.height - 1; y += 1) {
|
||||
for (let x = 1; x < params.width - 1; x += 1) {
|
||||
const gx =
|
||||
-1 * pixelAt(x - 1, y - 1) +
|
||||
1 * pixelAt(x + 1, y - 1) +
|
||||
-2 * pixelAt(x - 1, y) +
|
||||
2 * pixelAt(x + 1, y) +
|
||||
-1 * pixelAt(x - 1, y + 1) +
|
||||
1 * pixelAt(x + 1, y + 1);
|
||||
const gy =
|
||||
-1 * pixelAt(x - 1, y - 1) +
|
||||
-2 * pixelAt(x, y - 1) +
|
||||
-1 * pixelAt(x + 1, y - 1) +
|
||||
1 * pixelAt(x - 1, y + 1) +
|
||||
2 * pixelAt(x, y + 1) +
|
||||
1 * pixelAt(x + 1, y + 1);
|
||||
const edge = Math.sqrt(gx * gx + gy * gy);
|
||||
|
||||
const centerBiasX = 1 - Math.abs(x / params.width - 0.5) * 1.5;
|
||||
const centerBiasY = 1 - Math.abs(y / params.height - 0.56) * 1.3;
|
||||
const centerBias = Math.max(0.15, centerBiasX * centerBiasY);
|
||||
const brightness = pixelAt(x, y) / 255;
|
||||
const contrast = Math.abs(brightness - 0.5);
|
||||
const score = edge * 0.72 + contrast * 60 + centerBias * 90;
|
||||
|
||||
const index = y * params.width + x;
|
||||
scoreGrid[index] = score;
|
||||
scores.push(score);
|
||||
}
|
||||
}
|
||||
|
||||
const threshold = percentile(scores, 0.84);
|
||||
let minX = params.width;
|
||||
let minY = params.height;
|
||||
let maxX = 0;
|
||||
let maxY = 0;
|
||||
|
||||
for (let y = 0; y < params.height; y += 1) {
|
||||
for (let x = 0; x < params.width; x += 1) {
|
||||
const score = scoreGrid[y * params.width + x];
|
||||
if (score < threshold) continue;
|
||||
minX = Math.min(minX, x);
|
||||
minY = Math.min(minY, y);
|
||||
maxX = Math.max(maxX, x);
|
||||
maxY = Math.max(maxY, y);
|
||||
}
|
||||
}
|
||||
|
||||
if (minX >= maxX || minY >= maxY) {
|
||||
minX = Math.round(params.width * 0.24);
|
||||
maxX = Math.round(params.width * 0.78);
|
||||
minY = Math.round(params.height * 0.18);
|
||||
maxY = Math.round(params.height * 0.92);
|
||||
}
|
||||
|
||||
const expandX = Math.round((maxX - minX) * 0.08);
|
||||
const expandY = Math.round((maxY - minY) * 0.08);
|
||||
minX = Math.max(0, minX - expandX);
|
||||
minY = Math.max(0, minY - expandY);
|
||||
maxX = Math.min(params.width - 1, maxX + expandX);
|
||||
maxY = Math.min(params.height - 1, maxY + expandY);
|
||||
|
||||
const subjectBox = {
|
||||
x: clampNormalized(minX / params.width),
|
||||
y: clampNormalized(minY / params.height),
|
||||
width: clampNormalized((maxX - minX) / params.width),
|
||||
height: clampNormalized((maxY - minY) / params.height),
|
||||
confidence: 0.72,
|
||||
};
|
||||
|
||||
return {
|
||||
subjectBox,
|
||||
safeZones: buildSafeZones({ subjectBox }),
|
||||
dominantHorizontal:
|
||||
subjectBox.x + subjectBox.width / 2 < 0.42
|
||||
? "left"
|
||||
: subjectBox.x + subjectBox.width / 2 > 0.58
|
||||
? "right"
|
||||
: "center",
|
||||
dominantVertical:
|
||||
subjectBox.y + subjectBox.height / 2 < 0.42
|
||||
? "top"
|
||||
: subjectBox.y + subjectBox.height / 2 > 0.58
|
||||
? "bottom"
|
||||
: "center",
|
||||
};
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||||
}
|
||||
|
||||
const body = (await request.json()) as { imageUrl?: string };
|
||||
const imageUrl = String(body.imageUrl || "").trim();
|
||||
if (!/^https?:\/\//i.test(imageUrl)) {
|
||||
return NextResponse.json({ success: false, error: "缺少图片地址" }, { status: 400 });
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), ANALYZE_FETCH_TIMEOUT_MS);
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
const response = await fetch(imageUrl, {
|
||||
method: "GET",
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`图片拉取失败: HTTP ${response.status}`);
|
||||
}
|
||||
buffer = Buffer.from(await response.arrayBuffer());
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
|
||||
const { data, info } = await sharp(buffer)
|
||||
.rotate()
|
||||
.resize({
|
||||
width: ANALYZE_MAX_SIDE,
|
||||
height: ANALYZE_MAX_SIDE,
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.grayscale()
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true });
|
||||
|
||||
const analysis = analyzeSubjectRegion({
|
||||
grayscale: data,
|
||||
width: info.width || ANALYZE_MAX_SIDE,
|
||||
height: info.height || ANALYZE_MAX_SIDE,
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: analysis,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Canvas Detail Subject Analyze] Error:", error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : "主体分析失败",
|
||||
},
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -6307,6 +6307,26 @@ export function CanvasChatPanel({
|
||||
const preflightLabel = planData.preflight?.rewriteApplied
|
||||
? "双总监预审重写完成"
|
||||
: "双总监预审通过";
|
||||
const requestDetailSubjectAnalysis = async (imageUrl?: string) => {
|
||||
const normalized = String(imageUrl || "").trim();
|
||||
if (!/^https?:\/\//i.test(normalized)) return null;
|
||||
|
||||
try {
|
||||
const response = await fetch("/api/canvas/detail-subject-analyze", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ imageUrl: normalized }),
|
||||
});
|
||||
const result = await response.json().catch(() => ({}));
|
||||
if (!response.ok || !result?.success) {
|
||||
return null;
|
||||
}
|
||||
return result.data || null;
|
||||
} catch (error) {
|
||||
console.warn("[CanvasChat] Detail subject analyze failed:", error);
|
||||
return null;
|
||||
}
|
||||
};
|
||||
for (let index = 0; index < prompts.length; index += 1) {
|
||||
patchDetailPlaceholder(index, {
|
||||
label: preflightLabel,
|
||||
@ -6711,6 +6731,11 @@ export function CanvasChatPanel({
|
||||
const dimensions = await getImageDimensions(
|
||||
finalGenerated.originalUrl || finalGenerated.thumbnailUrl
|
||||
);
|
||||
const detailSubjectAnalysis = isAgentLayoutMode
|
||||
? await requestDetailSubjectAnalysis(
|
||||
finalGenerated.originalUrl || finalGenerated.thumbnailUrl,
|
||||
)
|
||||
: null;
|
||||
if (targetNodeId) {
|
||||
const storeState = useCanvasStore.getState();
|
||||
const currentNode = storeState.getNode(targetNodeId);
|
||||
@ -6752,6 +6777,7 @@ export function CanvasChatPanel({
|
||||
surface: currentDetailSurfaceSnapshot,
|
||||
screenIndex: index + 1,
|
||||
}),
|
||||
detailSubjectAnalysis,
|
||||
detailGeneratingPlaceholder: false,
|
||||
detailProgressLabel: `该屏完成 ${index + 1}/${prompts.length}`,
|
||||
},
|
||||
|
||||
@ -4,6 +4,10 @@ import type { CSSProperties } from "react";
|
||||
import { ArrowRight, CheckCircle2, ShieldCheck, Sparkles } from "lucide-react";
|
||||
import { cn } from "@/lib/utils";
|
||||
import type { DetailUiScreenOverlaySnapshot } from "@/lib/detail-ui-surface-adapter";
|
||||
import type {
|
||||
DetailUiSafeZone,
|
||||
DetailUiSubjectAnalysis,
|
||||
} from "@/lib/detail-ui-subject-analysis";
|
||||
|
||||
type RenderableOverlayComponent =
|
||||
DetailUiScreenOverlaySnapshot["components"][number];
|
||||
@ -147,8 +151,14 @@ function getOverlayComponentStyle(params: {
|
||||
component: RenderableOverlayComponent;
|
||||
snapshot: DetailUiScreenOverlaySnapshot;
|
||||
content: string;
|
||||
frameOverride?: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
}) {
|
||||
const frame = (params.component.layout?.frame || {}) as {
|
||||
const frame = (params.frameOverride || params.component.layout?.frame || {}) as {
|
||||
x?: number;
|
||||
y?: number;
|
||||
width?: number;
|
||||
@ -233,6 +243,207 @@ function getOverlayComponentStyle(params: {
|
||||
return overlayStyle;
|
||||
}
|
||||
|
||||
function normalizeZoneToPixels(params: {
|
||||
zone: DetailUiSafeZone;
|
||||
width: number;
|
||||
height: number;
|
||||
}) {
|
||||
return {
|
||||
x: params.zone.x * params.width,
|
||||
y: params.zone.y * params.height,
|
||||
width: params.zone.width * params.width,
|
||||
height: params.zone.height * params.height,
|
||||
};
|
||||
}
|
||||
|
||||
function chooseGroupZone(params: {
|
||||
subjectAnalysis: DetailUiSubjectAnalysis;
|
||||
imageWidth: number;
|
||||
imageHeight: number;
|
||||
preferred: Array<"left" | "right" | "top" | "bottom">;
|
||||
minimumWidth: number;
|
||||
minimumHeight: number;
|
||||
}) {
|
||||
for (const key of params.preferred) {
|
||||
const zone = params.subjectAnalysis.safeZones[key];
|
||||
if (!zone) continue;
|
||||
const pixelZone = normalizeZoneToPixels({
|
||||
zone,
|
||||
width: params.imageWidth,
|
||||
height: params.imageHeight,
|
||||
});
|
||||
if (
|
||||
pixelZone.width >= params.minimumWidth + 12 &&
|
||||
pixelZone.height >= params.minimumHeight + 12
|
||||
) {
|
||||
return { key, zone: pixelZone };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function moveGroupToZone(params: {
|
||||
frames: Array<{ id: string; x: number; y: number; width: number; height: number }>;
|
||||
zone: { x: number; y: number; width: number; height: number };
|
||||
align: "start" | "end" | "center";
|
||||
}) {
|
||||
const minX = Math.min(...params.frames.map((frame) => frame.x));
|
||||
const minY = Math.min(...params.frames.map((frame) => frame.y));
|
||||
const maxX = Math.max(...params.frames.map((frame) => frame.x + frame.width));
|
||||
const maxY = Math.max(...params.frames.map((frame) => frame.y + frame.height));
|
||||
const groupWidth = maxX - minX;
|
||||
const groupHeight = maxY - minY;
|
||||
|
||||
let targetX = params.zone.x + 8;
|
||||
if (params.align === "end") {
|
||||
targetX = params.zone.x + Math.max(8, params.zone.width - groupWidth - 8);
|
||||
} else if (params.align === "center") {
|
||||
targetX = params.zone.x + Math.max(0, (params.zone.width - groupWidth) / 2);
|
||||
}
|
||||
|
||||
const targetY = params.zone.y + 8;
|
||||
const dx = targetX - minX;
|
||||
const dy = targetY - minY;
|
||||
|
||||
return new Map(
|
||||
params.frames.map((frame) => [
|
||||
frame.id,
|
||||
{
|
||||
x: frame.x + dx,
|
||||
y: frame.y + dy,
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
},
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
function buildFrameOverrides(params: {
|
||||
overlay: DetailUiScreenOverlaySnapshot;
|
||||
components: RenderableOverlayComponent[];
|
||||
subjectAnalysis?: DetailUiSubjectAnalysis | null;
|
||||
imageWidth: number;
|
||||
imageHeight: number;
|
||||
}) {
|
||||
if (!params.subjectAnalysis) return new Map<string, { x: number; y: number; width: number; height: number }>();
|
||||
|
||||
const pixelFrames = params.components.map((component) => {
|
||||
const frame = (component.layout?.frame || {}) as {
|
||||
x?: number;
|
||||
y?: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
};
|
||||
return {
|
||||
id: component.id,
|
||||
role: String(component.metadata?.componentRole || ""),
|
||||
type: component.type,
|
||||
x: (Number(frame.x || 0) / params.overlay.screenSize.width) * params.imageWidth,
|
||||
y: (Number(frame.y || 0) / params.overlay.screenSize.height) * params.imageHeight,
|
||||
width: (Number(frame.width || 0) / params.overlay.screenSize.width) * params.imageWidth,
|
||||
height: (Number(frame.height || 0) / params.overlay.screenSize.height) * params.imageHeight,
|
||||
};
|
||||
});
|
||||
|
||||
const heroFrames = pixelFrames.filter((frame) =>
|
||||
["badge", "title", "subtitle"].includes(frame.role),
|
||||
);
|
||||
const supportFrames = pixelFrames.filter((frame) =>
|
||||
[
|
||||
"feature_tag",
|
||||
"support_card",
|
||||
"spec_table_section",
|
||||
"table_label",
|
||||
"table_value",
|
||||
"comparison_before",
|
||||
"comparison_after",
|
||||
"proof_bar",
|
||||
].includes(frame.role),
|
||||
);
|
||||
const ctaFrames = pixelFrames.filter((frame) =>
|
||||
["cta_support", "cta_button", "cta_banner"].includes(frame.role),
|
||||
);
|
||||
|
||||
const overrides = new Map<string, { x: number; y: number; width: number; height: number }>();
|
||||
|
||||
const heroPreferred =
|
||||
params.subjectAnalysis.dominantHorizontal === "left"
|
||||
? ["right", "top", "bottom", "left"]
|
||||
: params.subjectAnalysis.dominantHorizontal === "right"
|
||||
? ["left", "top", "bottom", "right"]
|
||||
: ["top", "left", "right", "bottom"];
|
||||
const supportPreferred =
|
||||
params.subjectAnalysis.dominantVertical === "bottom"
|
||||
? ["top", "left", "right", "bottom"]
|
||||
: ["bottom", "right", "left", "top"];
|
||||
const ctaPreferred = ["bottom", "right", "left", "top"] as const;
|
||||
|
||||
if (heroFrames.length > 0) {
|
||||
const groupWidth = Math.max(...heroFrames.map((frame) => frame.x + frame.width)) - Math.min(...heroFrames.map((frame) => frame.x));
|
||||
const groupHeight = Math.max(...heroFrames.map((frame) => frame.y + frame.height)) - Math.min(...heroFrames.map((frame) => frame.y));
|
||||
const zone = chooseGroupZone({
|
||||
subjectAnalysis: params.subjectAnalysis,
|
||||
imageWidth: params.imageWidth,
|
||||
imageHeight: params.imageHeight,
|
||||
preferred: heroPreferred as any,
|
||||
minimumWidth: groupWidth,
|
||||
minimumHeight: groupHeight,
|
||||
});
|
||||
if (zone) {
|
||||
const moved = moveGroupToZone({
|
||||
frames: heroFrames,
|
||||
zone: zone.zone,
|
||||
align: zone.key === "right" ? "end" : zone.key === "top" ? "center" : "start",
|
||||
});
|
||||
moved.forEach((frame, id) => overrides.set(id, frame));
|
||||
}
|
||||
}
|
||||
|
||||
if (supportFrames.length > 0) {
|
||||
const groupWidth = Math.max(...supportFrames.map((frame) => frame.x + frame.width)) - Math.min(...supportFrames.map((frame) => frame.x));
|
||||
const groupHeight = Math.max(...supportFrames.map((frame) => frame.y + frame.height)) - Math.min(...supportFrames.map((frame) => frame.y));
|
||||
const zone = chooseGroupZone({
|
||||
subjectAnalysis: params.subjectAnalysis,
|
||||
imageWidth: params.imageWidth,
|
||||
imageHeight: params.imageHeight,
|
||||
preferred: supportPreferred as any,
|
||||
minimumWidth: groupWidth,
|
||||
minimumHeight: groupHeight,
|
||||
});
|
||||
if (zone) {
|
||||
const moved = moveGroupToZone({
|
||||
frames: supportFrames,
|
||||
zone: zone.zone,
|
||||
align: zone.key === "right" ? "end" : "start",
|
||||
});
|
||||
moved.forEach((frame, id) => overrides.set(id, frame));
|
||||
}
|
||||
}
|
||||
|
||||
if (ctaFrames.length > 0) {
|
||||
const groupWidth = Math.max(...ctaFrames.map((frame) => frame.x + frame.width)) - Math.min(...ctaFrames.map((frame) => frame.x));
|
||||
const groupHeight = Math.max(...ctaFrames.map((frame) => frame.y + frame.height)) - Math.min(...ctaFrames.map((frame) => frame.y));
|
||||
const zone = chooseGroupZone({
|
||||
subjectAnalysis: params.subjectAnalysis,
|
||||
imageWidth: params.imageWidth,
|
||||
imageHeight: params.imageHeight,
|
||||
preferred: [...ctaPreferred] as any,
|
||||
minimumWidth: groupWidth,
|
||||
minimumHeight: groupHeight,
|
||||
});
|
||||
if (zone) {
|
||||
const moved = moveGroupToZone({
|
||||
frames: ctaFrames,
|
||||
zone: zone.zone,
|
||||
align: zone.key === "right" ? "end" : zone.key === "top" ? "center" : "start",
|
||||
});
|
||||
moved.forEach((frame, id) => overrides.set(id, frame));
|
||||
}
|
||||
}
|
||||
|
||||
return overrides;
|
||||
}
|
||||
|
||||
export function DetailSurfaceOverlayRenderer(params: {
|
||||
overlay: DetailUiScreenOverlaySnapshot | null;
|
||||
imageMetrics?: {
|
||||
@ -241,12 +452,28 @@ export function DetailSurfaceOverlayRenderer(params: {
|
||||
renderedWidth: number;
|
||||
renderedHeight: number;
|
||||
} | null;
|
||||
subjectAnalysis?: DetailUiSubjectAnalysis | null;
|
||||
className?: string;
|
||||
}) {
|
||||
if (!params.overlay) return null;
|
||||
|
||||
const components = collectRenderableComponents(params.overlay);
|
||||
if (components.length === 0) return null;
|
||||
const overlayWidth =
|
||||
params.imageMetrics && params.imageMetrics.renderedWidth > 0
|
||||
? params.imageMetrics.renderedWidth
|
||||
: params.overlay.screenSize.width;
|
||||
const overlayHeight =
|
||||
params.imageMetrics && params.imageMetrics.renderedHeight > 0
|
||||
? params.imageMetrics.renderedHeight
|
||||
: params.overlay.screenSize.height;
|
||||
const frameOverrides = buildFrameOverrides({
|
||||
overlay: params.overlay,
|
||||
components,
|
||||
subjectAnalysis: params.subjectAnalysis,
|
||||
imageWidth: overlayWidth,
|
||||
imageHeight: overlayHeight,
|
||||
});
|
||||
|
||||
const containerStyle =
|
||||
params.imageMetrics && params.imageMetrics.renderedWidth > 0 && params.imageMetrics.renderedHeight > 0
|
||||
@ -281,6 +508,16 @@ export function DetailSurfaceOverlayRenderer(params: {
|
||||
component,
|
||||
snapshot: params.overlay!,
|
||||
content,
|
||||
frameOverride: (() => {
|
||||
const frame = frameOverrides.get(component.id);
|
||||
if (!frame) return undefined;
|
||||
return {
|
||||
x: (frame.x / overlayWidth) * params.overlay!.screenSize.width,
|
||||
y: (frame.y / overlayHeight) * params.overlay!.screenSize.height,
|
||||
width: (frame.width / overlayWidth) * params.overlay!.screenSize.width,
|
||||
height: (frame.height / overlayHeight) * params.overlay!.screenSize.height,
|
||||
};
|
||||
})(),
|
||||
})}
|
||||
>
|
||||
{renderOverlayComponentContent(params.overlay!, component)}
|
||||
|
||||
@ -38,6 +38,10 @@ import { AgentExecutionSnapshotDialog } from "@/components/agents/AgentExecution
|
||||
import { DetailSurfaceOverlayRenderer } from "@/components/canvas/DetailSurfaceOverlayRenderer";
|
||||
import type { PickedObject } from "@/store/canvas";
|
||||
import type { DetailUiScreenOverlaySnapshot } from "@/lib/detail-ui-surface-adapter";
|
||||
import {
|
||||
normalizeDetailUiSubjectAnalysis,
|
||||
type DetailUiSubjectAnalysis,
|
||||
} from "@/lib/detail-ui-subject-analysis";
|
||||
|
||||
// 已标注的物品标记(本地状态)
|
||||
interface PickedMarker {
|
||||
@ -374,6 +378,9 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
|
||||
return raw as DetailUiScreenOverlaySnapshot;
|
||||
}, [data.metadata]);
|
||||
const detailSubjectAnalysis = useMemo(() => {
|
||||
return normalizeDetailUiSubjectAnalysis(data.metadata?.detailSubjectAnalysis);
|
||||
}, [data.metadata]);
|
||||
const shouldRenderDetailOverlay =
|
||||
data.metadata?.detailGenerationMode === "agent_layout" && Boolean(detailUiScreenOverlay);
|
||||
const shortAgentExecutionSnapshotId = agentExecutionSnapshotId
|
||||
@ -2541,6 +2548,7 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
<DetailSurfaceOverlayRenderer
|
||||
overlay={detailUiScreenOverlay}
|
||||
imageMetrics={boxCutoutMetrics}
|
||||
subjectAnalysis={detailSubjectAnalysis}
|
||||
/>
|
||||
) : null}
|
||||
</>
|
||||
|
||||
91
lib/detail-ui-subject-analysis.ts
Normal file
91
lib/detail-ui-subject-analysis.ts
Normal file
@ -0,0 +1,91 @@
|
||||
export interface DetailUiSubjectBox {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
confidence?: number;
|
||||
}
|
||||
|
||||
export interface DetailUiSafeZone {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface DetailUiSubjectAnalysis {
|
||||
subjectBox: DetailUiSubjectBox;
|
||||
safeZones: {
|
||||
left?: DetailUiSafeZone | null;
|
||||
right?: DetailUiSafeZone | null;
|
||||
top?: DetailUiSafeZone | null;
|
||||
bottom?: DetailUiSafeZone | null;
|
||||
};
|
||||
dominantHorizontal: "left" | "right" | "center";
|
||||
dominantVertical: "top" | "bottom" | "center";
|
||||
}
|
||||
|
||||
export function clampNormalized(value: number) {
|
||||
if (!Number.isFinite(value)) return 0;
|
||||
return Math.max(0, Math.min(1, value));
|
||||
}
|
||||
|
||||
export function normalizeSafeZone(zone?: Partial<DetailUiSafeZone> | null) {
|
||||
if (!zone) return null;
|
||||
const width = clampNormalized(Number(zone.width || 0));
|
||||
const height = clampNormalized(Number(zone.height || 0));
|
||||
if (width <= 0 || height <= 0) return null;
|
||||
|
||||
return {
|
||||
x: clampNormalized(Number(zone.x || 0)),
|
||||
y: clampNormalized(Number(zone.y || 0)),
|
||||
width,
|
||||
height,
|
||||
} satisfies DetailUiSafeZone;
|
||||
}
|
||||
|
||||
export function normalizeDetailUiSubjectAnalysis(
|
||||
value: unknown,
|
||||
): DetailUiSubjectAnalysis | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
const raw = value as Record<string, unknown>;
|
||||
const subjectRaw =
|
||||
raw.subjectBox && typeof raw.subjectBox === "object" && !Array.isArray(raw.subjectBox)
|
||||
? (raw.subjectBox as Record<string, unknown>)
|
||||
: null;
|
||||
if (!subjectRaw) return null;
|
||||
|
||||
const subjectBox = {
|
||||
x: clampNormalized(Number(subjectRaw.x || 0)),
|
||||
y: clampNormalized(Number(subjectRaw.y || 0)),
|
||||
width: clampNormalized(Number(subjectRaw.width || 0)),
|
||||
height: clampNormalized(Number(subjectRaw.height || 0)),
|
||||
confidence: Number.isFinite(Number(subjectRaw.confidence))
|
||||
? Math.max(0, Math.min(1, Number(subjectRaw.confidence)))
|
||||
: undefined,
|
||||
} satisfies DetailUiSubjectBox;
|
||||
|
||||
if (subjectBox.width <= 0 || subjectBox.height <= 0) return null;
|
||||
|
||||
const zonesRaw =
|
||||
raw.safeZones && typeof raw.safeZones === "object" && !Array.isArray(raw.safeZones)
|
||||
? (raw.safeZones as Record<string, unknown>)
|
||||
: {};
|
||||
|
||||
const centerX = subjectBox.x + subjectBox.width / 2;
|
||||
const centerY = subjectBox.y + subjectBox.height / 2;
|
||||
|
||||
return {
|
||||
subjectBox,
|
||||
safeZones: {
|
||||
left: normalizeSafeZone(zonesRaw.left as any),
|
||||
right: normalizeSafeZone(zonesRaw.right as any),
|
||||
top: normalizeSafeZone(zonesRaw.top as any),
|
||||
bottom: normalizeSafeZone(zonesRaw.bottom as any),
|
||||
},
|
||||
dominantHorizontal:
|
||||
centerX < 0.42 ? "left" : centerX > 0.58 ? "right" : "center",
|
||||
dominantVertical:
|
||||
centerY < 0.42 ? "top" : centerY > 0.58 ? "bottom" : "center",
|
||||
} satisfies DetailUiSubjectAnalysis;
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user