feat: add subject-aware detail overlay layout
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-03-22 03:00:56 +08:00
parent 139b14b3c6
commit 409770d386
5 changed files with 596 additions and 1 deletions

View File

@ -0,0 +1,233 @@
import { NextResponse } from "next/server";
import sharp from "sharp";
import { auth } from "@/app/(auth)/auth";
import {
clampNormalized,
type DetailUiSubjectAnalysis,
} from "@/lib/detail-ui-subject-analysis";
export const runtime = "nodejs";
export const maxDuration = 30;
const ANALYZE_MAX_SIDE = 128;
const ANALYZE_FETCH_TIMEOUT_MS = 12000;
function percentile(values: number[], ratio: number) {
if (values.length === 0) return 0;
const sorted = [...values].sort((left, right) => left - right);
const index = Math.max(0, Math.min(sorted.length - 1, Math.floor(sorted.length * ratio)));
return sorted[index] || 0;
}
function buildSafeZones(params: {
subjectBox: { x: number; y: number; width: number; height: number };
}) {
const gap = 0.035;
const leftWidth = Math.max(0, params.subjectBox.x - gap);
const rightX = clampNormalized(params.subjectBox.x + params.subjectBox.width + gap);
const rightWidth = Math.max(0, 1 - rightX);
const topHeight = Math.max(0, params.subjectBox.y - gap);
const bottomY = clampNormalized(params.subjectBox.y + params.subjectBox.height + gap);
const bottomHeight = Math.max(0, 1 - bottomY);
return {
left:
leftWidth > 0.16
? {
x: 0,
y: 0,
width: leftWidth,
height: 1,
}
: null,
right:
rightWidth > 0.16
? {
x: rightX,
y: 0,
width: rightWidth,
height: 1,
}
: null,
top:
topHeight > 0.16
? {
x: 0,
y: 0,
width: 1,
height: topHeight,
}
: null,
bottom:
bottomHeight > 0.14
? {
x: 0,
y: bottomY,
width: 1,
height: bottomHeight,
}
: null,
};
}
function analyzeSubjectRegion(params: {
grayscale: Uint8Array;
width: number;
height: number;
}): DetailUiSubjectAnalysis {
const scores: number[] = [];
const scoreGrid = new Float32Array(params.width * params.height);
const pixelAt = (x: number, y: number) =>
params.grayscale[Math.max(0, Math.min(params.grayscale.length - 1, y * params.width + x))] || 0;
for (let y = 1; y < params.height - 1; y += 1) {
for (let x = 1; x < params.width - 1; x += 1) {
const gx =
-1 * pixelAt(x - 1, y - 1) +
1 * pixelAt(x + 1, y - 1) +
-2 * pixelAt(x - 1, y) +
2 * pixelAt(x + 1, y) +
-1 * pixelAt(x - 1, y + 1) +
1 * pixelAt(x + 1, y + 1);
const gy =
-1 * pixelAt(x - 1, y - 1) +
-2 * pixelAt(x, y - 1) +
-1 * pixelAt(x + 1, y - 1) +
1 * pixelAt(x - 1, y + 1) +
2 * pixelAt(x, y + 1) +
1 * pixelAt(x + 1, y + 1);
const edge = Math.sqrt(gx * gx + gy * gy);
const centerBiasX = 1 - Math.abs(x / params.width - 0.5) * 1.5;
const centerBiasY = 1 - Math.abs(y / params.height - 0.56) * 1.3;
const centerBias = Math.max(0.15, centerBiasX * centerBiasY);
const brightness = pixelAt(x, y) / 255;
const contrast = Math.abs(brightness - 0.5);
const score = edge * 0.72 + contrast * 60 + centerBias * 90;
const index = y * params.width + x;
scoreGrid[index] = score;
scores.push(score);
}
}
const threshold = percentile(scores, 0.84);
let minX = params.width;
let minY = params.height;
let maxX = 0;
let maxY = 0;
for (let y = 0; y < params.height; y += 1) {
for (let x = 0; x < params.width; x += 1) {
const score = scoreGrid[y * params.width + x];
if (score < threshold) continue;
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x);
maxY = Math.max(maxY, y);
}
}
if (minX >= maxX || minY >= maxY) {
minX = Math.round(params.width * 0.24);
maxX = Math.round(params.width * 0.78);
minY = Math.round(params.height * 0.18);
maxY = Math.round(params.height * 0.92);
}
const expandX = Math.round((maxX - minX) * 0.08);
const expandY = Math.round((maxY - minY) * 0.08);
minX = Math.max(0, minX - expandX);
minY = Math.max(0, minY - expandY);
maxX = Math.min(params.width - 1, maxX + expandX);
maxY = Math.min(params.height - 1, maxY + expandY);
const subjectBox = {
x: clampNormalized(minX / params.width),
y: clampNormalized(minY / params.height),
width: clampNormalized((maxX - minX) / params.width),
height: clampNormalized((maxY - minY) / params.height),
confidence: 0.72,
};
return {
subjectBox,
safeZones: buildSafeZones({ subjectBox }),
dominantHorizontal:
subjectBox.x + subjectBox.width / 2 < 0.42
? "left"
: subjectBox.x + subjectBox.width / 2 > 0.58
? "right"
: "center",
dominantVertical:
subjectBox.y + subjectBox.height / 2 < 0.42
? "top"
: subjectBox.y + subjectBox.height / 2 > 0.58
? "bottom"
: "center",
};
}
export async function POST(request: Request) {
try {
const session = await auth();
if (!session?.user?.id) {
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
}
const body = (await request.json()) as { imageUrl?: string };
const imageUrl = String(body.imageUrl || "").trim();
if (!/^https?:\/\//i.test(imageUrl)) {
return NextResponse.json({ success: false, error: "缺少图片地址" }, { status: 400 });
}
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), ANALYZE_FETCH_TIMEOUT_MS);
let buffer: Buffer;
try {
const response = await fetch(imageUrl, {
method: "GET",
signal: controller.signal,
});
if (!response.ok) {
throw new Error(`图片拉取失败: HTTP ${response.status}`);
}
buffer = Buffer.from(await response.arrayBuffer());
} finally {
clearTimeout(timeout);
}
const { data, info } = await sharp(buffer)
.rotate()
.resize({
width: ANALYZE_MAX_SIDE,
height: ANALYZE_MAX_SIDE,
fit: "inside",
withoutEnlargement: true,
})
.grayscale()
.raw()
.toBuffer({ resolveWithObject: true });
const analysis = analyzeSubjectRegion({
grayscale: data,
width: info.width || ANALYZE_MAX_SIDE,
height: info.height || ANALYZE_MAX_SIDE,
});
return NextResponse.json({
success: true,
data: analysis,
});
} catch (error) {
console.error("[Canvas Detail Subject Analyze] Error:", error);
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : "主体分析失败",
},
{ status: 500 },
);
}
}

View File

@ -6307,6 +6307,26 @@ export function CanvasChatPanel({
const preflightLabel = planData.preflight?.rewriteApplied
? "双总监预审重写完成"
: "双总监预审通过";
const requestDetailSubjectAnalysis = async (imageUrl?: string) => {
const normalized = String(imageUrl || "").trim();
if (!/^https?:\/\//i.test(normalized)) return null;
try {
const response = await fetch("/api/canvas/detail-subject-analyze", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ imageUrl: normalized }),
});
const result = await response.json().catch(() => ({}));
if (!response.ok || !result?.success) {
return null;
}
return result.data || null;
} catch (error) {
console.warn("[CanvasChat] Detail subject analyze failed:", error);
return null;
}
};
for (let index = 0; index < prompts.length; index += 1) {
patchDetailPlaceholder(index, {
label: preflightLabel,
@ -6711,6 +6731,11 @@ export function CanvasChatPanel({
const dimensions = await getImageDimensions(
finalGenerated.originalUrl || finalGenerated.thumbnailUrl
);
const detailSubjectAnalysis = isAgentLayoutMode
? await requestDetailSubjectAnalysis(
finalGenerated.originalUrl || finalGenerated.thumbnailUrl,
)
: null;
if (targetNodeId) {
const storeState = useCanvasStore.getState();
const currentNode = storeState.getNode(targetNodeId);
@ -6752,6 +6777,7 @@ export function CanvasChatPanel({
surface: currentDetailSurfaceSnapshot,
screenIndex: index + 1,
}),
detailSubjectAnalysis,
detailGeneratingPlaceholder: false,
detailProgressLabel: `该屏完成 ${index + 1}/${prompts.length}`,
},

View File

@ -4,6 +4,10 @@ import type { CSSProperties } from "react";
import { ArrowRight, CheckCircle2, ShieldCheck, Sparkles } from "lucide-react";
import { cn } from "@/lib/utils";
import type { DetailUiScreenOverlaySnapshot } from "@/lib/detail-ui-surface-adapter";
import type {
DetailUiSafeZone,
DetailUiSubjectAnalysis,
} from "@/lib/detail-ui-subject-analysis";
type RenderableOverlayComponent =
DetailUiScreenOverlaySnapshot["components"][number];
@ -147,8 +151,14 @@ function getOverlayComponentStyle(params: {
component: RenderableOverlayComponent;
snapshot: DetailUiScreenOverlaySnapshot;
content: string;
frameOverride?: {
x: number;
y: number;
width: number;
height: number;
};
}) {
const frame = (params.component.layout?.frame || {}) as {
const frame = (params.frameOverride || params.component.layout?.frame || {}) as {
x?: number;
y?: number;
width?: number;
@ -233,6 +243,207 @@ function getOverlayComponentStyle(params: {
return overlayStyle;
}
function normalizeZoneToPixels(params: {
zone: DetailUiSafeZone;
width: number;
height: number;
}) {
return {
x: params.zone.x * params.width,
y: params.zone.y * params.height,
width: params.zone.width * params.width,
height: params.zone.height * params.height,
};
}
function chooseGroupZone(params: {
subjectAnalysis: DetailUiSubjectAnalysis;
imageWidth: number;
imageHeight: number;
preferred: Array<"left" | "right" | "top" | "bottom">;
minimumWidth: number;
minimumHeight: number;
}) {
for (const key of params.preferred) {
const zone = params.subjectAnalysis.safeZones[key];
if (!zone) continue;
const pixelZone = normalizeZoneToPixels({
zone,
width: params.imageWidth,
height: params.imageHeight,
});
if (
pixelZone.width >= params.minimumWidth + 12 &&
pixelZone.height >= params.minimumHeight + 12
) {
return { key, zone: pixelZone };
}
}
return null;
}
function moveGroupToZone(params: {
frames: Array<{ id: string; x: number; y: number; width: number; height: number }>;
zone: { x: number; y: number; width: number; height: number };
align: "start" | "end" | "center";
}) {
const minX = Math.min(...params.frames.map((frame) => frame.x));
const minY = Math.min(...params.frames.map((frame) => frame.y));
const maxX = Math.max(...params.frames.map((frame) => frame.x + frame.width));
const maxY = Math.max(...params.frames.map((frame) => frame.y + frame.height));
const groupWidth = maxX - minX;
const groupHeight = maxY - minY;
let targetX = params.zone.x + 8;
if (params.align === "end") {
targetX = params.zone.x + Math.max(8, params.zone.width - groupWidth - 8);
} else if (params.align === "center") {
targetX = params.zone.x + Math.max(0, (params.zone.width - groupWidth) / 2);
}
const targetY = params.zone.y + 8;
const dx = targetX - minX;
const dy = targetY - minY;
return new Map(
params.frames.map((frame) => [
frame.id,
{
x: frame.x + dx,
y: frame.y + dy,
width: frame.width,
height: frame.height,
},
]),
);
}
function buildFrameOverrides(params: {
overlay: DetailUiScreenOverlaySnapshot;
components: RenderableOverlayComponent[];
subjectAnalysis?: DetailUiSubjectAnalysis | null;
imageWidth: number;
imageHeight: number;
}) {
if (!params.subjectAnalysis) return new Map<string, { x: number; y: number; width: number; height: number }>();
const pixelFrames = params.components.map((component) => {
const frame = (component.layout?.frame || {}) as {
x?: number;
y?: number;
width?: number;
height?: number;
};
return {
id: component.id,
role: String(component.metadata?.componentRole || ""),
type: component.type,
x: (Number(frame.x || 0) / params.overlay.screenSize.width) * params.imageWidth,
y: (Number(frame.y || 0) / params.overlay.screenSize.height) * params.imageHeight,
width: (Number(frame.width || 0) / params.overlay.screenSize.width) * params.imageWidth,
height: (Number(frame.height || 0) / params.overlay.screenSize.height) * params.imageHeight,
};
});
const heroFrames = pixelFrames.filter((frame) =>
["badge", "title", "subtitle"].includes(frame.role),
);
const supportFrames = pixelFrames.filter((frame) =>
[
"feature_tag",
"support_card",
"spec_table_section",
"table_label",
"table_value",
"comparison_before",
"comparison_after",
"proof_bar",
].includes(frame.role),
);
const ctaFrames = pixelFrames.filter((frame) =>
["cta_support", "cta_button", "cta_banner"].includes(frame.role),
);
const overrides = new Map<string, { x: number; y: number; width: number; height: number }>();
const heroPreferred =
params.subjectAnalysis.dominantHorizontal === "left"
? ["right", "top", "bottom", "left"]
: params.subjectAnalysis.dominantHorizontal === "right"
? ["left", "top", "bottom", "right"]
: ["top", "left", "right", "bottom"];
const supportPreferred =
params.subjectAnalysis.dominantVertical === "bottom"
? ["top", "left", "right", "bottom"]
: ["bottom", "right", "left", "top"];
const ctaPreferred = ["bottom", "right", "left", "top"] as const;
if (heroFrames.length > 0) {
const groupWidth = Math.max(...heroFrames.map((frame) => frame.x + frame.width)) - Math.min(...heroFrames.map((frame) => frame.x));
const groupHeight = Math.max(...heroFrames.map((frame) => frame.y + frame.height)) - Math.min(...heroFrames.map((frame) => frame.y));
const zone = chooseGroupZone({
subjectAnalysis: params.subjectAnalysis,
imageWidth: params.imageWidth,
imageHeight: params.imageHeight,
preferred: heroPreferred as any,
minimumWidth: groupWidth,
minimumHeight: groupHeight,
});
if (zone) {
const moved = moveGroupToZone({
frames: heroFrames,
zone: zone.zone,
align: zone.key === "right" ? "end" : zone.key === "top" ? "center" : "start",
});
moved.forEach((frame, id) => overrides.set(id, frame));
}
}
if (supportFrames.length > 0) {
const groupWidth = Math.max(...supportFrames.map((frame) => frame.x + frame.width)) - Math.min(...supportFrames.map((frame) => frame.x));
const groupHeight = Math.max(...supportFrames.map((frame) => frame.y + frame.height)) - Math.min(...supportFrames.map((frame) => frame.y));
const zone = chooseGroupZone({
subjectAnalysis: params.subjectAnalysis,
imageWidth: params.imageWidth,
imageHeight: params.imageHeight,
preferred: supportPreferred as any,
minimumWidth: groupWidth,
minimumHeight: groupHeight,
});
if (zone) {
const moved = moveGroupToZone({
frames: supportFrames,
zone: zone.zone,
align: zone.key === "right" ? "end" : "start",
});
moved.forEach((frame, id) => overrides.set(id, frame));
}
}
if (ctaFrames.length > 0) {
const groupWidth = Math.max(...ctaFrames.map((frame) => frame.x + frame.width)) - Math.min(...ctaFrames.map((frame) => frame.x));
const groupHeight = Math.max(...ctaFrames.map((frame) => frame.y + frame.height)) - Math.min(...ctaFrames.map((frame) => frame.y));
const zone = chooseGroupZone({
subjectAnalysis: params.subjectAnalysis,
imageWidth: params.imageWidth,
imageHeight: params.imageHeight,
preferred: [...ctaPreferred] as any,
minimumWidth: groupWidth,
minimumHeight: groupHeight,
});
if (zone) {
const moved = moveGroupToZone({
frames: ctaFrames,
zone: zone.zone,
align: zone.key === "right" ? "end" : zone.key === "top" ? "center" : "start",
});
moved.forEach((frame, id) => overrides.set(id, frame));
}
}
return overrides;
}
export function DetailSurfaceOverlayRenderer(params: {
overlay: DetailUiScreenOverlaySnapshot | null;
imageMetrics?: {
@ -241,12 +452,28 @@ export function DetailSurfaceOverlayRenderer(params: {
renderedWidth: number;
renderedHeight: number;
} | null;
subjectAnalysis?: DetailUiSubjectAnalysis | null;
className?: string;
}) {
if (!params.overlay) return null;
const components = collectRenderableComponents(params.overlay);
if (components.length === 0) return null;
const overlayWidth =
params.imageMetrics && params.imageMetrics.renderedWidth > 0
? params.imageMetrics.renderedWidth
: params.overlay.screenSize.width;
const overlayHeight =
params.imageMetrics && params.imageMetrics.renderedHeight > 0
? params.imageMetrics.renderedHeight
: params.overlay.screenSize.height;
const frameOverrides = buildFrameOverrides({
overlay: params.overlay,
components,
subjectAnalysis: params.subjectAnalysis,
imageWidth: overlayWidth,
imageHeight: overlayHeight,
});
const containerStyle =
params.imageMetrics && params.imageMetrics.renderedWidth > 0 && params.imageMetrics.renderedHeight > 0
@ -281,6 +508,16 @@ export function DetailSurfaceOverlayRenderer(params: {
component,
snapshot: params.overlay!,
content,
frameOverride: (() => {
const frame = frameOverrides.get(component.id);
if (!frame) return undefined;
return {
x: (frame.x / overlayWidth) * params.overlay!.screenSize.width,
y: (frame.y / overlayHeight) * params.overlay!.screenSize.height,
width: (frame.width / overlayWidth) * params.overlay!.screenSize.width,
height: (frame.height / overlayHeight) * params.overlay!.screenSize.height,
};
})(),
})}
>
{renderOverlayComponentContent(params.overlay!, component)}

View File

@ -38,6 +38,10 @@ import { AgentExecutionSnapshotDialog } from "@/components/agents/AgentExecution
import { DetailSurfaceOverlayRenderer } from "@/components/canvas/DetailSurfaceOverlayRenderer";
import type { PickedObject } from "@/store/canvas";
import type { DetailUiScreenOverlaySnapshot } from "@/lib/detail-ui-surface-adapter";
import {
normalizeDetailUiSubjectAnalysis,
type DetailUiSubjectAnalysis,
} from "@/lib/detail-ui-subject-analysis";
// 已标注的物品标记(本地状态)
interface PickedMarker {
@ -374,6 +378,9 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
return raw as DetailUiScreenOverlaySnapshot;
}, [data.metadata]);
const detailSubjectAnalysis = useMemo(() => {
return normalizeDetailUiSubjectAnalysis(data.metadata?.detailSubjectAnalysis);
}, [data.metadata]);
const shouldRenderDetailOverlay =
data.metadata?.detailGenerationMode === "agent_layout" && Boolean(detailUiScreenOverlay);
const shortAgentExecutionSnapshotId = agentExecutionSnapshotId
@ -2541,6 +2548,7 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
<DetailSurfaceOverlayRenderer
overlay={detailUiScreenOverlay}
imageMetrics={boxCutoutMetrics}
subjectAnalysis={detailSubjectAnalysis}
/>
) : null}
</>

View File

@ -0,0 +1,91 @@
export interface DetailUiSubjectBox {
x: number;
y: number;
width: number;
height: number;
confidence?: number;
}
export interface DetailUiSafeZone {
x: number;
y: number;
width: number;
height: number;
}
export interface DetailUiSubjectAnalysis {
subjectBox: DetailUiSubjectBox;
safeZones: {
left?: DetailUiSafeZone | null;
right?: DetailUiSafeZone | null;
top?: DetailUiSafeZone | null;
bottom?: DetailUiSafeZone | null;
};
dominantHorizontal: "left" | "right" | "center";
dominantVertical: "top" | "bottom" | "center";
}
export function clampNormalized(value: number) {
if (!Number.isFinite(value)) return 0;
return Math.max(0, Math.min(1, value));
}
export function normalizeSafeZone(zone?: Partial<DetailUiSafeZone> | null) {
if (!zone) return null;
const width = clampNormalized(Number(zone.width || 0));
const height = clampNormalized(Number(zone.height || 0));
if (width <= 0 || height <= 0) return null;
return {
x: clampNormalized(Number(zone.x || 0)),
y: clampNormalized(Number(zone.y || 0)),
width,
height,
} satisfies DetailUiSafeZone;
}
export function normalizeDetailUiSubjectAnalysis(
value: unknown,
): DetailUiSubjectAnalysis | null {
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
const raw = value as Record<string, unknown>;
const subjectRaw =
raw.subjectBox && typeof raw.subjectBox === "object" && !Array.isArray(raw.subjectBox)
? (raw.subjectBox as Record<string, unknown>)
: null;
if (!subjectRaw) return null;
const subjectBox = {
x: clampNormalized(Number(subjectRaw.x || 0)),
y: clampNormalized(Number(subjectRaw.y || 0)),
width: clampNormalized(Number(subjectRaw.width || 0)),
height: clampNormalized(Number(subjectRaw.height || 0)),
confidence: Number.isFinite(Number(subjectRaw.confidence))
? Math.max(0, Math.min(1, Number(subjectRaw.confidence)))
: undefined,
} satisfies DetailUiSubjectBox;
if (subjectBox.width <= 0 || subjectBox.height <= 0) return null;
const zonesRaw =
raw.safeZones && typeof raw.safeZones === "object" && !Array.isArray(raw.safeZones)
? (raw.safeZones as Record<string, unknown>)
: {};
const centerX = subjectBox.x + subjectBox.width / 2;
const centerY = subjectBox.y + subjectBox.height / 2;
return {
subjectBox,
safeZones: {
left: normalizeSafeZone(zonesRaw.left as any),
right: normalizeSafeZone(zonesRaw.right as any),
top: normalizeSafeZone(zonesRaw.top as any),
bottom: normalizeSafeZone(zonesRaw.bottom as any),
},
dominantHorizontal:
centerX < 0.42 ? "left" : centerX > 0.58 ? "right" : "center",
dominantVertical:
centerY < 0.42 ? "top" : centerY > 0.58 ? "bottom" : "center",
} satisfies DetailUiSubjectAnalysis;
}