feat: refine world snapshots with image generation
This commit is contained in:
Your Name
parent
c20dc8d3a1
commit
09800c313f
@ -75,6 +75,93 @@ const WORLD_ASSET_HYDRATION_INTERVAL_MS = 5000;
|
||||
const WORLD_ASSET_HYDRATION_MAX_ATTEMPTS = 60;
|
||||
const WORLD_GENERATION_RECOVERY_INTERVAL_MS = 5000;
|
||||
const WORLD_GENERATION_RECOVERY_MAX_ATTEMPTS = 360;
|
||||
const WORLD_SNAPSHOT_REPAINT_POLL_INTERVAL_MS = 1600;
|
||||
const WORLD_SNAPSHOT_REPAINT_TIMEOUT_MS = 4 * 60 * 1000;
|
||||
const WORLD_SNAPSHOT_REPAINT_PROMPT = [
|
||||
"以参考图中的 3D 世界拍照为严格构图和视角基础，生成一张真实摄影风格的场景图片。",
|
||||
"保持原始镜头位置、地形布局、道路/草地/树木/灌木/远景层次、光照方向和空间关系，不要改变场景类型。",
|
||||
"深度修复 3D Gaussian Splat 或 Mesh 截图中的模糊、破面、孔洞、黑边、拉伸网格、重复纹理、噪点、漂浮碎片和不自然阴影。",
|
||||
"补充真实世界细节：自然草叶、树干纹理、灌木叶片、道路材质、景深过渡、环境光和合理接触阴影。",
|
||||
"最终输出清晰、连贯、照片级、可信的真实场景图片。",
|
||||
].join("\n");
|
||||
const WORLD_SNAPSHOT_REPAINT_NEGATIVE_PROMPT = [
|
||||
"界面按钮",
|
||||
"UI",
|
||||
"文字",
|
||||
"水印",
|
||||
"logo",
|
||||
"准星",
|
||||
"操作提示",
|
||||
"3D 破面",
|
||||
"网格纹理",
|
||||
"孔洞",
|
||||
"黑边",
|
||||
"重复纹理",
|
||||
"扭曲树木",
|
||||
"漂浮碎片",
|
||||
"低清晰度",
|
||||
"卡通",
|
||||
"插画",
|
||||
].join(", ");
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolve) => window.setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function guessNearestAspectRatio(width: number, height: number) {
|
||||
const ratio = width > 0 && height > 0 ? width / height : 1;
|
||||
const candidates = [
|
||||
{ label: "1:1", value: 1 },
|
||||
{ label: "4:3", value: 4 / 3 },
|
||||
{ label: "3:4", value: 3 / 4 },
|
||||
{ label: "16:9", value: 16 / 9 },
|
||||
{ label: "9:16", value: 9 / 16 },
|
||||
{ label: "3:2", value: 3 / 2 },
|
||||
{ label: "2:3", value: 2 / 3 },
|
||||
];
|
||||
|
||||
return candidates.reduce((best, candidate) =>
|
||||
Math.abs(candidate.value - ratio) < Math.abs(best.value - ratio)
|
||||
? candidate
|
||||
: best
|
||||
).label;
|
||||
}
|
||||
|
||||
async function waitForWorldSnapshotRepaint(taskId: string) {
|
||||
const deadline = Date.now() + WORLD_SNAPSHOT_REPAINT_TIMEOUT_MS;
|
||||
let lastError = "";
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
await sleep(WORLD_SNAPSHOT_REPAINT_POLL_INTERVAL_MS);
|
||||
const taskResult = await imageGenerationService.getImageTask(taskId);
|
||||
const task = taskResult.data?.task;
|
||||
const status = String(task?.status || "").toLowerCase();
|
||||
|
||||
if (!taskResult.success || !task) {
|
||||
lastError = taskResult.error || "查询 AI 实景优化任务失败";
|
||||
continue;
|
||||
}
|
||||
|
||||
if (status === "succeeded" || status === "completed") {
|
||||
const image = task.result?.images?.[0];
|
||||
const imageUrl = image?.originalUrl || image?.thumbnailUrl || "";
|
||||
if (!imageUrl) {
|
||||
throw new Error("AI 实景优化已完成但未返回图片");
|
||||
}
|
||||
return {
|
||||
imageUrl,
|
||||
model: task.result?.model || task.result?.modelRef || undefined,
|
||||
prompt: task.result?.prompt || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
if (status === "failed") {
|
||||
throw new Error(String(task.error || "AI 实景优化失败"));
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(lastError || "AI 实景优化超时，请稍后重试");
|
||||
}
|
||||
|
||||
function resolveAvailableViews(data: World3DNodeData) {
|
||||
const views: World3DViewMode[] = [];
|
||||
@ -824,6 +911,7 @@ export function World3DNode({
|
||||
}
|
||||
|
||||
setIsCapturingSnapshot(true);
|
||||
let resultNodeId: string | null = null;
|
||||
try {
|
||||
const blob = await viewerCapture();
|
||||
if (!blob) {
|
||||
@ -843,8 +931,15 @@ export function World3DNode({
|
||||
220,
|
||||
Math.round(snapshotWidth * (node.size.height / Math.max(node.size.width, 1)))
|
||||
);
|
||||
const sourceSnapshotUrl = uploadResult.data.url;
|
||||
const titleBase = (data.prompt || "3D世界").trim() || "3D世界";
|
||||
const aspectRatio = guessNearestAspectRatio(snapshotWidth, snapshotHeight);
|
||||
const sourceWorldPrompt = String(data.prompt || "").trim();
|
||||
const refinePrompt = sourceWorldPrompt
|
||||
? `${WORLD_SNAPSHOT_REPAINT_PROMPT}\n原始世界需求：${sourceWorldPrompt}`
|
||||
: WORLD_SNAPSHOT_REPAINT_PROMPT;
|
||||
|
||||
const newNodeId = addNode({
|
||||
resultNodeId = addNode({
|
||||
type: "image",
|
||||
position: {
|
||||
x: node.position.x + node.size.width + 24,
|
||||
@ -852,25 +947,97 @@ export function World3DNode({
|
||||
},
|
||||
size: { width: snapshotWidth, height: snapshotHeight },
|
||||
data: {
|
||||
src: uploadResult.data.url,
|
||||
assetId: uploadResult.data.assetId,
|
||||
title: `${(data.prompt || "3D世界").trim() || "3D世界"}-拍照`,
|
||||
prompt: data.prompt || "3D 世界拍照",
|
||||
status: "completed",
|
||||
originalSrc: uploadResult.data.url,
|
||||
src: "",
|
||||
title: `${titleBase}-AI实景优化`,
|
||||
prompt: refinePrompt,
|
||||
status: "generating",
|
||||
metadata: {
|
||||
aspectRatio,
|
||||
captureMode: "world3d-ai-refine",
|
||||
sourceSnapshotAssetId: uploadResult.data.assetId,
|
||||
sourceSnapshotUrl,
|
||||
sourceWorldNodeId: node.id,
|
||||
sourceWorldPrompt,
|
||||
sourceWorldView: activeView,
|
||||
sourceWorldPrompt: data.prompt || "",
|
||||
captureMode: "world3d",
|
||||
},
|
||||
},
|
||||
connections: [node.id],
|
||||
});
|
||||
selectNode(newNodeId);
|
||||
toast.success("已将当前世界视角添加到画布");
|
||||
selectNode(resultNodeId);
|
||||
toast.success("已开始 AI 优化当前世界视角");
|
||||
|
||||
const generationResult = await imageGenerationService.generateImage({
|
||||
prompt: refinePrompt,
|
||||
negativePrompt: WORLD_SNAPSHOT_REPAINT_NEGATIVE_PROMPT,
|
||||
referenceImage: sourceSnapshotUrl,
|
||||
aspectRatio,
|
||||
imageSize: "2K",
|
||||
imageCount: 1,
|
||||
promptExtend: true,
|
||||
billingFeature: "repaint",
|
||||
async: true,
|
||||
});
|
||||
|
||||
if (!generationResult.success) {
|
||||
throw new Error(generationResult.error || "AI 实景优化提交失败");
|
||||
}
|
||||
|
||||
const immediateImage = generationResult.data?.images?.[0];
|
||||
const immediateUrl =
|
||||
immediateImage?.originalUrl || immediateImage?.thumbnailUrl || "";
|
||||
const finalResult = immediateUrl
|
||||
? {
|
||||
imageUrl: immediateUrl,
|
||||
model: generationResult.data?.model,
|
||||
prompt: generationResult.data?.prompt,
|
||||
}
|
||||
: generationResult.data?.taskId
|
||||
? await waitForWorldSnapshotRepaint(generationResult.data.taskId)
|
||||
: null;
|
||||
|
||||
if (!finalResult?.imageUrl) {
|
||||
throw new Error("AI 实景优化未返回图片");
|
||||
}
|
||||
|
||||
updateNode(resultNodeId, {
|
||||
data: {
|
||||
src: finalResult.imageUrl,
|
||||
title: `${titleBase}-AI实景`,
|
||||
prompt: finalResult.prompt || refinePrompt,
|
||||
status: "completed",
|
||||
originalSrc: finalResult.imageUrl,
|
||||
model: finalResult.model,
|
||||
metadata: {
|
||||
aspectRatio,
|
||||
captureMode: "world3d-ai-refine",
|
||||
sourceSnapshotAssetId: uploadResult.data.assetId,
|
||||
sourceSnapshotUrl,
|
||||
sourceWorldNodeId: node.id,
|
||||
sourceWorldPrompt,
|
||||
sourceWorldView: activeView,
|
||||
},
|
||||
},
|
||||
});
|
||||
toast.success("AI 实景优化图片已生成");
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "世界拍照失败";
|
||||
if (resultNodeId) {
|
||||
updateNode(resultNodeId, {
|
||||
data: {
|
||||
src: "",
|
||||
title: `${(data.prompt || "3D世界").trim() || "3D世界"}-AI实景失败`,
|
||||
prompt: message,
|
||||
status: "failed",
|
||||
metadata: {
|
||||
captureMode: "world3d-ai-refine",
|
||||
error: message,
|
||||
sourceWorldNodeId: node.id,
|
||||
sourceWorldPrompt: data.prompt || "",
|
||||
sourceWorldView: activeView,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
toast.error(message);
|
||||
} finally {
|
||||
setIsCapturingSnapshot(false);
|
||||
@ -885,6 +1052,7 @@ export function World3DNode({
|
||||
node.size.height,
|
||||
node.size.width,
|
||||
selectNode,
|
||||
updateNode,
|
||||
viewerCapture,
|
||||
]);
|
||||
|
||||
@ -1062,7 +1230,7 @@ export function World3DNode({
|
||||
disabled={!viewerCapture || isCapturingSnapshot}
|
||||
onClick={() => void handleCaptureSnapshot()}
|
||||
size="sm"
|
||||
title="拍照"
|
||||
title="AI 实景拍照"
|
||||
type="button"
|
||||
variant="ghost"
|
||||
>
|
||||
|
||||
@ -278,12 +278,23 @@ export const imageGenerationService = {
|
||||
negativePrompt?: string;
|
||||
model?: string;
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
imageCount?: number;
|
||||
promptExtend?: boolean;
|
||||
referenceImage?: string;
|
||||
referenceImages?: string[];
|
||||
billingFeature?: 'multi-view' | 'repaint';
|
||||
}): Promise<ApiResponse<{ images: Array<{ thumbnailUrl: string; originalUrl: string }>; model?: string; prompt?: string }>> {
|
||||
async?: boolean;
|
||||
}): Promise<
|
||||
ApiResponse<{
|
||||
images: Array<{ thumbnailUrl: string; originalUrl: string }>;
|
||||
model?: string;
|
||||
prompt?: string;
|
||||
queueStatus?: unknown;
|
||||
status?: string;
|
||||
taskId?: string;
|
||||
}>
|
||||
> {
|
||||
try {
|
||||
const response = await fetch('/api/canvas/generate/image', {
|
||||
method: 'POST',
|
||||
@ -291,6 +302,17 @@ export const imageGenerationService = {
|
||||
body: JSON.stringify(params),
|
||||
});
|
||||
const data = await response.json();
|
||||
if (data?.success && data?.taskId) {
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
images: [],
|
||||
queueStatus: data.queueStatus,
|
||||
status: data.status,
|
||||
taskId: data.taskId,
|
||||
},
|
||||
};
|
||||
}
|
||||
return data;
|
||||
} catch (error) {
|
||||
return { success: false, error: '生成图片失败' };
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user