From f6979f0196db5673d0d949820d5500e746d62929 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 28 Feb 2026 22:33:16 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=8C=E5=96=84=E6=97=A0=E9=99=90?= =?UTF-8?q?=E7=94=BB=E5=B8=833D=E7=94=9F=E6=88=90=E4=B8=8E=E5=AE=9E?= =?UTF-8?q?=E6=97=B6=E7=94=BB=E5=B8=83=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../settings/admin-settings-client.tsx | 174 +- .../admin/realtime/model-capability/route.ts | 41 + app/api/admin/settings/route.ts | 77 +- app/api/canvas/3d/tasks/route.ts | 161 +- app/api/enhance/route.ts | 179 +- app/api/realtime/config/route.ts | 15 + app/api/realtime/generate/route.ts | 738 +++++++- app/api/rmbg/route.ts | 77 +- app/canvas/[id]/model3d/page.tsx | 1570 +++++++++++++++-- app/canvas/[id]/page.tsx | 43 +- app/canvas/[id]/realtime/page.tsx | 82 + components/canvas/RealtimeCanvasDialog.tsx | 394 ++++- components/canvas/model3d/Model3DViewer.tsx | 170 +- lib/realtime/runware-capability.ts | 302 ++++ lib/realtime/runware-ws.ts | 279 +++ lib/tripo/client.ts | 37 +- lib/tripo/types.ts | 16 +- 17 files changed, 3984 insertions(+), 371 deletions(-) create mode 100644 app/api/admin/realtime/model-capability/route.ts create mode 100644 app/canvas/[id]/realtime/page.tsx create mode 100644 lib/realtime/runware-capability.ts create mode 100644 lib/realtime/runware-ws.ts diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index 021182cb2..0cda518f2 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -91,6 +91,8 @@ import { } from "@/lib/payment/dulupay-config"; const fetcher = (url: string) => fetch(url).then((res) => res.json()); +const DEFAULT_RUNWARE_REALTIME_MODEL = "runware:101@1"; +const DEFAULT_RUNWARE_STYLIZE_MODEL = "runware:106@1"; // 类型定义 type User = { @@ -732,6 +734,11 @@ export function AdminSettingsClient() { const [realtimeCanvasConfig, setRealtimeCanvasConfig] = useState({ enabled: true, falApiKey: "", + runwareModel: DEFAULT_RUNWARE_REALTIME_MODEL, + runwareStylizeModel: DEFAULT_RUNWARE_STYLIZE_MODEL, + runwareControlModel: "runware:25@1", + runwareControlPreprocessor: "canny" as "canny" | "lineart" | "depth", + renderMode: "balanced" as "strict-structure" | "balanced" | "stylize", throttleInterval: 100, maxBuffering: 2, maxInputResolution: 1024, @@ -877,7 +884,13 @@ export function AdminSettingsClient() { if (realtimeCanvasConfigData?.setting?.value) { const config = realtimeCanvasConfigData.setting.value as { enabled?: boolean; + runwareApiKey?: string; falApiKey?: string; + runwareModel?: string; + runwareStylizeModel?: string; + runwareControlModel?: string; + runwareControlPreprocessor?: "canny" | "lineart" | "depth"; + renderMode?: "strict-structure" | "balanced" | "stylize"; throttleInterval?: number; maxBuffering?: number; maxInputResolution?: number; @@ -891,7 +904,19 @@ export function AdminSettingsClient() { }; setRealtimeCanvasConfig({ enabled: config.enabled ?? true, - falApiKey: config.falApiKey || "", + falApiKey: config.runwareApiKey || config.falApiKey || "", + runwareModel: config.runwareModel || DEFAULT_RUNWARE_REALTIME_MODEL, + runwareStylizeModel: config.runwareStylizeModel || DEFAULT_RUNWARE_STYLIZE_MODEL, + runwareControlModel: config.runwareControlModel || "runware:25@1", + runwareControlPreprocessor: + config.runwareControlPreprocessor === "lineart" || + config.runwareControlPreprocessor === "depth" + ? config.runwareControlPreprocessor + : "canny", + renderMode: + config.renderMode === "strict-structure" || config.renderMode === "stylize" + ? config.renderMode + : "balanced", throttleInterval: Number(config.throttleInterval || 100), maxBuffering: Number(config.maxBuffering || 2), maxInputResolution: Number(config.maxInputResolution || 1024), @@ -1756,11 +1781,39 @@ export function AdminSettingsClient() { // --- 实时画布API配置逻辑 --- const handleSaveRealtimeCanvasConfig = async () => { try { - if (realtimeCanvasConfig.enabled && !realtimeCanvasConfig.falApiKey.trim()) { - toast.error("启用实时画布功能需要配置 fal.ai API Key"); + const runwareApiKey = realtimeCanvasConfig.falApiKey.trim(); + const runwareModel = + realtimeCanvasConfig.runwareModel.trim() || DEFAULT_RUNWARE_REALTIME_MODEL; + const runwareStylizeModel = + realtimeCanvasConfig.runwareStylizeModel.trim() || DEFAULT_RUNWARE_STYLIZE_MODEL; + const runwareControlModel = + realtimeCanvasConfig.runwareControlModel.trim() || "runware:25@1"; + + if (realtimeCanvasConfig.enabled && !runwareApiKey) { + toast.error("启用实时画布功能需要配置 Runware API Key"); return; } + if (realtimeCanvasConfig.enabled) { + for (const modelId of [runwareModel, runwareStylizeModel].filter(Boolean)) { + const probeResponse = await fetch("/api/admin/realtime/model-capability", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + apiKey: runwareApiKey, + modelId, + }), + }); + const probeData = await probeResponse.json().catch(() => null); + if (!probeResponse.ok || probeData?.supported === false) { + toast.error( + probeData?.message || "模型能力探针失败：当前模型不支持 seedImage，已阻止保存" + ); + return; + } + } + } + const response = await fetch("/api/admin/settings", { method: "POST", headers: { "Content-Type": "application/json" }, @@ -1768,7 +1821,13 @@ export function AdminSettingsClient() { key: "realtime_canvas_config", value: { enabled: realtimeCanvasConfig.enabled, - falApiKey: realtimeCanvasConfig.falApiKey.trim(), + runwareApiKey, + falApiKey: runwareApiKey, + runwareModel, + runwareStylizeModel, + runwareControlModel, + runwareControlPreprocessor: realtimeCanvasConfig.runwareControlPreprocessor, + renderMode: realtimeCanvasConfig.renderMode, throttleInterval: Math.min(260, Math.max(60, Number(realtimeCanvasConfig.throttleInterval) || 100)), maxBuffering: Math.min(6, Math.max(1, Number(realtimeCanvasConfig.maxBuffering) || 2)), maxInputResolution: Math.min( @@ -1786,10 +1845,13 @@ export function AdminSettingsClient() { }), }); - if (!response.ok) throw new Error("保存失败"); + if (!response.ok) { + const errorData = await response.json().catch(() => null); + throw new Error(errorData?.message || "保存失败"); + } toast.success("实时画布配置保存成功"); - } catch (_error) { - toast.error("保存实时画布配置失败"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "保存实时画布配置失败"); } }; @@ -7598,7 +7660,7 @@ export function AdminSettingsClient() { 实时画布配置 - 配置实时画布（左画右预览）功能使用的 fal realtime 参数与 API Key。该 Key 存储在数据库，前端不暴露。 + 配置实时画布功能的 Runware 参数、模型与 API Key。保存时会自动探针校验模型是否支持 seedImage。 @@ -7624,7 +7686,7 @@ export function AdminSettingsClient() {
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, falApiKey: e.target.value }) } - placeholder="fal_xxxxxxxxxxxxx" + placeholder="rw_xxxxxxxxxxxxx" disabled={!realtimeCanvasConfig.enabled} />
+
+ + + setRealtimeCanvasConfig({ ...realtimeCanvasConfig, runwareModel: e.target.value }) + } + placeholder={DEFAULT_RUNWARE_REALTIME_MODEL} + disabled={!realtimeCanvasConfig.enabled} + /> +
+ +
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + runwareStylizeModel: e.target.value, + }) + } + placeholder={DEFAULT_RUNWARE_STYLIZE_MODEL} + disabled={!realtimeCanvasConfig.enabled} + /> +
+
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + runwareControlModel: e.target.value, + }) + } + placeholder="runware:25@1" + disabled={!realtimeCanvasConfig.enabled} + /> +
+
+
+ + +
+
+ + +
+
+

+ 推荐：主模型 {DEFAULT_RUNWARE_REALTIME_MODEL} + ControlNet {`runware:25@1`}。写实模型 {DEFAULT_RUNWARE_STYLIZE_MODEL} 作为回退链路。 +

+
diff --git a/app/api/admin/realtime/model-capability/route.ts b/app/api/admin/realtime/model-capability/route.ts new file mode 100644 index 000000000..cc9a8dcdd --- /dev/null +++ b/app/api/admin/realtime/model-capability/route.ts @@ -0,0 +1,41 @@ +import { z } from "zod"; +import { requireAdmin } from "@/lib/middleware/admin"; +import { ChatSDKError } from "@/lib/errors"; +import { probeRunwareSeedImageSupport } from "@/lib/realtime/runware-capability"; + +const schema = z.object({ + apiKey: z.string().min(1), + modelId: z.string().min(1), +}); + +function mapStatus(code: string) { + if (code === "UPSTREAM_ERROR") return 502; + return 400; +} + +export async function POST(request: Request) { + try { + await requireAdmin(); + + const body = await request.json(); + const data = schema.parse(body); + const result = await probeRunwareSeedImageSupport({ + apiKey: data.apiKey, + modelId: data.modelId, + }); + + if (!result.supported) { + return Response.json(result, { status: mapStatus(result.code) }); + } + + return Response.json(result); + } catch (error) { + if (error instanceof ChatSDKError) { + return error.toResponse(); + } + if (error instanceof z.ZodError) { + return new ChatSDKError("bad_request:api", error.message).toResponse(); + } + return new ChatSDKError("offline:admin").toResponse(); + } +} diff --git a/app/api/admin/settings/route.ts b/app/api/admin/settings/route.ts index 0042d4c2e..032a5d9fd 100644 --- a/app/api/admin/settings/route.ts +++ b/app/api/admin/settings/route.ts @@ -6,12 +6,26 @@ import { upsertSystemSetting, } from "@/lib/db/queries"; import { ChatSDKError } from "@/lib/errors"; +import { probeRunwareSeedImageSupport } from "@/lib/realtime/runware-capability"; const settingSchema = z.object({ key: z.string(), value: z.any(), }); +const DEFAULT_RUNWARE_MODEL = "runware:101@1"; +const DEFAULT_RUNWARE_STYLIZE_MODEL = "runware:106@1"; +const DEFAULT_RUNWARE_CONTROL_MODEL = "runware:25@1"; + +function isObjectLike(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function mapProbeStatus(code: string) { + if (code === "UPSTREAM_ERROR") return 502; + return 400; +} + export async function GET(request: Request) { try { await requireAdmin(); @@ -41,9 +55,69 @@ export async function POST(request: Request) { const body = await request.json(); const { key, value } = settingSchema.parse(body); + let nextValue = value; + + if (key === "realtime_canvas_config") { + if (!isObjectLike(value)) { + return new ChatSDKError("bad_request:api", "realtime_canvas_config value 必须为对象").toResponse(); + } + + const enabled = value.enabled !== false; + const runwareApiKey = String(value.runwareApiKey || value.falApiKey || value.apiKey || "").trim(); + const runwareModel = String(value.runwareModel || DEFAULT_RUNWARE_MODEL).trim() || DEFAULT_RUNWARE_MODEL; + const runwareStylizeModel = + String(value.runwareStylizeModel || DEFAULT_RUNWARE_STYLIZE_MODEL).trim() || + DEFAULT_RUNWARE_STYLIZE_MODEL; + const runwareControlModel = + String(value.runwareControlModel || DEFAULT_RUNWARE_CONTROL_MODEL).trim() || + DEFAULT_RUNWARE_CONTROL_MODEL; + const runwareControlPreprocessor = + value.runwareControlPreprocessor === "lineart" || value.runwareControlPreprocessor === "depth" + ? value.runwareControlPreprocessor + : "canny"; + const renderMode = + value.renderMode === "strict-structure" || + value.renderMode === "stylize" || + value.renderMode === "balanced" + ? value.renderMode + : "balanced"; + + if (enabled) { + if (!runwareApiKey) { + return new ChatSDKError("bad_request:api", "启用实时画布必须配置 Runware API Key").toResponse(); + } + for (const modelId of [runwareModel, runwareStylizeModel].filter(Boolean)) { + const probeResult = await probeRunwareSeedImageSupport({ + apiKey: runwareApiKey, + modelId, + }); + if (!probeResult.supported) { + return Response.json( + { + code: probeResult.code, + message: probeResult.message, + }, + { status: mapProbeStatus(probeResult.code) } + ); + } + } + } + + nextValue = { + ...value, + runwareApiKey, + falApiKey: runwareApiKey, + runwareModel, + runwareStylizeModel, + runwareControlModel, + runwareControlPreprocessor, + renderMode, + }; + } + const setting = await upsertSystemSetting({ key, - value, + value: nextValue, updatedBy: session.user.id, }); @@ -58,4 +132,3 @@ export async function POST(request: Request) { return new ChatSDKError("offline:admin").toResponse(); } } - diff --git a/app/api/canvas/3d/tasks/route.ts b/app/api/canvas/3d/tasks/route.ts index 6a72c99ca..f382fdda6 100644 --- a/app/api/canvas/3d/tasks/route.ts +++ b/app/api/canvas/3d/tasks/route.ts @@ -7,14 +7,44 @@ export const runtime = "nodejs"; export const maxDuration = 60; type CreateTaskRequest = { - taskType: "text_to_model" | "image_to_model" | "convert_model"; + taskType: + | "text_to_model" + | "image_to_model" + | "multiview_to_model" + | "convert_model" + | "refine_model" + | "stylize_model" + | "texture_model" + | "animate_prerigcheck" + | "animate_rig" + | "animate_retarget" + | "mesh_segmentation" + | "mesh_completion" + | "highpoly_to_lowpoly"; prompt?: string; imageToken?: string; imageUrl?: string; imageType?: string; file?: Record; + files?: Array>; + imageTokens?: string[]; + imageUrls?: string[]; originalModelTaskId?: string; + draftModelTaskId?: string; format?: "GLTF" | "USDZ" | "FBX" | "OBJ" | "STL" | "3MF"; + style?: "lego" | "voxel" | "voronoi" | "minecraft"; + blockSize?: number; + outFormat?: "glb" | "fbx"; + rigType?: "biped" | "quadruped" | "hexapod" | "octopod" | "avian" | "serpentine" | "aquatic" | "others"; + spec?: "mixamo" | "tripo"; + animation?: string; + animations?: string[]; + texturePromptText?: string; + texturePromptImageToken?: string; + texturePromptStyleImageToken?: string; + partNames?: string[]; + bakeAnimation?: boolean; + exportWithGeometry?: boolean; modelVersion?: string; negativePrompt?: string; faceLimit?: number; @@ -116,6 +146,48 @@ export async function POST(request: Request) { payload.file = inputFile; } + if (taskType === "multiview_to_model") { + const inputFiles: Array> = []; + + if (Array.isArray(body.files)) { + for (const item of body.files) { + if (item && typeof item === "object") { + inputFiles.push(item); + } + } + } + + if (inputFiles.length === 0 && Array.isArray(body.imageTokens)) { + for (const token of body.imageTokens) { + const normalized = String(token || "").trim(); + if (!normalized) continue; + inputFiles.push({ + type: String(body.imageType || "jpg"), + file_token: normalized, + }); + } + } + + if (inputFiles.length === 0 && Array.isArray(body.imageUrls)) { + for (const imageUrl of body.imageUrls) { + const normalized = String(imageUrl || "").trim(); + if (!normalized) continue; + inputFiles.push({ + type: String(body.imageType || "jpg"), + url: normalized, + }); + } + } + + if (inputFiles.length < 2) { + return NextResponse.json( + { success: false, error: "多视图生成至少需要 2 张参考图" }, + { status: 400 } + ); + } + payload.files = inputFiles; + } + if (taskType === "convert_model") { const originalModelTaskId = String(body.originalModelTaskId || "").trim(); if (!originalModelTaskId) { @@ -131,6 +203,85 @@ export async function POST(request: Request) { payload.format = body.format; } + if ( + taskType === "stylize_model" || + taskType === "texture_model" || + taskType === "animate_prerigcheck" || + taskType === "animate_rig" || + taskType === "animate_retarget" || + taskType === "mesh_segmentation" || + taskType === "mesh_completion" || + taskType === "highpoly_to_lowpoly" + ) { + const originalModelTaskId = String(body.originalModelTaskId || "").trim(); + if (!originalModelTaskId) { + return NextResponse.json( + { success: false, error: "缺少 originalModelTaskId" }, + { status: 400 } + ); + } + payload.original_model_task_id = originalModelTaskId; + } + + if (taskType === "stylize_model") { + if (!body.style) { + return NextResponse.json({ success: false, error: "缺少 style" }, { status: 400 }); + } + payload.style = body.style; + } + + if (taskType === "animate_retarget") { + const singleAnimation = String(body.animation || "").trim(); + const animationList = Array.isArray(body.animations) + ? body.animations.map((item) => String(item || "").trim()).filter(Boolean) + : []; + if (!singleAnimation && animationList.length === 0) { + return NextResponse.json( + { success: false, error: "缺少 animation / animations" }, + { status: 400 } + ); + } + if (singleAnimation) { + payload.animation = singleAnimation; + } else { + payload.animations = animationList; + } + } + + if (taskType === "refine_model") { + const draftModelTaskId = String(body.draftModelTaskId || "").trim(); + if (!draftModelTaskId) { + return NextResponse.json( + { success: false, error: "缺少 draftModelTaskId" }, + { status: 400 } + ); + } + payload.draft_model_task_id = draftModelTaskId; + } + + if (taskType === "texture_model") { + const texturePromptText = String(body.texturePromptText || "").trim(); + const texturePromptImageToken = String(body.texturePromptImageToken || "").trim(); + const texturePromptStyleImageToken = String(body.texturePromptStyleImageToken || "").trim(); + if (texturePromptText || texturePromptImageToken || texturePromptStyleImageToken) { + const texturePrompt: Record = {}; + if (texturePromptText) texturePrompt.text = texturePromptText; + if (texturePromptImageToken) { + texturePrompt.image = { + type: String(body.imageType || "jpg"), + file_token: texturePromptImageToken, + }; + } + if (texturePromptStyleImageToken) { + texturePrompt.style_image = { + type: String(body.imageType || "jpg"), + file_token: texturePromptStyleImageToken, + }; + } + payload.texture_prompt = texturePrompt; + } + } + assignMapped(payload, body as Record, { modelVersion: "model_version", negativePrompt: "negative_prompt", @@ -154,6 +305,13 @@ export async function POST(request: Request) { textureSize: "texture_size", textureFormat: "texture_format", scaleFactor: "scale_factor", + blockSize: "block_size", + outFormat: "out_format", + rigType: "rig_type", + spec: "spec", + partNames: "part_names", + bakeAnimation: "bake_animation", + exportWithGeometry: "export_with_geometry", pivotToCenterBottom: "pivot_to_center_bottom", withAnimation: "with_animation", packUv: "pack_uv", @@ -220,4 +378,3 @@ export async function POST(request: Request) { ); } } - diff --git a/app/api/enhance/route.ts b/app/api/enhance/route.ts index adf912959..800007e45 100644 --- a/app/api/enhance/route.ts +++ b/app/api/enhance/route.ts @@ -19,6 +19,74 @@ type EnhanceApiConfig = { backupModel?: string; }; +const AOS_MAX_RESOLUTION = 4096; +const ENHANCE_CREATE_TIMEOUT_MS = 120_000; +const ENHANCE_CREATE_RETRY_COUNT = 1; + +function getAosFriendlyErrorMessage(detail: string): string | null { + const resolutionMatch = detail.match(/resolution excess,\s*width=(\d+),\s*height=(\d+)/i); + if (resolutionMatch) { + const [, width, height] = resolutionMatch; + return `图片分辨率超限（${width}x${height}），AOS要求宽高都不超过 ${AOS_MAX_RESOLUTION}px。请缩小后重试。`; + } + + if (/invalid file/i.test(detail)) { + return "图片文件无法识别，请尝试转换为 JPG/PNG 后重试。"; + } + + return null; +} + +function isRetryableNetworkError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + if (error.name === "AbortError") return true; + + const message = error.message || ""; + const cause = (error as Error & { cause?: { message?: string; code?: string } }).cause; + const causeMessage = cause?.message || ""; + const causeCode = cause?.code || ""; + const combined = `${message} ${causeMessage} ${causeCode}`; + + return /EHOSTUNREACH|ENOTFOUND|ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|No route to host/i.test(combined); +} + +async function ensureAosResolutionLimit( + buffer: Buffer, + logPrefix: string +): Promise { + try { + const sharp = await import("sharp"); + const image = sharp.default(buffer); + const metadata = await image.metadata(); + const width = metadata.width; + const height = metadata.height; + + if (!width || !height) { + return buffer; + } + + if (width <= AOS_MAX_RESOLUTION && height <= AOS_MAX_RESOLUTION) { + return buffer; + } + + const resizedBuffer = await image + .resize(AOS_MAX_RESOLUTION, AOS_MAX_RESOLUTION, { + fit: "inside", + withoutEnlargement: true, + }) + .toBuffer(); + const resizedMeta = await sharp.default(resizedBuffer).metadata(); + + console.warn( + `[${logPrefix}] 输入图片分辨率超限，已自动缩放: ${width}x${height} -> ${resizedMeta.width}x${resizedMeta.height}` + ); + return resizedBuffer; + } catch (error) { + console.error(`[${logPrefix}] 自动缩放失败，继续使用原图:`, error); + return buffer; + } +} + /** * 压缩图片生成缩略图（使用 sharp） */ @@ -277,7 +345,10 @@ async function pollTaskResult( // state < 0 表示失败 if (state < 0) { - throw result; + const errMessage = String( + result.data?.err_message || result.data?.err_info || result.data?.state_detail || `state=${state}` + ); + throw new Error(`AOS任务失败: ${errMessage}`); } // progress >= 100 表示完成 @@ -294,11 +365,11 @@ async function pollTaskResult( const result = await getTaskResult(); return result; } catch (error) { - if (error && typeof error === 'object' && 'data' in error) { + if (error && typeof error === "object" && "data" in error) { throw new Error(JSON.stringify(error, null, 2)); } if (timeout <= 0) { - throw new Error('轮询超时'); + throw new Error("轮询超时"); } await new Promise((resolve) => { setTimeout(() => { @@ -533,7 +604,8 @@ export async function POST(request: NextRequest) { // 将Base64转换为Buffer console.log("[Enhance API] 转换Base64为Buffer，长度:", imageBase64.length); - const imageBuffer = Buffer.from(imageBase64, "base64"); + const rawImageBuffer = Buffer.from(imageBase64, "base64"); + const imageBuffer = await ensureAosResolutionLimit(rawImageBuffer, "Enhance API"); console.log("[Enhance API] Buffer创建完成，大小:", imageBuffer.length, "bytes"); // 创建FormData（使用form-data包，Node.js环境） @@ -564,10 +636,6 @@ export async function POST(request: NextRequest) { try { console.log("[Enhance API] 开始创建任务..."); - // 添加超时控制（60秒） - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 60000); - // 将FormData转换为Buffer（form-data包是stream） console.log("[Enhance API] 转换FormData为Buffer..."); const formDataBuffer = await new Promise((resolve, reject) => { @@ -603,20 +671,44 @@ export async function POST(request: NextRequest) { }); console.log("[Enhance API] FormData Buffer转换完成，大小:", formDataBuffer.length, "bytes"); - - // 将Buffer转换为Uint8Array用于fetch（更兼容） const uint8Array = new Uint8Array(formDataBuffer); - - console.log("[Enhance API] 发送创建任务请求..."); - const createResponse = await fetch(enhanceUrl, { - method: "POST", - headers, - body: uint8Array, - signal: controller.signal, - }); - - console.log("[Enhance API] 创建任务响应状态:", createResponse.status, createResponse.statusText); - clearTimeout(timeoutId); + + let createResponse: Response | null = null; + let lastFetchError: unknown; + + for (let attempt = 1; attempt <= ENHANCE_CREATE_RETRY_COUNT + 1; attempt++) { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), ENHANCE_CREATE_TIMEOUT_MS); + + try { + console.log(`[Enhance API] 发送创建任务请求... attempt=${attempt}`); + createResponse = await fetch(enhanceUrl, { + method: "POST", + headers, + body: uint8Array, + signal: controller.signal, + }); + console.log("[Enhance API] 创建任务响应状态:", createResponse.status, createResponse.statusText); + clearTimeout(timeoutId); + break; + } catch (fetchError) { + clearTimeout(timeoutId); + lastFetchError = fetchError; + const canRetry = attempt <= ENHANCE_CREATE_RETRY_COUNT && isRetryableNetworkError(fetchError); + if (canRetry) { + console.warn( + `[Enhance API] 创建任务失败，准备重试: attempt=${attempt}, error=${fetchError instanceof Error ? fetchError.message : String(fetchError)}` + ); + await new Promise((resolve) => setTimeout(resolve, 1500)); + continue; + } + throw fetchError; + } + } + + if (!createResponse) { + throw lastFetchError instanceof Error ? lastFetchError : new Error("创建任务请求失败"); + } if (!createResponse.ok) { const errorText = await createResponse.text(); @@ -641,34 +733,34 @@ export async function POST(request: NextRequest) { // 检查是否是网络连接错误 if (fetchError instanceof Error) { - if (fetchError.name === 'AbortError') { + if (fetchError.name === "AbortError") { return NextResponse.json( { error: "变清晰API请求超时", - detail: "创建任务请求超过60秒未响应，请检查API服务器状态或网络连接", + detail: "创建任务请求超过120秒未响应，请检查API服务器状态或网络连接", }, { status: 504 } ); } // 检查错误消息和cause中的网络错误 - const errorMessage = fetchError.message || ''; + const errorMessage = fetchError.message || ""; const errorCause = (fetchError as any).cause; - const causeMessage = errorCause?.message || ''; - const causeCode = errorCause?.code || ''; + const causeMessage = errorCause?.message || ""; + const causeCode = errorCause?.code || ""; // 检查各种网络连接错误 - if (errorMessage.includes('EHOSTUNREACH') || - errorMessage.includes('ENOTFOUND') || - errorMessage.includes('ECONNREFUSED') || - errorMessage.includes('No route to host') || - causeMessage.includes('EHOSTUNREACH') || - causeMessage.includes('ENOTFOUND') || - causeMessage.includes('ECONNREFUSED') || - causeMessage.includes('No route to host') || - causeCode === 'EHOSTUNREACH' || - causeCode === 'ENOTFOUND' || - causeCode === 'ECONNREFUSED') { + if (errorMessage.includes("EHOSTUNREACH") || + errorMessage.includes("ENOTFOUND") || + errorMessage.includes("ECONNREFUSED") || + errorMessage.includes("No route to host") || + causeMessage.includes("EHOSTUNREACH") || + causeMessage.includes("ENOTFOUND") || + causeMessage.includes("ECONNREFUSED") || + causeMessage.includes("No route to host") || + causeCode === "EHOSTUNREACH" || + causeCode === "ENOTFOUND" || + causeCode === "ECONNREFUSED") { return NextResponse.json( { error: "无法连接到变清晰API服务器", @@ -690,7 +782,17 @@ export async function POST(request: NextRequest) { // 4. 轮询获取任务结果（根据官方示例） console.log(`[Enhance] 开始轮询任务结果，task_id: ${taskId}`); - const taskResult = await pollTaskResult(apiUrl, config.apiKey!, taskId); + let taskResult: any; + try { + taskResult = await pollTaskResult(apiUrl, config.apiKey!, taskId); + } catch (pollError) { + const detail = pollError instanceof Error ? pollError.message : "轮询失败"; + const friendlyMessage = getAosFriendlyErrorMessage(detail); + return NextResponse.json( + { error: "变清晰处理失败", detail: friendlyMessage || detail }, + { status: friendlyMessage ? 400 : 500 } + ); + } const finalProcessingTime = (Date.now() - startTime) / 1000; @@ -847,4 +949,3 @@ export async function POST(request: NextRequest) { ); } } - diff --git a/app/api/realtime/config/route.ts b/app/api/realtime/config/route.ts index 150efce1a..c886af85a 100644 --- a/app/api/realtime/config/route.ts +++ b/app/api/realtime/config/route.ts @@ -5,6 +5,9 @@ type RealtimeCanvasConfig = { enabled?: boolean; runwareApiKey?: string; runwareModel?: string; + runwareControlModel?: string; + runwareControlPreprocessor?: "canny" | "lineart" | "depth"; + renderMode?: "strict-structure" | "balanced" | "stylize"; falApiKey?: string; apiKey?: string; strictSketch?: boolean; @@ -163,6 +166,18 @@ export async function GET() { }, behavior: { strictSketch: config.strictSketch !== false, + renderMode: + config.renderMode === "strict-structure" || config.renderMode === "stylize" + ? config.renderMode + : "balanced", + }, + control: { + model: String(config.runwareControlModel || "runware:25@1").trim() || "runware:25@1", + preprocessor: + config.runwareControlPreprocessor === "lineart" || + config.runwareControlPreprocessor === "depth" + ? config.runwareControlPreprocessor + : "canny", }, }); } diff --git a/app/api/realtime/generate/route.ts b/app/api/realtime/generate/route.ts index c899623ae..724a9ab0d 100644 --- a/app/api/realtime/generate/route.ts +++ b/app/api/realtime/generate/route.ts @@ -1,10 +1,18 @@ import { auth } from "@/app/(auth)/auth"; import { getSystemSetting } from "@/lib/db/queries"; +import { getRunwareWebSocketClient } from "@/lib/realtime/runware-ws"; type RealtimeCanvasConfig = { enabled?: boolean; runwareApiKey?: string; runwareModel?: string; + runwareStylizeModel?: string; + runwareControlModel?: string; + runwareControlPreprocessor?: "canny" | "lineart" | "depth"; + renderMode?: "strict-structure" | "balanced" | "stylize"; + runwareUseWebSocket?: boolean; + falRealtimeEnabled?: boolean; + falRealtimeModel?: string; falApiKey?: string; apiKey?: string; negativePrompt?: string; @@ -17,9 +25,23 @@ type RealtimeCanvasConfig = { }; }; +type AngleEditConfig = { + falApiKey?: string; +}; + type GeneratePayload = { requestId?: string; imageUrl?: string; + maskImageUrl?: string; + qualityProfile?: "preview" | "refine"; + roi?: + | { + x?: number; + y?: number; + width?: number; + height?: number; + } + | null; prompt?: string; seed?: number; strictSketch?: boolean; @@ -29,6 +51,12 @@ type GeneratePayload = { strength?: number; }; +const FALLBACK_REALTIME_PROMPT = + "严格按照输入草图生成：保持主体轮廓、位置、比例和数量一致，只做材质与光影细化；禁止改变构图，禁止新增无关背景纹理。"; +const FALLBACK_FAL_REALTIME_MODEL = "fal-ai/flux-2/klein/realtime"; +const FALLBACK_RUNWARE_STYLIZE_MODEL = "runware:106@1"; +const FALLBACK_RUNWARE_CONTROL_MODEL = "runware:25@1"; + function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); } @@ -63,6 +91,11 @@ function parseRunwareError(payload: any) { } if (typeof payload?.error === "string") return payload.error; if (typeof payload?.message === "string") return payload.message; + if (Array.isArray(payload?.errors) && payload.errors.length > 0) { + const firstError = payload.errors[0]; + if (typeof firstError?.message === "string") return firstError.message; + if (typeof firstError?.code === "string") return firstError.code; + } return ""; } @@ -74,6 +107,128 @@ function resolveSeed(seed: unknown) { return normalized; } +function shouldRetryWithoutMask(errorMessage: string) { + const normalized = errorMessage.toLowerCase(); + return ( + normalized.includes("unsupported use of 'maskimage'") || + normalized.includes("unsupported use of \"maskimage\"") || + normalized.includes("unsupported use of maskimage") || + normalized.includes("maskimage is not supported") + ); +} + +function shouldRetryWithoutNegativePrompt(errorMessage: string) { + const normalized = errorMessage.toLowerCase(); + return ( + normalized.includes("unsupportedarchitecturenegativeprompt") || + normalized.includes("negative prompt is not supported") || + normalized.includes("unsupported use of 'negativeprompt'") || + normalized.includes("unsupported use of \"negativeprompt\"") + ); +} + +function shouldRetryWithoutControlNet(errorMessage: string) { + const normalized = errorMessage.toLowerCase(); + return ( + normalized.includes("unsupportedcontrolnetmodel") || + normalized.includes("unsupported use of 'controlnet'") || + normalized.includes("unsupported use of \"controlnet\"") || + normalized.includes("controlnet is not supported") + ); +} + +function resolveRenderModeSettings( + mode: "strict-structure" | "balanced" | "stylize" +) { + if (mode === "strict-structure") { + return { + controlWeight: 1, + seedStrengthMin: 0.14, + seedStrengthMax: 0.58, + }; + } + if (mode === "stylize") { + return { + controlWeight: 0.72, + seedStrengthMin: 0.28, + seedStrengthMax: 0.8, + }; + } + return { + controlWeight: 0.86, + seedStrengthMin: 0.2, + seedStrengthMax: 0.7, + }; +} + +function extractFalImageUrl(payload: any) { + const direct = + payload?.images?.[0]?.url || + payload?.image?.url || + payload?.output?.[0]?.url || + payload?.data?.images?.[0]?.url; + if (typeof direct === "string" && direct.trim()) { + return direct.trim(); + } + + const queue: any[] = [payload]; + const visited = new Set(); + while (queue.length > 0) { + const current = queue.shift(); + if (!current || typeof current !== "object" || visited.has(current)) continue; + visited.add(current); + for (const value of Object.values(current)) { + if (typeof value === "string" && /^https?:\/\//i.test(value)) { + return value; + } + if (value && typeof value === "object") { + queue.push(value); + } + } + } + return ""; +} + +function extractRunwareOutputImage(payload: any, taskUUID: string) { + const resultList = Array.isArray(payload) + ? payload + : Array.isArray(payload?.data) + ? payload.data + : []; + const imageResult = + resultList.find( + (item: any) => + item?.taskUUID === taskUUID && (item?.imageDataURI || item?.imageURL) + ) || + resultList.find((item: any) => item?.imageDataURI || item?.imageURL); + const imageDataURI = String(imageResult?.imageDataURI || "").trim(); + const imageURL = String(imageResult?.imageURL || "").trim(); + const outputImage = imageDataURI || imageURL; + return { + imageResult, + imageDataURI, + imageURL, + outputImage, + }; +} + +function extractControlGuideImage(payload: any, taskUUID: string) { + const resultList = Array.isArray(payload) + ? payload + : Array.isArray(payload?.data) + ? payload.data + : []; + const guideResult = + resultList.find( + (item: any) => + item?.taskUUID === taskUUID && (item?.guideImageURL || item?.guideImageDataURI) + ) || + resultList.find((item: any) => item?.guideImageURL || item?.guideImageDataURI); + const guideImageDataURI = String(guideResult?.guideImageDataURI || "").trim(); + const guideImageURL = String(guideResult?.guideImageURL || "").trim(); + return guideImageDataURI || guideImageURL; +} + export async function POST(request: Request) { const session = await auth(); if (!session?.user?.id) { @@ -91,9 +246,14 @@ export async function POST(request: Request) { if (!imageUrl) { return Response.json({ code: "BAD_REQUEST", message: "缺少 imageUrl" }, { status: 400 }); } + const maskImageUrl = String(body.maskImageUrl || "").trim(); - const setting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null); + const [setting, angleEditSetting] = await Promise.all([ + getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null), + getSystemSetting({ key: "angle_edit_config" }).catch(() => null), + ]); const config = (setting?.value || {}) as RealtimeCanvasConfig; + const angleEditConfig = (angleEditSetting?.value || {}) as AngleEditConfig; if (config.enabled === false) { return Response.json( @@ -110,6 +270,13 @@ export async function POST(request: Request) { process.env.RUNWARE_KEY || "" ).trim(); + const falApiKey = String( + config.falApiKey || + angleEditConfig.falApiKey || + process.env.FAL_KEY || + process.env.FAL_API_KEY || + "" + ).trim(); if (!runwareApiKey) { return Response.json( { @@ -121,11 +288,43 @@ export async function POST(request: Request) { } const fallbackSeedImageModel = "runware:101@1"; - const model = String( + const configuredModel = String( config.runwareModel || process.env.RUNWARE_MODEL || fallbackSeedImageModel ).trim(); + const model = configuredModel || fallbackSeedImageModel; + const configuredStylizeModel = String( + config.runwareStylizeModel || + process.env.RUNWARE_REALTIME_STYLIZE_MODEL || + FALLBACK_RUNWARE_STYLIZE_MODEL + ).trim(); + const stylizeModel = configuredStylizeModel || FALLBACK_RUNWARE_STYLIZE_MODEL; + const configuredControlModel = String( + config.runwareControlModel || + process.env.RUNWARE_REALTIME_CONTROL_MODEL || + FALLBACK_RUNWARE_CONTROL_MODEL + ).trim(); + const controlModel = configuredControlModel || FALLBACK_RUNWARE_CONTROL_MODEL; + const controlPreprocessor = + config.runwareControlPreprocessor === "lineart" || + config.runwareControlPreprocessor === "depth" + ? config.runwareControlPreprocessor + : "canny"; + const renderMode = + config.renderMode === "strict-structure" || + config.renderMode === "stylize" || + config.renderMode === "balanced" + ? config.renderMode + : "balanced"; + const renderModeSettings = resolveRenderModeSettings(renderMode); + const falRealtimeModel = String( + config.falRealtimeModel || process.env.FAL_REALTIME_MODEL || FALLBACK_FAL_REALTIME_MODEL + ).trim(); + const preferWebSocket = + config.runwareUseWebSocket !== false && + process.env.RUNWARE_REALTIME_USE_WS !== "0"; const maxInputResolution = clamp(Number(config.maxInputResolution || 1024), 512, 1536); const size = resolveSquareSize(body.imageSize, maxInputResolution); + const qualityProfile = body.qualityProfile === "preview" ? "preview" : "refine"; const steps = clamp( Number(body.steps || config.defaults?.steps || 8), 1, @@ -144,64 +343,447 @@ export async function POST(request: Request) { const strictSketch = Boolean(body.strictSketch ?? config.strictSketch ?? true); const seed = resolveSeed(body.seed); - const positivePrompt = - String(body.prompt || "").trim() || - "严格按照输入草图生成：保持主体轮廓、位置、比例和数量一致，只做材质与光影细化；禁止改变构图，禁止新增无关背景纹理。"; + const rawPrompt = String(body.prompt || "").trim(); + const hasUserPrompt = + rawPrompt.length > 0 && rawPrompt !== FALLBACK_REALTIME_PROMPT; + const positivePrompt = hasUserPrompt + ? [ + `User prompt: ${rawPrompt}`, + "Keep the object layout and perspective from the sketch.", + "Keep the exact silhouette and part topology from the sketch (same count of legs/supports and same frame structure).", + "The main object category must exactly match the user prompt.", + "Do not swap object class (for example, chair must remain chair, never animal).", + "Do not add bulky cushions, thick armrests, or boxy masses unless they clearly exist in the sketch.", + "Render as a full-color photorealistic product image with realistic material, lighting, and texture.", + "No sketch style, no line art outlines, no monochrome drawing.", + ].join(" ") + : FALLBACK_REALTIME_PROMPT; const negativePrompt = String(config.negativePrompt || "").trim() || - "改变构图, 改变主体位置, 多余人物, 多余肢体, 重复物体, 镜像翻转, 错误透视, 模糊轮廓, 文字水印, 大面积背景噪点"; + "改变构图, 改变主体位置, 多余人物, 多余肢体, 重复物体, 镜像翻转, 错误透视, 模糊轮廓, 文字水印, 大面积背景噪点, 线稿, 线描, 草图, 黑白素描, 轮廓线, line art, sketch, outline drawing, monochrome drawing, wrong object category, unrelated subject, animal when furniture requested, furniture when animal requested"; + let fallbackWarning = ""; - const requestInference = async (modelId: string) => { + const useFalRealtime = + config.falRealtimeEnabled === true || process.env.FAL_REALTIME_ENABLED === "1"; + + if (hasUserPrompt && qualityProfile === "refine" && falApiKey && useFalRealtime) { + try { + const fal = await import("@fal-ai/serverless-client"); + fal.config({ + credentials: falApiKey, + }); + + const falInput: Record = { + prompt: rawPrompt, + negative_prompt: negativePrompt, + image_url: imageUrl, + image_size: body.imageSize === "square_hd" ? "square_hd" : "square", + num_inference_steps: 6, + schedule_mu: 2.3, + output_feedback_strength: 0.95, + sync_mode: true, + }; + + const falResult: any = await fal.subscribe(falRealtimeModel, { + input: falInput, + logs: false, + timeout: 90_000, + }); + + const falImageUrl = extractFalImageUrl(falResult); + if (falImageUrl) { + console.log("[RealtimeFal] frame success", { + userId: session.user.id, + requestId: body.requestId || null, + model: falRealtimeModel, + promptLength: rawPrompt.length, + qualityProfile, + }); + return Response.json({ + ok: true, + provider: "fal", + model: falRealtimeModel, + transport: "fal-http", + imageUrl: falImageUrl, + cost: 0, + }); + } + console.warn("[RealtimeFal] empty output, fallback to Runware", { + userId: session.user.id, + requestId: body.requestId || null, + model: falRealtimeModel, + }); + } catch (error) { + fallbackWarning = + error instanceof Error && /unauthorized|forbidden/i.test(error.message) + ? "FAL 实时模型鉴权失败，当前已回退为 Runware 草图模式；请在系统设置更新有效的 FAL API Key。" + : "FAL 实时模型调用失败，当前已回退为 Runware 草图模式。"; + console.warn("[RealtimeFal] request failed, fallback to Runware", { + userId: session.user.id, + requestId: body.requestId || null, + model: falRealtimeModel, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + type InferenceRequestResult = { + taskUUID: string; + modelId: string; + runtimeStrength: number; + runtimeCfgScale: number; + runtimeSteps: number; + effectiveStrictSketch: boolean; + runtimeSeed?: number; + usedMaskImage: boolean; + controlGuideImage: string; + controlNetModel: string; + path: "seed-image" | "controlnet"; + transport: "ws" | "http"; + upstreamStatus: number; + upstreamOk: boolean; + upstreamJson: any; + upstreamError: string; + wsFallbackError: string; + }; + + type RunwareTaskPayload = { + taskType: string; + taskUUID?: string; + [key: string]: unknown; + }; + + const sendRunwareTask = async (runwareTask: RunwareTaskPayload, timeoutMs = 90_000) => { + let transport: "ws" | "http" = "http"; + let upstreamStatus = 0; + let upstreamOk = false; + let upstreamJson: any = null; + let upstreamError = ""; + let wsFallbackError = ""; + + if (preferWebSocket) { + try { + const wsClient = getRunwareWebSocketClient(runwareApiKey); + upstreamJson = await wsClient.sendTask(runwareTask, timeoutMs); + upstreamError = parseRunwareError(upstreamJson); + upstreamStatus = upstreamError ? 502 : 200; + upstreamOk = !upstreamError; + transport = "ws"; + } catch (error) { + wsFallbackError = error instanceof Error ? error.message : "Runware WebSocket 调用失败"; + } + } + + if (!upstreamJson) { + const upstreamResponse = await fetch("https://api.runware.ai/v1", { + method: "POST", + headers: { + Authorization: `Bearer ${runwareApiKey}`, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify([runwareTask]), + cache: "no-store", + }); + upstreamJson = await upstreamResponse.json().catch(() => null); + upstreamError = parseRunwareError(upstreamJson); + upstreamStatus = upstreamResponse.status; + upstreamOk = upstreamResponse.ok && !upstreamError; + transport = "http"; + } + + return { + transport, + upstreamStatus, + upstreamOk, + upstreamJson, + upstreamError, + wsFallbackError, + }; + }; + + const requestInference = async ( + modelId: string, + useMaskImage: boolean, + sourceImageUrl: string + ): Promise => { const taskUUID = crypto.randomUUID(); - const runtimeStrength = strictSketch ? clamp(strength, 0.12, 0.55) : strength; - const runtimeCfgScale = strictSketch ? clamp(cfgScale, 6, 14) : cfgScale; - const runwareRequest = [ - { - taskType: "imageInference", - taskUUID, - model: modelId, - positivePrompt, - negativePrompt, - seedImage: imageUrl, - width: size, - height: size, - strength: runtimeStrength, - CFGScale: runtimeCfgScale, - steps, - includeCost: true, - numberResults: 1, - outputType: "dataURI", - outputFormat: "JPG", - ...(typeof seed === "number" ? { seed } : {}), - }, - ]; + const usePromptStylize = hasUserPrompt; + const previewMode = qualityProfile === "preview"; + const isStylizePass = + usePromptStylize && + qualityProfile === "refine" && + Boolean(stylizeModel) && + modelId === stylizeModel; + const relaxStructureForPrompt = usePromptStylize && !isStylizePass; + const effectiveStrictSketch = strictSketch && !relaxStructureForPrompt; + const profileStrength = usePromptStylize + ? previewMode + ? clamp(Math.max(strength, 0.58), 0.05, 1) + : isStylizePass + ? clamp(Math.max(strength, 0.72), 0.05, 1) + : clamp(Math.max(strength, 0.9), 0.05, 1) + : previewMode + ? clamp(strength * 0.85, 0.05, 1) + : strength; + const profileCfgScale = usePromptStylize + ? previewMode + ? clamp(Math.max(5.5, Math.min(cfgScale, 8.5)), 1, 14) + : isStylizePass + ? clamp(Math.max(6.5, Math.min(cfgScale, 8.5)), 1, 14) + : clamp(Math.max(6, Math.min(cfgScale, 8.5)), 1, 14) + : previewMode + ? clamp(cfgScale * 0.9, 1, 14) + : cfgScale; + const profileSteps = usePromptStylize + ? previewMode + ? clamp(Math.max(steps, 6), 1, 12) + : isStylizePass + ? clamp(Math.max(steps, 8), 1, 10) + : clamp(Math.max(steps, 12), 1, 12) + : previewMode + ? clamp(Math.min(steps, 6), 1, 12) + : clamp(Math.max(steps, 6), 1, 12); + const runtimeStrength = effectiveStrictSketch + ? (() => { + const minStrength = usePromptStylize && !previewMode ? 0.2 : 0.12; + const maxStrength = + isStylizePass ? 0.62 : usePromptStylize && !previewMode ? 0.72 : 0.55; + return clamp(profileStrength, minStrength, maxStrength); + })() + : profileStrength; + const runtimeCfgScale = effectiveStrictSketch + ? clamp( + profileCfgScale, + isStylizePass ? 6.5 : usePromptStylize && !previewMode ? 5 : 6, + 14 + ) + : profileCfgScale; + const runtimeSeed = relaxStructureForPrompt ? undefined : seed; + const isRunware106 = modelId.toLowerCase().startsWith("runware:106@"); + const inferenceSize = isRunware106 ? 1024 : size; + const runtimeStrengthLimited = clamp( + runtimeStrength, + renderModeSettings.seedStrengthMin, + renderModeSettings.seedStrengthMax + ); + const runwareTask = { + taskType: "imageInference", + taskUUID, + model: modelId, + positivePrompt, + negativePrompt, + seedImage: sourceImageUrl, + width: inferenceSize, + height: inferenceSize, + strength: runtimeStrengthLimited, + CFGScale: runtimeCfgScale, + steps: profileSteps, + includeCost: true, + numberResults: 1, + outputType: "dataURI", + outputFormat: "JPG", + ...(useMaskImage && maskImageUrl ? { maskImage: maskImageUrl } : {}), + ...(typeof runtimeSeed === "number" ? { seed: runtimeSeed } : {}), + }; - const upstreamResponse = await fetch("https://api.runware.ai/v1", { - method: "POST", - headers: { - Authorization: `Bearer ${runwareApiKey}`, - "Content-Type": "application/json", - Accept: "application/json", - }, - body: JSON.stringify(runwareRequest), - cache: "no-store", - }); - - const upstreamJson = await upstreamResponse.json().catch(() => null); - const upstreamError = parseRunwareError(upstreamJson); + const upstream = await sendRunwareTask(runwareTask); return { taskUUID, modelId, - runtimeStrength, + runtimeStrength: runtimeStrengthLimited, runtimeCfgScale, - upstreamResponse, - upstreamJson, - upstreamError, + runtimeSteps: profileSteps, + effectiveStrictSketch, + runtimeSeed, + usedMaskImage: useMaskImage && Boolean(maskImageUrl), + controlGuideImage: "", + controlNetModel: "", + path: "seed-image" as const, + ...upstream, }; }; - let requestResult = await requestInference(model); + const requestWithMaskFallback = async ( + modelId: string, + useMaskImage: boolean, + sourceImageUrl: string + ) => { + let requestResult = await requestInference(modelId, useMaskImage, sourceImageUrl); + let maskEnabled = useMaskImage; + + if ( + maskEnabled && + shouldRetryWithoutMask(requestResult.upstreamError || "") + ) { + requestResult = await requestInference(modelId, false, sourceImageUrl); + maskEnabled = false; + } + + return { requestResult, maskEnabled }; + }; + + const requestControlNetGuide = async (sourceImageUrl: string) => { + const taskUUID = crypto.randomUUID(); + const controlTask = { + taskType: "imageControlNetPreProcess", + taskUUID, + inputImage: sourceImageUrl, + preProcessorType: controlPreprocessor, + }; + const upstream = await sendRunwareTask(controlTask, 45_000); + return { + taskUUID, + guideImage: extractControlGuideImage(upstream.upstreamJson, taskUUID), + ...upstream, + }; + }; + + const requestControlNetInference = async ( + modelId: string, + guideImage: string, + includeNegativePrompt: boolean + ): Promise => { + const taskUUID = crypto.randomUUID(); + const previewMode = qualityProfile === "preview"; + const profileSteps = previewMode + ? clamp(Math.max(steps, 6), 1, 12) + : clamp(Math.max(steps, 10), 1, 12); + const profileCfgScale = previewMode + ? clamp(Math.max(5, Math.min(cfgScale, 8)), 1, 14) + : clamp(Math.max(6, Math.min(cfgScale, 9)), 1, 14); + const controlWeight = previewMode + ? clamp(renderModeSettings.controlWeight - 0.06, 0.55, 1) + : renderModeSettings.controlWeight; + + const runwareTask: RunwareTaskPayload = { + taskType: "imageInference", + taskUUID, + model: modelId, + positivePrompt, + width: size, + height: size, + CFGScale: profileCfgScale, + steps: profileSteps, + includeCost: true, + numberResults: 1, + outputType: "dataURI", + outputFormat: "JPG", + ...(typeof seed === "number" ? { seed } : {}), + ...(includeNegativePrompt ? { negativePrompt } : {}), + controlNet: [ + { + model: controlModel, + guideImage, + weight: controlWeight, + startStep: 0, + endStep: profileSteps, + controlMode: "balanced", + }, + ], + }; + + const upstream = await sendRunwareTask(runwareTask, 120_000); + return { + taskUUID, + modelId, + runtimeStrength: 0, + runtimeCfgScale: profileCfgScale, + runtimeSteps: profileSteps, + effectiveStrictSketch: strictSketch, + runtimeSeed: typeof seed === "number" ? seed : undefined, + usedMaskImage: false, + controlGuideImage: guideImage, + controlNetModel: controlModel, + path: "controlnet" as const, + ...upstream, + }; + }; + + const runTwoPassStylize = false; + let maskEnabled = Boolean(maskImageUrl); + let requestResult: InferenceRequestResult; + let pathLabel: "seed-image" | "controlnet" = "seed-image"; + let controlGuideUsed = ""; + let controlPreprocessUsed = false; + let controlGuideFallback = false; + + const shouldUseControlNetPath = + hasUserPrompt && qualityProfile === "refine" && strictSketch; + + if (shouldUseControlNetPath) { + const guideResult = await requestControlNetGuide(imageUrl); + const preprocessOk = guideResult.upstreamOk && !guideResult.upstreamError; + const guideImage = preprocessOk && guideResult.guideImage ? guideResult.guideImage : imageUrl; + controlGuideUsed = guideImage; + controlPreprocessUsed = preprocessOk; + controlGuideFallback = !preprocessOk || !guideResult.guideImage; + + if (!preprocessOk && (guideResult.upstreamError || "").trim()) { + fallbackWarning = "ControlNet 预处理失败，已回退使用原始线稿引导。"; + } + + let controlResult = await requestControlNetInference(model, guideImage, true); + if (shouldRetryWithoutNegativePrompt(controlResult.upstreamError || "")) { + controlResult = await requestControlNetInference(model, guideImage, false); + } + + if (controlResult.upstreamOk && !controlResult.upstreamError) { + requestResult = controlResult; + maskEnabled = false; + pathLabel = "controlnet"; + } else if (shouldRetryWithoutControlNet(controlResult.upstreamError || "")) { + fallbackWarning = "当前模型不支持 ControlNet，已回退到 seedImage 模式。"; + const fallbackPass = await requestWithMaskFallback(stylizeModel || model, maskEnabled, imageUrl); + requestResult = fallbackPass.requestResult; + maskEnabled = fallbackPass.maskEnabled; + pathLabel = "seed-image"; + } else { + requestResult = controlResult; + maskEnabled = false; + pathLabel = "controlnet"; + } + } else if (runTwoPassStylize) { + const basePass = await requestWithMaskFallback(model, maskEnabled, imageUrl); + requestResult = basePass.requestResult; + maskEnabled = basePass.maskEnabled; + + if (requestResult.upstreamOk && !requestResult.upstreamError) { + const baseImage = extractRunwareOutputImage( + requestResult.upstreamJson, + requestResult.taskUUID + ).outputImage; + if (baseImage) { + console.log("[RealtimeRunware] two-pass base success", { + userId: session.user.id, + requestId: body.requestId || null, + baseModel: model, + stylizeModel, + baseImageType: baseImage.startsWith("data:") ? "dataURI" : "url", + }); + const stylizePass = await requestWithMaskFallback(stylizeModel, false, baseImage); + requestResult = stylizePass.requestResult; + maskEnabled = stylizePass.maskEnabled; + } + } + } else { + const initialModel = + hasUserPrompt && qualityProfile === "refine" ? stylizeModel || model : model; + const initialPass = await requestWithMaskFallback(initialModel, maskEnabled, imageUrl); + requestResult = initialPass.requestResult; + maskEnabled = initialPass.maskEnabled; + pathLabel = "seed-image"; + } + + if (pathLabel === "controlnet" && (!requestResult.upstreamOk || requestResult.upstreamError)) { + fallbackWarning = "ControlNet 推理失败，已自动回退为 seedImage 模式。"; + const fallbackPass = await requestWithMaskFallback(stylizeModel || model, maskEnabled, imageUrl); + requestResult = fallbackPass.requestResult; + maskEnabled = fallbackPass.maskEnabled; + pathLabel = "seed-image"; + } + const shouldRetryWithSeedImageModel = (requestResult.upstreamError || "") .toLowerCase() @@ -209,36 +791,34 @@ export async function POST(request: Request) { model !== fallbackSeedImageModel; if (shouldRetryWithSeedImageModel) { - requestResult = await requestInference(fallbackSeedImageModel); + fallbackWarning = + "当前模型不支持草图引导，已自动切换到兼容的实时草图模型。"; + const fallbackResult = await requestWithMaskFallback( + fallbackSeedImageModel, + maskEnabled, + imageUrl + ); + requestResult = fallbackResult.requestResult; + maskEnabled = fallbackResult.maskEnabled; } - if (!requestResult.upstreamResponse.ok || requestResult.upstreamError) { + if (!requestResult.upstreamOk || requestResult.upstreamError) { return Response.json( { code: "RUNWARE_REQUEST_FAILED", message: requestResult.upstreamError || - `Runware 请求失败（${requestResult.upstreamResponse.status}）`, + requestResult.wsFallbackError || + `Runware 请求失败（${requestResult.upstreamStatus}）`, }, { status: 502 } ); } - const resultList = Array.isArray(requestResult.upstreamJson) - ? requestResult.upstreamJson - : Array.isArray(requestResult.upstreamJson?.data) - ? requestResult.upstreamJson.data - : []; - const imageResult = - resultList.find( - (item: any) => - item?.taskUUID === requestResult.taskUUID && (item?.imageDataURI || item?.imageURL) - ) || - resultList.find((item: any) => item?.imageDataURI || item?.imageURL); - - const imageDataURI = String(imageResult?.imageDataURI || "").trim(); - const imageURL = String(imageResult?.imageURL || "").trim(); - const outputImage = imageDataURI || imageURL; + const { imageResult, imageDataURI, imageURL, outputImage } = extractRunwareOutputImage( + requestResult.upstreamJson, + requestResult.taskUUID + ); if (!outputImage) { return Response.json( @@ -253,20 +833,44 @@ export async function POST(request: Request) { requestId: body.requestId || null, taskUUID: requestResult.taskUUID, model: requestResult.modelId, - seed: typeof seed === "number" ? seed : null, - steps, + transport: requestResult.transport, + seed: typeof requestResult.runtimeSeed === "number" ? requestResult.runtimeSeed : null, + hasUserPrompt, + promptLength: rawPrompt.length, + promptPreview: hasUserPrompt ? rawPrompt.slice(0, 32) : null, + qualityProfile, + steps: requestResult.runtimeSteps, cfgScale: requestResult.runtimeCfgScale, strength: requestResult.runtimeStrength, + strictSketchApplied: requestResult.effectiveStrictSketch, + maskEnabled, + roi: body.roi || null, cost, + renderMode, + path: pathLabel, + controlNetModel: pathLabel === "controlnet" ? requestResult.controlNetModel : null, + controlGuideType: + pathLabel === "controlnet" && controlGuideUsed + ? controlGuideUsed.startsWith("data:") + ? "dataURI" + : "url" + : null, + controlGuideFallback, + controlPreprocessUsed, + twoPassStylize: runTwoPassStylize, + fallbackWarning: fallbackWarning || null, }); return Response.json({ ok: true, provider: "runware", model: requestResult.modelId, + transport: requestResult.transport, taskUUID: requestResult.taskUUID, + path: pathLabel, imageDataURI: imageDataURI || undefined, imageUrl: imageURL || undefined, cost, + warning: fallbackWarning || undefined, }); } diff --git a/app/api/rmbg/route.ts b/app/api/rmbg/route.ts index 918a8dc49..c0312251c 100644 --- a/app/api/rmbg/route.ts +++ b/app/api/rmbg/route.ts @@ -89,6 +89,7 @@ async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/pn } const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const AOS_MAX_RESOLUTION = 4096; const AOS_CREATE_TIMEOUT_MS = 120_000; const AOS_POLL_TIMEOUT_MS = 60_000; const AOS_DOWNLOAD_TIMEOUT_MS = 60_000; @@ -106,6 +107,71 @@ const withTimeout = async ( } }; +function getAosFriendlyErrorMessage(detail: string): string | null { + const resolutionMatch = detail.match(/resolution excess,\s*width=(\d+),\s*height=(\d+)/i); + if (resolutionMatch) { + const [, width, height] = resolutionMatch; + return `图片分辨率超限（${width}x${height}），AOS要求宽高都不超过 ${AOS_MAX_RESOLUTION}px。请缩小后重试。`; + } + + if (/invalid file/i.test(detail)) { + return "图片文件无法识别，请尝试转换为 JPG/PNG 后重试。"; + } + + return null; +} + +async function prepareImageForAos( + imageBuffer: Buffer +): Promise<{ buffer: Buffer; mimeType: string; filename: string }> { + try { + const sharp = await import("sharp"); + const image = sharp.default(imageBuffer); + const metadata = await image.metadata(); + const width = metadata.width; + const height = metadata.height; + const format = (metadata.format || "png").toLowerCase(); + + const ext = format === "jpeg" ? "jpg" : format; + const filenameExt = + ext === "jpg" || ext === "png" || ext === "webp" || ext === "tiff" ? ext : "png"; + const mimeType = + ext === "jpg" ? "image/jpeg" : + ext === "webp" ? "image/webp" : + ext === "tiff" ? "image/tiff" : + "image/png"; + + if (!width || !height || (width <= AOS_MAX_RESOLUTION && height <= AOS_MAX_RESOLUTION)) { + return { + buffer: imageBuffer, + mimeType, + filename: `image.${filenameExt}`, + }; + } + + const resizedBuffer = await image + .resize(AOS_MAX_RESOLUTION, AOS_MAX_RESOLUTION, { + fit: "inside", + withoutEnlargement: true, + }) + .toBuffer(); + const resizedMeta = await sharp.default(resizedBuffer).metadata(); + + console.warn( + `[AOS RMBG] 输入图片分辨率超限，已自动缩放: ${width}x${height} -> ${resizedMeta.width}x${resizedMeta.height}` + ); + + return { + buffer: resizedBuffer, + mimeType, + filename: `image.${filenameExt}`, + }; + } catch (error) { + console.error("[AOS RMBG] 图片预处理失败，继续使用原图:", error); + return { buffer: imageBuffer, mimeType: "image/png", filename: "image.png" }; + } +} + /** * 使用 RMBG-2.0 API 进行抠图 */ @@ -254,9 +320,10 @@ async function processWithAosSegmentation( const headers = { "X-API-KEY": config.apiKey }; try { + const normalizedImage = await prepareImageForAos(imageBuffer); const createForm = new FormData(); - const imageBlob = new Blob([new Uint8Array(imageBuffer)], { type: "image/png" }); - createForm.append("image_file", imageBlob, "image.png"); + const imageBlob = new Blob([new Uint8Array(normalizedImage.buffer)], { type: normalizedImage.mimeType }); + createForm.append("image_file", imageBlob, normalizedImage.filename); createForm.append("sync", "0"); createForm.append("return_type", "1"); createForm.append("output_type", "2"); @@ -527,9 +594,11 @@ export async function POST(request: NextRequest) { }); if (!result.success || !result.imageBuffer) { + const detail = result.error || "AOS处理失败"; + const friendlyMessage = getAosFriendlyErrorMessage(detail); return NextResponse.json( - { error: "抠图处理失败", detail: result.error }, - { status: 500 } + { error: "抠图处理失败", detail: friendlyMessage || detail }, + { status: friendlyMessage ? 400 : 500 } ); } diff --git a/app/canvas/[id]/model3d/page.tsx b/app/canvas/[id]/model3d/page.tsx index 1dc79ae5e..fc5cfbc74 100644 --- a/app/canvas/[id]/model3d/page.tsx +++ b/app/canvas/[id]/model3d/page.tsx @@ -1,44 +1,79 @@ "use client"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useParams, useRouter } from "next/navigation"; import { ArrowLeft, Box, + ChevronDown, + ChevronUp, Download, + Eye, + EyeOff, + Layers3, Loader2, PlusCircle, RefreshCw, Sparkles, - Upload, + WandSparkles, + X, } from "lucide-react"; import { toast } from "sonner"; +import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { ScrollArea } from "@/components/ui/scroll-area"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Slider } from "@/components/ui/slider"; import { Switch } from "@/components/ui/switch"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Textarea } from "@/components/ui/textarea"; -import { Model3DViewer } from "@/components/canvas/model3d/Model3DViewer"; +import { + Model3DViewer, + type Model3DLayer, + type Model3DStats, + type Model3DViewerSettings, +} from "@/components/canvas/model3d/Model3DViewer"; import { canvasSelectors, useCanvasStore } from "@/store/canvas"; import { canvasService, model3DGenerationService, type Model3DTaskStatus } from "@/services/canvas"; import { cn } from "@/lib/utils"; -type GenerationMode = "text" | "image"; +type GenerationMode = "text" | "image" | "multiview"; +type TaskMode = + | GenerationMode + | "convert" + | "refine" + | "texture" + | "stylize" + | "rigcheck" + | "rig" + | "retarget" + | "segment" + | "complete" + | "lowpoly"; type TaskRecord = { taskId: string; status: Model3DTaskStatus; + mode: TaskMode; prompt: string; - mode: GenerationMode; modelVersion: string; createdAt: number; progress?: number | null; modelUrl?: string | null; previewImageUrl?: string | null; error?: string | null; + sourceTaskId?: string | null; + exportFormat?: string | null; +}; + +type MultiViewSlot = { + id: string; + file: File | null; + previewUrl: string | null; }; const MODEL_VERSION_OPTIONS = [ @@ -50,6 +85,44 @@ const MODEL_VERSION_OPTIONS = [ "v1.4-20240625", ] as const; +const ENV_PRESET_OPTIONS: Array = [ + "city", + "sunset", + "studio", + "dawn", + "night", + "warehouse", + "forest", + "apartment", + "park", + "lobby", +]; + +const EXPORT_FORMAT_OPTIONS = ["GLTF", "USDZ", "FBX", "OBJ", "STL", "3MF"] as const; +const STYLE_OPTIONS = ["lego", "voxel", "voronoi", "minecraft"] as const; +const RIG_TYPE_OPTIONS = [ + "biped", + "quadruped", + "hexapod", + "octopod", + "avian", + "serpentine", + "aquatic", + "others", +] as const; +const RIG_SPEC_OPTIONS = ["tripo", "mixamo"] as const; +const RIG_FORMAT_OPTIONS = ["glb", "fbx"] as const; +const ANIMATION_OPTIONS = [ + "preset:idle", + "preset:walk", + "preset:run", + "preset:jump", + "preset:turn", + "preset:slash", + "preset:shoot", + "preset:hurt", + "preset:fall", +] as const; const FINAL_TASK_STATUS = new Set([ "success", "failed", @@ -58,6 +131,8 @@ const FINAL_TASK_STATUS = new Set([ "cancelled", "unknown", ]); +const TASK_HISTORY_LIMIT = 40; +const TASK_HISTORY_STORAGE_PREFIX = "canvas-model3d-task-history-v2"; function normalizeTaskStatus(status: string | undefined): Model3DTaskStatus { switch (status) { @@ -87,7 +162,7 @@ function formatBalance(data: Record | undefined): string | null { function extractModelFormat(modelUrl?: string | null): string { const value = String(modelUrl || "").toLowerCase(); - if (!value) return "3D"; + if (!value) return "3d"; if (value.includes(".gltf")) return "gltf"; if (value.includes(".glb")) return "glb"; if (value.includes(".usdz")) return "usdz"; @@ -98,10 +173,77 @@ function extractModelFormat(modelUrl?: string | null): string { return "3d"; } +function createMultiViewSlots(count = 4): MultiViewSlot[] { + return Array.from({ length: count }, (_, index) => ({ + id: `slot_${index + 1}`, + file: null, + previewUrl: null, + })); +} + +function modeLabel(mode: TaskMode) { + switch (mode) { + case "text": + return "文生3D"; + case "image": + return "图生3D"; + case "multiview": + return "多视图3D"; + case "convert": + return "格式转换"; + case "refine": + return "模型精修"; + case "texture": + return "纹理重绘"; + case "stylize": + return "风格化"; + case "rigcheck": + return "可绑定检测"; + case "rig": + return "自动绑定"; + case "retarget": + return "动作重定向"; + case "segment": + return "网格分割"; + case "complete": + return "网格补全"; + case "lowpoly": + return "低模优化"; + default: + return mode; + } +} + +function statusLabel(status: Model3DTaskStatus) { + switch (status) { + case "queued": + return "排队中"; + case "running": + return "生成中"; + case "success": + return "成功"; + case "failed": + return "失败"; + case "banned": + return "审核拦截"; + case "expired": + return "已过期"; + case "cancelled": + return "已取消"; + default: + return "未知"; + } +} + +function makeStorageKey(projectId: string) { + return `${TASK_HISTORY_STORAGE_PREFIX}:${projectId}`; +} + export default function CanvasModel3DPage() { const router = useRouter(); const routeParams = useParams(); const id = String(routeParams?.id || ""); + const historyStorageKey = useMemo(() => makeStorageKey(id), [id]); const viewport = useCanvasStore(canvasSelectors.viewport); const addNode = useCanvasStore((state) => state.addNode); @@ -109,6 +251,7 @@ export default function CanvasModel3DPage() { const [mode, setMode] = useState("text"); const [prompt, setPrompt] = useState(""); + const [negativePrompt, setNegativePrompt] = useState(""); const [modelVersion, setModelVersion] = useState<(typeof MODEL_VERSION_OPTIONS)[number]>( "Turbo-v1.0-20250506" ); @@ -116,13 +259,67 @@ export default function CanvasModel3DPage() { const [texture, setTexture] = useState(true); const [pbr, setPbr] = useState(true); const [autoSize, setAutoSize] = useState(false); + const [quad, setQuad] = useState(false); + const [generateParts, setGenerateParts] = useState(false); + const [smartLowPoly, setSmartLowPoly] = useState(false); + const [geometryQuality, setGeometryQuality] = useState<"standard" | "detailed">("standard"); + const [textureQuality, setTextureQuality] = useState<"standard" | "detailed">("standard"); + const [textureAlignment, setTextureAlignment] = useState<"original_image" | "geometry">( + "original_image" + ); + const [orientation, setOrientation] = useState<"default" | "align_image">("default"); + const [modelSeed, setModelSeed] = useState(""); + const [textureSeed, setTextureSeed] = useState(""); + const [isSubmitting, setIsSubmitting] = useState(false); const [uploadingImage, setUploadingImage] = useState(false); const [referenceImage, setReferenceImage] = useState(null); const [referencePreviewUrl, setReferencePreviewUrl] = useState(null); - const [currentTask, setCurrentTask] = useState(null); + const [multiViewSlots, setMultiViewSlots] = useState(() => createMultiViewSlots()); + const [taskHistory, setTaskHistory] = useState([]); + const [currentTask, setCurrentTask] = useState(null); const [balanceLabel, setBalanceLabel] = useState(null); + const [autoDownloadTaskId, setAutoDownloadTaskId] = useState(null); + + const [advancedOpen, setAdvancedOpen] = useState(false); + const [viewerOpen, setViewerOpen] = useState(true); + const [postOpen, setPostOpen] = useState(true); + + const [exportFormat, setExportFormat] = useState<(typeof EXPORT_FORMAT_OPTIONS)[number]>("GLTF"); + const [stylizeStyle, setStylizeStyle] = useState<(typeof STYLE_OPTIONS)[number]>("lego"); + const [stylizeBlockSize, setStylizeBlockSize] = useState(80); + const [texturePromptText, setTexturePromptText] = useState(""); + const [rigType, setRigType] = useState<(typeof RIG_TYPE_OPTIONS)[number]>("biped"); + const [rigSpec, setRigSpec] = useState<(typeof RIG_SPEC_OPTIONS)[number]>("tripo"); + const [rigFormat, setRigFormat] = useState<(typeof RIG_FORMAT_OPTIONS)[number]>("glb"); + const [retargetAnimation, setRetargetAnimation] = useState<(typeof ANIMATION_OPTIONS)[number]>( + "preset:walk" + ); + const [retargetBake, setRetargetBake] = useState(true); + const [retargetExportWithGeometry, setRetargetExportWithGeometry] = useState(false); + const [retargetInPlace, setRetargetInPlace] = useState(false); + const [completionPartNames, setCompletionPartNames] = useState(""); + const [lowpolyFaceLimit, setLowpolyFaceLimit] = useState(30000); + const [lowpolyQuad, setLowpolyQuad] = useState(false); + const [lowpolyBake, setLowpolyBake] = useState(true); + const [viewerSettings, setViewerSettings] = useState({ + envPreset: "city", + lightIntensity: 1, + autoRotate: false, + showGrid: true, + wireframe: false, + }); + const [viewerStats, setViewerStats] = useState({ + triangles: 0, + vertices: 0, + meshCount: 0, + }); + const [viewerLayers, setViewerLayers] = useState([]); + const [layerVisibility, setLayerVisibility] = useState>({}); + const [isolatedLayerId, setIsolatedLayerId] = useState(null); + const pollAttemptsRef = useRef(0); + const multiViewSlotsRef = useRef([]); useEffect(() => { const loadBalance = async () => { @@ -134,41 +331,101 @@ export default function CanvasModel3DPage() { void loadBalance(); }, []); - const upsertTaskHistory = useCallback((next: TaskRecord) => { - setTaskHistory((prev) => { - const index = prev.findIndex((item) => item.taskId === next.taskId); - if (index >= 0) { - const updated = [...prev]; - updated[index] = { ...updated[index], ...next }; - return updated; - } - return [next, ...prev].slice(0, 20); - }); - }, []); + useEffect(() => { + if (typeof window === "undefined") return; + const raw = window.localStorage.getItem(historyStorageKey); + if (!raw) return; + try { + const parsed = JSON.parse(raw) as TaskRecord[]; + if (!Array.isArray(parsed)) return; + const normalized = parsed + .filter((item) => item && typeof item.taskId === "string") + .slice(0, TASK_HISTORY_LIMIT); + setTaskHistory(normalized); + const pending = normalized.find((item) => !FINAL_TASK_STATUS.has(item.status)); + setCurrentTask(pending || normalized[0] || null); + } catch { + window.localStorage.removeItem(historyStorageKey); + } + }, [historyStorageKey]); + + useEffect(() => { + if (typeof window === "undefined") return; + window.localStorage.setItem(historyStorageKey, JSON.stringify(taskHistory.slice(0, TASK_HISTORY_LIMIT))); + }, [historyStorageKey, taskHistory]); useEffect(() => { if (!referenceImage) { + if (referencePreviewUrl) { + URL.revokeObjectURL(referencePreviewUrl); + } setReferencePreviewUrl(null); return; } const objectUrl = URL.createObjectURL(referenceImage); - setReferencePreviewUrl(objectUrl); + setReferencePreviewUrl((prev) => { + if (prev) URL.revokeObjectURL(prev); + return objectUrl; + }); return () => { URL.revokeObjectURL(objectUrl); }; }, [referenceImage]); + useEffect(() => { + multiViewSlotsRef.current = multiViewSlots; + }, [multiViewSlots]); + + useEffect(() => { + return () => { + for (const slot of multiViewSlotsRef.current) { + if (slot.previewUrl) { + URL.revokeObjectURL(slot.previewUrl); + } + } + }; + }, []); + + const upsertTaskHistory = useCallback((next: TaskRecord) => { + setTaskHistory((prev) => { + const index = prev.findIndex((item) => item.taskId === next.taskId); + if (index >= 0) { + const updated = [...prev]; + updated[index] = { ...updated[index], ...next }; + return updated.slice(0, TASK_HISTORY_LIMIT); + } + return [next, ...prev].slice(0, TASK_HISTORY_LIMIT); + }); + }, []); + + const effectiveLayerVisibility = useMemo(() => { + if (!isolatedLayerId) return layerVisibility; + const next: Record = {}; + for (const layer of viewerLayers) { + next[layer.id] = layer.id === isolatedLayerId && layerVisibility[layer.id] !== false; + } + return next; + }, [isolatedLayerId, layerVisibility, viewerLayers]); + + const currentModelUrl = currentTask?.modelUrl || null; + const currentPreviewImage = + currentTask?.previewImageUrl || + (mode === "image" ? referencePreviewUrl : null) || + multiViewSlots.find((slot) => slot.previewUrl)?.previewUrl || + null; + useEffect(() => { if (!currentTask || FINAL_TASK_STATUS.has(currentTask.status)) { return; } + pollAttemptsRef.current = 0; let cancelled = false; - let attempts = 0; const runPoll = async () => { if (cancelled) return; - attempts += 1; + pollAttemptsRef.current += 1; + const result = await model3DGenerationService.getTask(currentTask.taskId); if (!result.success || !result.data) { const failMessage = result.error || "任务查询失败"; @@ -196,10 +453,17 @@ export default function CanvasModel3DPage() { upsertTaskHistory(nextRecord); if (status === "success") { - toast.success("3D 模型生成完成"); + toast.success(`${modeLabel(nextRecord.mode)}任务已完成`); + if (autoDownloadTaskId && autoDownloadTaskId === nextRecord.taskId && nextRecord.modelUrl) { + const link = document.createElement("a"); + link.href = nextRecord.modelUrl; + link.download = `tripo-model-${nextRecord.taskId}.${extractModelFormat(nextRecord.modelUrl)}`; + link.click(); + setAutoDownloadTaskId(null); + } } - if (attempts >= 300 && !FINAL_TASK_STATUS.has(status)) { + if (pollAttemptsRef.current >= 360 && !FINAL_TASK_STATUS.has(status)) { const timeoutRecord: TaskRecord = { ...nextRecord, status: "failed", @@ -220,7 +484,112 @@ export default function CanvasModel3DPage() { cancelled = true; window.clearInterval(timer); }; - }, [currentTask, upsertTaskHistory]); + }, [autoDownloadTaskId, currentTask, upsertTaskHistory]); + + const handleMultiViewFileChange = (index: number, file: File | null) => { + setMultiViewSlots((prev) => { + const next = [...prev]; + const target = next[index]; + if (!target) return prev; + + if (target.previewUrl) URL.revokeObjectURL(target.previewUrl); + next[index] = { + ...target, + file, + previewUrl: file ? URL.createObjectURL(file) : null, + }; + return next; + }); + }; + + const clearMultiViewSlot = (index: number) => { + handleMultiViewFileChange(index, null); + }; + + const buildGenerationPayload = (taskType: "text_to_model" | "image_to_model" | "multiview_to_model") => { + const payload: Record = { + taskType, + prompt: prompt.trim(), + negativePrompt: negativePrompt.trim(), + modelVersion, + faceLimit, + texture, + pbr, + autoSize, + quad, + generateParts, + smartLowPoly, + geometryQuality, + textureQuality, + modelSeed: modelSeed.trim() ? Number(modelSeed) : undefined, + textureSeed: textureSeed.trim() ? Number(textureSeed) : undefined, + }; + + if (taskType === "image_to_model" || taskType === "multiview_to_model") { + payload.textureAlignment = textureAlignment; + payload.orientation = orientation; + } + + return payload; + }; + + const submitTask = async (params: { + payload: Record; + mode: TaskMode; + promptText: string; + sourceTaskId?: string | null; + exportFormat?: string | null; + }) => { + const createResult = await model3DGenerationService.createTask(params.payload); + if (!createResult.success || !createResult.data?.taskId) { + throw new Error(createResult.error || "创建任务失败"); + } + + const newTask: TaskRecord = { + taskId: createResult.data.taskId, + status: normalizeTaskStatus(createResult.data.status), + prompt: params.promptText, + mode: params.mode, + modelVersion, + createdAt: Date.now(), + sourceTaskId: params.sourceTaskId || null, + exportFormat: params.exportFormat || null, + }; + setCurrentTask(newTask); + upsertTaskHistory(newTask); + return newTask; + }; + + const getReadyTask = () => { + if (!currentTask || currentTask.status !== "success") { + toast.error("请先完成一个成功任务"); + return null; + } + return currentTask; + }; + + const submitDerivedTask = async (params: { + payload: Record; + mode: TaskMode; + promptText: string; + sourceTaskId: string; + successMessage: string; + }) => { + setIsSubmitting(true); + try { + await submitTask({ + payload: params.payload, + mode: params.mode, + promptText: params.promptText, + sourceTaskId: params.sourceTaskId, + }); + toast.success(params.successMessage); + } catch (error) { + toast.error(error instanceof Error ? error.message : "提交任务失败"); + } finally { + setIsSubmitting(false); + } + }; const handleGenerate = async () => { const normalizedPrompt = prompt.trim(); @@ -232,59 +601,77 @@ export default function CanvasModel3DPage() { toast.error("图生3D请先上传参考图"); return; } + const selectedMultiFiles = multiViewSlots.map((slot) => slot.file).filter(Boolean) as File[]; + if (mode === "multiview" && selectedMultiFiles.length < 2) { + toast.error("多视图生成至少上传 2 张图片"); + return; + } setIsSubmitting(true); + setUploadingImage(false); try { - let imageToken: string | undefined; - let filePayload: Record | undefined; + if (mode === "text") { + await submitTask({ + payload: buildGenerationPayload("text_to_model"), + mode: "text", + promptText: normalizedPrompt, + }); + toast.success("已提交文生3D任务"); + return; + } - if (mode === "image" && referenceImage) { + if (mode === "image") { setUploadingImage(true); - const uploadResult = await model3DGenerationService.uploadImage(referenceImage); - setUploadingImage(false); + const uploadResult = await model3DGenerationService.uploadImage(referenceImage as File); if (!uploadResult.success || !uploadResult.data) { throw new Error(uploadResult.error || "上传参考图失败"); } - imageToken = uploadResult.data.imageToken; - filePayload = uploadResult.data.file; - } - - const createPayload: Record = { - taskType: mode === "text" ? "text_to_model" : "image_to_model", - prompt: normalizedPrompt, - modelVersion, - faceLimit, - texture, - pbr, - autoSize, - }; - - if (mode === "image") { - if (imageToken) { - createPayload.imageToken = imageToken; - } else if (filePayload) { - createPayload.file = filePayload; + const payload = buildGenerationPayload("image_to_model"); + if (uploadResult.data.imageToken) { + payload.imageToken = uploadResult.data.imageToken; + } else if (uploadResult.data.file) { + payload.file = uploadResult.data.file; } else { throw new Error("参考图上传未返回可用 token"); } + await submitTask({ + payload, + mode: "image", + promptText: normalizedPrompt || "图生3D", + }); + toast.success("已提交图生3D任务"); + return; } - const createResult = await model3DGenerationService.createTask(createPayload); - if (!createResult.success || !createResult.data?.taskId) { - throw new Error(createResult.error || "创建任务失败"); + setUploadingImage(true); + const uploadResults = await Promise.all( + selectedMultiFiles.map(async (file) => model3DGenerationService.uploadImage(file)) + ); + const filesPayload: Array> = []; + for (const result of uploadResults) { + if (!result.success || !result.data) { + throw new Error(result.error || "上传多视图图片失败"); + } + if (result.data.imageToken) { + filesPayload.push({ + type: "jpg", + file_token: result.data.imageToken, + }); + } else if (result.data.file) { + filesPayload.push(result.data.file); + } } - - const newTask: TaskRecord = { - taskId: createResult.data.taskId, - status: normalizeTaskStatus(createResult.data.status), - prompt: normalizedPrompt, - mode, - modelVersion, - createdAt: Date.now(), - }; - setCurrentTask(newTask); - upsertTaskHistory(newTask); - toast.success("已提交 3D 生成任务"); + if (filesPayload.length < 2) { + throw new Error("多视图图片上传失败，请重试"); + } + const payload = buildGenerationPayload("multiview_to_model"); + payload.files = filesPayload; + await submitTask({ + payload, + mode: "multiview", + promptText: normalizedPrompt || "多视图3D", + }); + toast.success("已提交多视图3D任务"); } catch (error) { toast.error(error instanceof Error ? error.message : "创建任务失败"); } finally { @@ -293,13 +680,8 @@ export default function CanvasModel3DPage() { } }; - const currentModelUrl = currentTask?.modelUrl || null; - const currentPreviewImage = currentTask?.previewImageUrl || referencePreviewUrl || null; - - const importDisabled = !currentTask || currentTask.status !== "success" || !currentModelUrl; - const handleImportToCanvas = async () => { - if (!currentModelUrl || !currentTask) { + if (!currentModelUrl || !currentTask || currentTask.status !== "success") { toast.error("暂无可导入的 3D 模型"); return; } @@ -309,7 +691,7 @@ export default function CanvasModel3DPage() { const viewportZoom = viewport.zoom || 1; const stageWidth = typeof window !== "undefined" ? window.innerWidth : 1280; const stageHeight = typeof window !== "undefined" ? window.innerHeight : 720; - const targetWidth = 360; + const targetWidth = 380; const targetHeight = 300; const x = (stageWidth * 0.48 - viewportX) / viewportZoom - targetWidth / 2; const y = (stageHeight * 0.45 - viewportY) / viewportZoom - targetHeight / 2; @@ -328,7 +710,11 @@ export default function CanvasModel3DPage() { metadata: { taskId: currentTask.taskId, mode: currentTask.mode, + sourceTaskId: currentTask.sourceTaskId, progress: currentTask.progress, + triangles: viewerStats.triangles, + vertices: viewerStats.vertices, + meshCount: viewerStats.meshCount, }, }, connections: [], @@ -355,17 +741,250 @@ export default function CanvasModel3DPage() { link.click(); }; - const statusLabel = useMemo(() => { - if (!currentTask) return "未开始"; - if (currentTask.status === "queued") return "排队中"; - if (currentTask.status === "running") return "生成中"; - if (currentTask.status === "success") return "生成成功"; - if (currentTask.status === "failed") return "生成失败"; - if (currentTask.status === "cancelled") return "已取消"; - if (currentTask.status === "expired") return "已过期"; - if (currentTask.status === "banned") return "审核拦截"; - return "状态未知"; - }, [currentTask]); + const handleConvertAndDownload = async () => { + const readyTask = getReadyTask(); + if (!readyTask) { + return; + } + + setIsSubmitting(true); + try { + const baseTaskId = readyTask.taskId; + const payload: Record = { + taskType: "convert_model", + originalModelTaskId: baseTaskId, + format: exportFormat, + textureSize: 4096, + textureFormat: "JPEG", + }; + const task = await submitTask({ + payload, + mode: "convert", + promptText: `${modeLabel(readyTask.mode)} -> ${exportFormat}`, + sourceTaskId: baseTaskId, + exportFormat, + }); + setAutoDownloadTaskId(task.taskId); + toast.success(`已提交${exportFormat}转换任务，完成后自动下载`); + } catch (error) { + toast.error(error instanceof Error ? error.message : "格式转换失败"); + } finally { + setIsSubmitting(false); + } + }; + + const handleRefineModel = async () => { + const readyTask = getReadyTask(); + if (!readyTask) { + return; + } + + setIsSubmitting(true); + try { + const payload: Record = { + taskType: "refine_model", + draftModelTaskId: readyTask.taskId, + }; + await submitTask({ + payload, + mode: "refine", + promptText: `精修任务（来源 ${readyTask.taskId.slice(0, 8)}）`, + sourceTaskId: readyTask.taskId, + }); + toast.success("已提交模型精修任务"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "提交精修任务失败"); + } finally { + setIsSubmitting(false); + } + }; + + const handleTextureModel = async () => { + const readyTask = getReadyTask(); + if (!readyTask) return; + + await submitDerivedTask({ + payload: { + taskType: "texture_model", + originalModelTaskId: readyTask.taskId, + texture, + pbr, + textureQuality, + textureAlignment, + modelSeed: modelSeed.trim() ? Number(modelSeed) : undefined, + textureSeed: textureSeed.trim() ? Number(textureSeed) : undefined, + texturePromptText: texturePromptText.trim() || undefined, + }, + mode: "texture", + promptText: `纹理重绘（来源 ${readyTask.taskId.slice(0, 8)}）`, + sourceTaskId: readyTask.taskId, + successMessage: "已提交纹理重绘任务", + }); + }; + + const handleStylizeModel = async () => { + const readyTask = getReadyTask(); + if (!readyTask) return; + + await submitDerivedTask({ + payload: { + taskType: "stylize_model", + originalModelTaskId: readyTask.taskId, + style: stylizeStyle, + blockSize: Number.isFinite(stylizeBlockSize) ? stylizeBlockSize : undefined, + }, + mode: "stylize", + promptText: `风格化 ${stylizeStyle}（来源 ${readyTask.taskId.slice(0, 8)}）`, + sourceTaskId: readyTask.taskId, + successMessage: "已提交风格化任务", + }); + }; + + const handleRigCheck = async () => { + const readyTask = getReadyTask(); + if (!readyTask) return; + + await submitDerivedTask({ + payload: { + taskType: "animate_prerigcheck", + originalModelTaskId: readyTask.taskId, + }, + mode: "rigcheck", + promptText: `可绑定检测（来源 ${readyTask.taskId.slice(0, 8)}）`, + sourceTaskId: readyTask.taskId, + successMessage: "已提交可绑定检测任务", + }); + }; + + const handleRigModel = async () => { + const readyTask = getReadyTask(); + if (!readyTask) return; + + await submitDerivedTask({ + payload: { + taskType: "animate_rig", + originalModelTaskId: readyTask.taskId, + outFormat: rigFormat, + rigType, + spec: rigSpec, + }, + mode: "rig", + promptText: `自动绑定 ${rigType}（来源 ${readyTask.taskId.slice(0, 8)}）`, + sourceTaskId: readyTask.taskId, + successMessage: "已提交自动绑定任务", + }); + }; + + const handleRetargetAnimation = async () => { + const readyTask = getReadyTask(); + if (!readyTask) return; + + await submitDerivedTask({ + payload: { + taskType: "animate_retarget", + originalModelTaskId: readyTask.taskId, + animation: retargetAnimation, + outFormat: rigFormat, + bakeAnimation: retargetBake, + exportWithGeometry: retargetExportWithGeometry, + animateInPlace: retargetInPlace, + }, + mode: "retarget", + promptText: `动作 ${retargetAnimation}（来源 ${readyTask.taskId.slice(0, 8)}）`, + sourceTaskId: readyTask.taskId, + successMessage: "已提交动作重定向任务", + }); + }; + + const handleMeshSegmentation = async () => { + const readyTask = getReadyTask(); + if (!readyTask) return; + + await submitDerivedTask({ + payload: { + taskType: "mesh_segmentation", + originalModelTaskId: readyTask.taskId, + }, + mode: "segment", + promptText: `网格分割（来源 ${readyTask.taskId.slice(0, 8)}）`, + sourceTaskId: readyTask.taskId, + successMessage: "已提交网格分割任务", + }); + }; + + const handleMeshCompletion = async () => { + const readyTask = getReadyTask(); + if (!readyTask) return; + + const parts = completionPartNames + .split(",") + .map((item) => item.trim()) + .filter(Boolean); + + await submitDerivedTask({ + payload: { + taskType: "mesh_completion", + originalModelTaskId: readyTask.taskId, + partNames: parts.length ? parts : undefined, + }, + mode: "complete", + promptText: `网格补全（来源 ${readyTask.taskId.slice(0, 8)}）`, + sourceTaskId: readyTask.taskId, + successMessage: "已提交网格补全任务", + }); + }; + + const handleLowpolyOptimize = async () => { + const readyTask = getReadyTask(); + if (!readyTask) return; + + await submitDerivedTask({ + payload: { + taskType: "highpoly_to_lowpoly", + originalModelTaskId: readyTask.taskId, + faceLimit: Number.isFinite(lowpolyFaceLimit) ? lowpolyFaceLimit : undefined, + quad: lowpolyQuad, + bake: lowpolyBake, + }, + mode: "lowpoly", + promptText: `低模优化（来源 ${readyTask.taskId.slice(0, 8)}）`, + sourceTaskId: readyTask.taskId, + successMessage: "已提交低模优化任务", + }); + }; + + const handleSelectTask = (task: TaskRecord) => { + setCurrentTask(task); + if (task.status === "success" && task.modelUrl) { + setViewerLayers([]); + setLayerVisibility({}); + setIsolatedLayerId(null); + } + }; + + const handleClearHistory = () => { + setTaskHistory([]); + setCurrentTask(null); + if (typeof window !== "undefined") { + window.localStorage.removeItem(historyStorageKey); + } + }; + + const handleLayersChange = useCallback((layers: Model3DLayer[]) => { + setViewerLayers(layers); + setLayerVisibility((prev) => { + const next: Record = {}; + for (const layer of layers) { + next[layer.id] = prev[layer.id] !== false; + } + return next; + }); + setIsolatedLayerId((prev) => (prev && layers.some((layer) => layer.id === prev) ? prev : null)); + }, []); + + const importDisabled = !currentTask || currentTask.status !== "success" || !currentModelUrl; + const activeStatusLabel = currentTask ? statusLabel(currentTask.status) : "未开始"; + const activePromptSummary = currentTask?.prompt?.trim() || "尚未创建任务"; return (
@@ -377,7 +996,7 @@ export default function CanvasModel3DPage() {

3D 生成工作台

- Tripo3D 文生模型 / 图生模型 · 一键导入无限画布 + Tripo3D 文生模型 / 图生模型 / 多视图模型 · 导入无限画布

@@ -391,28 +1010,35 @@ export default function CanvasModel3DPage() {
-
+
生成参数 - 参考图布局，使用站内 UI 规范 + 补齐多视图与高级参数 -
+
+
@@ -421,8 +1047,18 @@ export default function CanvasModel3DPage() { id="model3d-prompt" value={prompt} onChange={(event) => setPrompt(event.target.value)} - placeholder="例如：一个Q版机器人手办，白色机身，PBR材质" - className="min-h-24 resize-none" + placeholder="例如：北欧风单人椅，白色框架，棕色皮革坐垫" + className="min-h-20 resize-none" + /> +
+ +
+ + setNegativePrompt(event.target.value)} + placeholder="例如：低质量、破损、模糊" />
@@ -436,8 +1072,8 @@ export default function CanvasModel3DPage() { "hover:bg-muted/50" )} > - - {referenceImage ? referenceImage.name : "点击上传 PNG/JPG/WEBP"} + + {referenceImage ? referenceImage.name : "上传 PNG/JPG/WEBP（<=10MB）"} setReferenceImage(event.target.files?.[0] || null)} /> {referencePreviewUrl && ( - 参考图预览 +
+ 参考图预览 + +
)}
)} + {mode === "multiview" && ( +
+ +
+ {multiViewSlots.map((slot, index) => ( +
+ + {slot.previewUrl ? ( + + ) : null} +
+ ))} +
+
+ )} +
+
+
+ + +
+
+ + +
+
+
-
-
- 生成贴图 - -
-
- PBR 材质 - -
-
- 自动尺寸 - -
-
+ + + + + + {(mode === "image" || mode === "multiview") && ( +
+
+ + +
+
+ + +
+
+ )} + +
+
+ + setModelSeed(event.target.value)} + /> +
+
+ + setTextureSeed(event.target.value)} + /> +
+
+ +
+
+ 生成贴图 + +
+
+ PBR 材质 + +
+
+ 自动尺寸 + +
+
+ 四边面优化 + +
+
+ 分部件生成 + +
+
+ 智能低模 + +
+
+ + + + +
+
+ + +
+
+ + + setViewerSettings((prev) => ({ ...prev, lightIntensity: values[0] ?? 1 })) + } + /> +
+
+
+
+ 自动旋转 + + setViewerSettings((prev) => ({ ...prev, autoRotate: checked })) + } + /> +
+
+ 显示网格 + + setViewerSettings((prev) => ({ ...prev, showGrid: checked })) + } + /> +
+
+ 线框模式 + + setViewerSettings((prev) => ({ ...prev, wireframe: checked })) + } + /> +
+
+ + + +
- +
+ +
+ + +
+ + + + + + +
+

风格化 / 纹理重绘

+
+ + setStylizeBlockSize(Number(event.target.value || 80))} + min={10} + max={300} + placeholder="block size" + /> +
+