feat: 完善无限画布3D生成与实时画布能力
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-02-28 22:33:16 +08:00
parent d272bd5ee8
commit f6979f0196
17 changed files with 3984 additions and 371 deletions

View File

@ -91,6 +91,8 @@ import {
} from "@/lib/payment/dulupay-config";
const fetcher = (url: string) => fetch(url).then((res) => res.json());
const DEFAULT_RUNWARE_REALTIME_MODEL = "runware:101@1";
const DEFAULT_RUNWARE_STYLIZE_MODEL = "runware:106@1";
// 类型定义
type User = {
@ -732,6 +734,11 @@ export function AdminSettingsClient() {
const [realtimeCanvasConfig, setRealtimeCanvasConfig] = useState({
enabled: true,
falApiKey: "",
runwareModel: DEFAULT_RUNWARE_REALTIME_MODEL,
runwareStylizeModel: DEFAULT_RUNWARE_STYLIZE_MODEL,
runwareControlModel: "runware:25@1",
runwareControlPreprocessor: "canny" as "canny" | "lineart" | "depth",
renderMode: "balanced" as "strict-structure" | "balanced" | "stylize",
throttleInterval: 100,
maxBuffering: 2,
maxInputResolution: 1024,
@ -877,7 +884,13 @@ export function AdminSettingsClient() {
if (realtimeCanvasConfigData?.setting?.value) {
const config = realtimeCanvasConfigData.setting.value as {
enabled?: boolean;
runwareApiKey?: string;
falApiKey?: string;
runwareModel?: string;
runwareStylizeModel?: string;
runwareControlModel?: string;
runwareControlPreprocessor?: "canny" | "lineart" | "depth";
renderMode?: "strict-structure" | "balanced" | "stylize";
throttleInterval?: number;
maxBuffering?: number;
maxInputResolution?: number;
@ -891,7 +904,19 @@ export function AdminSettingsClient() {
};
setRealtimeCanvasConfig({
enabled: config.enabled ?? true,
falApiKey: config.falApiKey || "",
falApiKey: config.runwareApiKey || config.falApiKey || "",
runwareModel: config.runwareModel || DEFAULT_RUNWARE_REALTIME_MODEL,
runwareStylizeModel: config.runwareStylizeModel || DEFAULT_RUNWARE_STYLIZE_MODEL,
runwareControlModel: config.runwareControlModel || "runware:25@1",
runwareControlPreprocessor:
config.runwareControlPreprocessor === "lineart" ||
config.runwareControlPreprocessor === "depth"
? config.runwareControlPreprocessor
: "canny",
renderMode:
config.renderMode === "strict-structure" || config.renderMode === "stylize"
? config.renderMode
: "balanced",
throttleInterval: Number(config.throttleInterval || 100),
maxBuffering: Number(config.maxBuffering || 2),
maxInputResolution: Number(config.maxInputResolution || 1024),
@ -1756,11 +1781,39 @@ export function AdminSettingsClient() {
// --- 实时画布API配置逻辑 ---
const handleSaveRealtimeCanvasConfig = async () => {
try {
if (realtimeCanvasConfig.enabled && !realtimeCanvasConfig.falApiKey.trim()) {
toast.error("启用实时画布功能需要配置 fal.ai API Key");
const runwareApiKey = realtimeCanvasConfig.falApiKey.trim();
const runwareModel =
realtimeCanvasConfig.runwareModel.trim() || DEFAULT_RUNWARE_REALTIME_MODEL;
const runwareStylizeModel =
realtimeCanvasConfig.runwareStylizeModel.trim() || DEFAULT_RUNWARE_STYLIZE_MODEL;
const runwareControlModel =
realtimeCanvasConfig.runwareControlModel.trim() || "runware:25@1";
if (realtimeCanvasConfig.enabled && !runwareApiKey) {
toast.error("启用实时画布功能需要配置 Runware API Key");
return;
}
if (realtimeCanvasConfig.enabled) {
for (const modelId of [runwareModel, runwareStylizeModel].filter(Boolean)) {
const probeResponse = await fetch("/api/admin/realtime/model-capability", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
apiKey: runwareApiKey,
modelId,
}),
});
const probeData = await probeResponse.json().catch(() => null);
if (!probeResponse.ok || probeData?.supported === false) {
toast.error(
probeData?.message || "模型能力探针失败:当前模型不支持 seedImage,已阻止保存"
);
return;
}
}
}
const response = await fetch("/api/admin/settings", {
method: "POST",
headers: { "Content-Type": "application/json" },
@ -1768,7 +1821,13 @@ export function AdminSettingsClient() {
key: "realtime_canvas_config",
value: {
enabled: realtimeCanvasConfig.enabled,
falApiKey: realtimeCanvasConfig.falApiKey.trim(),
runwareApiKey,
falApiKey: runwareApiKey,
runwareModel,
runwareStylizeModel,
runwareControlModel,
runwareControlPreprocessor: realtimeCanvasConfig.runwareControlPreprocessor,
renderMode: realtimeCanvasConfig.renderMode,
throttleInterval: Math.min(260, Math.max(60, Number(realtimeCanvasConfig.throttleInterval) || 100)),
maxBuffering: Math.min(6, Math.max(1, Number(realtimeCanvasConfig.maxBuffering) || 2)),
maxInputResolution: Math.min(
@ -1786,10 +1845,13 @@ export function AdminSettingsClient() {
}),
});
if (!response.ok) throw new Error("保存失败");
if (!response.ok) {
const errorData = await response.json().catch(() => null);
throw new Error(errorData?.message || "保存失败");
}
toast.success("实时画布配置保存成功");
} catch (_error) {
toast.error("保存实时画布配置失败");
} catch (error) {
toast.error(error instanceof Error ? error.message : "保存实时画布配置失败");
}
};
@ -7598,7 +7660,7 @@ export function AdminSettingsClient() {
实时画布配置
</CardTitle>
<CardDescription>
配置实时画布(左画右预览)功能使用的 fal realtime 参数与 API Key。该 Key 存储在数据库,前端不暴露。
配置实时画布功能的 Runware 参数、模型与 API Key。保存时会自动探针校验模型是否支持 seedImage。
</CardDescription>
</CardHeader>
<CardContent className="space-y-6 pt-6">
@ -7624,7 +7686,7 @@ export function AdminSettingsClient() {
<div className="space-y-2">
<Label htmlFor="realtimeFalApiKey">
fal.ai API Key <span className="text-destructive">*</span>
Runware API Key <span className="text-destructive">*</span>
</Label>
<Input
id="realtimeFalApiKey"
@ -7633,11 +7695,103 @@ export function AdminSettingsClient() {
onChange={(e) =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, falApiKey: e.target.value })
}
placeholder="fal_xxxxxxxxxxxxx"
placeholder="rw_xxxxxxxxxxxxx"
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="runwareModel">
Runware 模型 ID <span className="text-destructive">*</span>
</Label>
<Input
id="runwareModel"
value={realtimeCanvasConfig.runwareModel}
onChange={(e) =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, runwareModel: e.target.value })
}
placeholder={DEFAULT_RUNWARE_REALTIME_MODEL}
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="runwareStylizeModel">写实增强模型 ID</Label>
<Input
id="runwareStylizeModel"
value={realtimeCanvasConfig.runwareStylizeModel}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
runwareStylizeModel: e.target.value,
})
}
placeholder={DEFAULT_RUNWARE_STYLIZE_MODEL}
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="runwareControlModel">ControlNet 模型 ID</Label>
<Input
id="runwareControlModel"
value={realtimeCanvasConfig.runwareControlModel}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
runwareControlModel: e.target.value,
})
}
placeholder="runware:25@1"
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
<div className="grid grid-cols-2 gap-4">
<div className="space-y-2">
<Label>实时渲染模式</Label>
<Select
value={realtimeCanvasConfig.renderMode}
onValueChange={(
value: "strict-structure" | "balanced" | "stylize"
) =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, renderMode: value })
}
>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="strict-structure">结构优先</SelectItem>
<SelectItem value="balanced">平衡模式</SelectItem>
<SelectItem value="stylize">风格优先</SelectItem>
</SelectContent>
</Select>
</div>
<div className="space-y-2">
<Label>ControlNet 预处理</Label>
<Select
value={realtimeCanvasConfig.runwareControlPreprocessor}
onValueChange={(value: "canny" | "lineart" | "depth") =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
runwareControlPreprocessor: value,
})
}
>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="canny">canny</SelectItem>
<SelectItem value="lineart">lineart</SelectItem>
<SelectItem value="depth">depth</SelectItem>
</SelectContent>
</Select>
</div>
</div>
<p className="text-xs text-muted-foreground">
推荐:主模型 {DEFAULT_RUNWARE_REALTIME_MODEL} + ControlNet {`runware:25@1`}。写实模型 {DEFAULT_RUNWARE_STYLIZE_MODEL} 作为回退链路。
</p>
<div className="grid grid-cols-2 gap-4">
<div className="space-y-2">
<Label>发送节流(ms)</Label>

View File

@ -0,0 +1,41 @@
import { z } from "zod";
import { requireAdmin } from "@/lib/middleware/admin";
import { ChatSDKError } from "@/lib/errors";
import { probeRunwareSeedImageSupport } from "@/lib/realtime/runware-capability";
const schema = z.object({
apiKey: z.string().min(1),
modelId: z.string().min(1),
});
function mapStatus(code: string) {
if (code === "UPSTREAM_ERROR") return 502;
return 400;
}
export async function POST(request: Request) {
try {
await requireAdmin();
const body = await request.json();
const data = schema.parse(body);
const result = await probeRunwareSeedImageSupport({
apiKey: data.apiKey,
modelId: data.modelId,
});
if (!result.supported) {
return Response.json(result, { status: mapStatus(result.code) });
}
return Response.json(result);
} catch (error) {
if (error instanceof ChatSDKError) {
return error.toResponse();
}
if (error instanceof z.ZodError) {
return new ChatSDKError("bad_request:api", error.message).toResponse();
}
return new ChatSDKError("offline:admin").toResponse();
}
}

View File

@ -6,12 +6,26 @@ import {
upsertSystemSetting,
} from "@/lib/db/queries";
import { ChatSDKError } from "@/lib/errors";
import { probeRunwareSeedImageSupport } from "@/lib/realtime/runware-capability";
const settingSchema = z.object({
key: z.string(),
value: z.any(),
});
const DEFAULT_RUNWARE_MODEL = "runware:101@1";
const DEFAULT_RUNWARE_STYLIZE_MODEL = "runware:106@1";
const DEFAULT_RUNWARE_CONTROL_MODEL = "runware:25@1";
function isObjectLike(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
function mapProbeStatus(code: string) {
if (code === "UPSTREAM_ERROR") return 502;
return 400;
}
export async function GET(request: Request) {
try {
await requireAdmin();
@ -41,9 +55,69 @@ export async function POST(request: Request) {
const body = await request.json();
const { key, value } = settingSchema.parse(body);
let nextValue = value;
if (key === "realtime_canvas_config") {
if (!isObjectLike(value)) {
return new ChatSDKError("bad_request:api", "realtime_canvas_config value 必须为对象").toResponse();
}
const enabled = value.enabled !== false;
const runwareApiKey = String(value.runwareApiKey || value.falApiKey || value.apiKey || "").trim();
const runwareModel = String(value.runwareModel || DEFAULT_RUNWARE_MODEL).trim() || DEFAULT_RUNWARE_MODEL;
const runwareStylizeModel =
String(value.runwareStylizeModel || DEFAULT_RUNWARE_STYLIZE_MODEL).trim() ||
DEFAULT_RUNWARE_STYLIZE_MODEL;
const runwareControlModel =
String(value.runwareControlModel || DEFAULT_RUNWARE_CONTROL_MODEL).trim() ||
DEFAULT_RUNWARE_CONTROL_MODEL;
const runwareControlPreprocessor =
value.runwareControlPreprocessor === "lineart" || value.runwareControlPreprocessor === "depth"
? value.runwareControlPreprocessor
: "canny";
const renderMode =
value.renderMode === "strict-structure" ||
value.renderMode === "stylize" ||
value.renderMode === "balanced"
? value.renderMode
: "balanced";
if (enabled) {
if (!runwareApiKey) {
return new ChatSDKError("bad_request:api", "启用实时画布必须配置 Runware API Key").toResponse();
}
for (const modelId of [runwareModel, runwareStylizeModel].filter(Boolean)) {
const probeResult = await probeRunwareSeedImageSupport({
apiKey: runwareApiKey,
modelId,
});
if (!probeResult.supported) {
return Response.json(
{
code: probeResult.code,
message: probeResult.message,
},
{ status: mapProbeStatus(probeResult.code) }
);
}
}
}
nextValue = {
...value,
runwareApiKey,
falApiKey: runwareApiKey,
runwareModel,
runwareStylizeModel,
runwareControlModel,
runwareControlPreprocessor,
renderMode,
};
}
const setting = await upsertSystemSetting({
key,
value,
value: nextValue,
updatedBy: session.user.id,
});
@ -58,4 +132,3 @@ export async function POST(request: Request) {
return new ChatSDKError("offline:admin").toResponse();
}
}

View File

@ -7,14 +7,44 @@ export const runtime = "nodejs";
export const maxDuration = 60;
type CreateTaskRequest = {
taskType: "text_to_model" | "image_to_model" | "convert_model";
taskType:
| "text_to_model"
| "image_to_model"
| "multiview_to_model"
| "convert_model"
| "refine_model"
| "stylize_model"
| "texture_model"
| "animate_prerigcheck"
| "animate_rig"
| "animate_retarget"
| "mesh_segmentation"
| "mesh_completion"
| "highpoly_to_lowpoly";
prompt?: string;
imageToken?: string;
imageUrl?: string;
imageType?: string;
file?: Record<string, unknown>;
files?: Array<Record<string, unknown>>;
imageTokens?: string[];
imageUrls?: string[];
originalModelTaskId?: string;
draftModelTaskId?: string;
format?: "GLTF" | "USDZ" | "FBX" | "OBJ" | "STL" | "3MF";
style?: "lego" | "voxel" | "voronoi" | "minecraft";
blockSize?: number;
outFormat?: "glb" | "fbx";
rigType?: "biped" | "quadruped" | "hexapod" | "octopod" | "avian" | "serpentine" | "aquatic" | "others";
spec?: "mixamo" | "tripo";
animation?: string;
animations?: string[];
texturePromptText?: string;
texturePromptImageToken?: string;
texturePromptStyleImageToken?: string;
partNames?: string[];
bakeAnimation?: boolean;
exportWithGeometry?: boolean;
modelVersion?: string;
negativePrompt?: string;
faceLimit?: number;
@ -116,6 +146,48 @@ export async function POST(request: Request) {
payload.file = inputFile;
}
if (taskType === "multiview_to_model") {
const inputFiles: Array<Record<string, unknown>> = [];
if (Array.isArray(body.files)) {
for (const item of body.files) {
if (item && typeof item === "object") {
inputFiles.push(item);
}
}
}
if (inputFiles.length === 0 && Array.isArray(body.imageTokens)) {
for (const token of body.imageTokens) {
const normalized = String(token || "").trim();
if (!normalized) continue;
inputFiles.push({
type: String(body.imageType || "jpg"),
file_token: normalized,
});
}
}
if (inputFiles.length === 0 && Array.isArray(body.imageUrls)) {
for (const imageUrl of body.imageUrls) {
const normalized = String(imageUrl || "").trim();
if (!normalized) continue;
inputFiles.push({
type: String(body.imageType || "jpg"),
url: normalized,
});
}
}
if (inputFiles.length < 2) {
return NextResponse.json(
{ success: false, error: "多视图生成至少需要 2 张参考图" },
{ status: 400 }
);
}
payload.files = inputFiles;
}
if (taskType === "convert_model") {
const originalModelTaskId = String(body.originalModelTaskId || "").trim();
if (!originalModelTaskId) {
@ -131,6 +203,85 @@ export async function POST(request: Request) {
payload.format = body.format;
}
if (
taskType === "stylize_model" ||
taskType === "texture_model" ||
taskType === "animate_prerigcheck" ||
taskType === "animate_rig" ||
taskType === "animate_retarget" ||
taskType === "mesh_segmentation" ||
taskType === "mesh_completion" ||
taskType === "highpoly_to_lowpoly"
) {
const originalModelTaskId = String(body.originalModelTaskId || "").trim();
if (!originalModelTaskId) {
return NextResponse.json(
{ success: false, error: "缺少 originalModelTaskId" },
{ status: 400 }
);
}
payload.original_model_task_id = originalModelTaskId;
}
if (taskType === "stylize_model") {
if (!body.style) {
return NextResponse.json({ success: false, error: "缺少 style" }, { status: 400 });
}
payload.style = body.style;
}
if (taskType === "animate_retarget") {
const singleAnimation = String(body.animation || "").trim();
const animationList = Array.isArray(body.animations)
? body.animations.map((item) => String(item || "").trim()).filter(Boolean)
: [];
if (!singleAnimation && animationList.length === 0) {
return NextResponse.json(
{ success: false, error: "缺少 animation / animations" },
{ status: 400 }
);
}
if (singleAnimation) {
payload.animation = singleAnimation;
} else {
payload.animations = animationList;
}
}
if (taskType === "refine_model") {
const draftModelTaskId = String(body.draftModelTaskId || "").trim();
if (!draftModelTaskId) {
return NextResponse.json(
{ success: false, error: "缺少 draftModelTaskId" },
{ status: 400 }
);
}
payload.draft_model_task_id = draftModelTaskId;
}
if (taskType === "texture_model") {
const texturePromptText = String(body.texturePromptText || "").trim();
const texturePromptImageToken = String(body.texturePromptImageToken || "").trim();
const texturePromptStyleImageToken = String(body.texturePromptStyleImageToken || "").trim();
if (texturePromptText || texturePromptImageToken || texturePromptStyleImageToken) {
const texturePrompt: Record<string, unknown> = {};
if (texturePromptText) texturePrompt.text = texturePromptText;
if (texturePromptImageToken) {
texturePrompt.image = {
type: String(body.imageType || "jpg"),
file_token: texturePromptImageToken,
};
}
if (texturePromptStyleImageToken) {
texturePrompt.style_image = {
type: String(body.imageType || "jpg"),
file_token: texturePromptStyleImageToken,
};
}
payload.texture_prompt = texturePrompt;
}
}
assignMapped(payload, body as Record<string, unknown>, {
modelVersion: "model_version",
negativePrompt: "negative_prompt",
@ -154,6 +305,13 @@ export async function POST(request: Request) {
textureSize: "texture_size",
textureFormat: "texture_format",
scaleFactor: "scale_factor",
blockSize: "block_size",
outFormat: "out_format",
rigType: "rig_type",
spec: "spec",
partNames: "part_names",
bakeAnimation: "bake_animation",
exportWithGeometry: "export_with_geometry",
pivotToCenterBottom: "pivot_to_center_bottom",
withAnimation: "with_animation",
packUv: "pack_uv",
@ -220,4 +378,3 @@ export async function POST(request: Request) {
);
}
}

View File

@ -19,6 +19,74 @@ type EnhanceApiConfig = {
backupModel?: string;
};
const AOS_MAX_RESOLUTION = 4096;
const ENHANCE_CREATE_TIMEOUT_MS = 120_000;
const ENHANCE_CREATE_RETRY_COUNT = 1;
function getAosFriendlyErrorMessage(detail: string): string | null {
const resolutionMatch = detail.match(/resolution excess,\s*width=(\d+),\s*height=(\d+)/i);
if (resolutionMatch) {
const [, width, height] = resolutionMatch;
return `图片分辨率超限(${width}x${height}),AOS要求宽高都不超过 ${AOS_MAX_RESOLUTION}px。请缩小后重试。`;
}
if (/invalid file/i.test(detail)) {
return "图片文件无法识别,请尝试转换为 JPG/PNG 后重试。";
}
return null;
}
function isRetryableNetworkError(error: unknown): boolean {
if (!(error instanceof Error)) return false;
if (error.name === "AbortError") return true;
const message = error.message || "";
const cause = (error as Error & { cause?: { message?: string; code?: string } }).cause;
const causeMessage = cause?.message || "";
const causeCode = cause?.code || "";
const combined = `${message} ${causeMessage} ${causeCode}`;
return /EHOSTUNREACH|ENOTFOUND|ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|No route to host/i.test(combined);
}
async function ensureAosResolutionLimit(
buffer: Buffer,
logPrefix: string
): Promise<Buffer> {
try {
const sharp = await import("sharp");
const image = sharp.default(buffer);
const metadata = await image.metadata();
const width = metadata.width;
const height = metadata.height;
if (!width || !height) {
return buffer;
}
if (width <= AOS_MAX_RESOLUTION && height <= AOS_MAX_RESOLUTION) {
return buffer;
}
const resizedBuffer = await image
.resize(AOS_MAX_RESOLUTION, AOS_MAX_RESOLUTION, {
fit: "inside",
withoutEnlargement: true,
})
.toBuffer();
const resizedMeta = await sharp.default(resizedBuffer).metadata();
console.warn(
`[${logPrefix}] 输入图片分辨率超限,已自动缩放: ${width}x${height} -> ${resizedMeta.width}x${resizedMeta.height}`
);
return resizedBuffer;
} catch (error) {
console.error(`[${logPrefix}] 自动缩放失败,继续使用原图:`, error);
return buffer;
}
}
/**
* 压缩图片生成缩略图(使用 sharp)
*/
@ -277,7 +345,10 @@ async function pollTaskResult(
// state < 0 表示失败
if (state < 0) {
throw result;
const errMessage = String(
result.data?.err_message || result.data?.err_info || result.data?.state_detail || `state=${state}`
);
throw new Error(`AOS任务失败: ${errMessage}`);
}
// progress >= 100 表示完成
@ -294,11 +365,11 @@ async function pollTaskResult(
const result = await getTaskResult();
return result;
} catch (error) {
if (error && typeof error === 'object' && 'data' in error) {
if (error && typeof error === "object" && "data" in error) {
throw new Error(JSON.stringify(error, null, 2));
}
if (timeout <= 0) {
throw new Error('轮询超时');
throw new Error("轮询超时");
}
await new Promise((resolve) => {
setTimeout(() => {
@ -533,7 +604,8 @@ export async function POST(request: NextRequest) {
// 将Base64转换为Buffer
console.log("[Enhance API] 转换Base64为Buffer,长度:", imageBase64.length);
const imageBuffer = Buffer.from(imageBase64, "base64");
const rawImageBuffer = Buffer.from(imageBase64, "base64");
const imageBuffer = await ensureAosResolutionLimit(rawImageBuffer, "Enhance API");
console.log("[Enhance API] Buffer创建完成,大小:", imageBuffer.length, "bytes");
// 创建FormData(使用form-data包,Node.js环境)
@ -564,10 +636,6 @@ export async function POST(request: NextRequest) {
try {
console.log("[Enhance API] 开始创建任务...");
// 添加超时控制(60秒)
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 60000);
// 将FormData转换为Buffer(form-data包是stream)
console.log("[Enhance API] 转换FormData为Buffer...");
const formDataBuffer = await new Promise<Buffer>((resolve, reject) => {
@ -603,20 +671,44 @@ export async function POST(request: NextRequest) {
});
console.log("[Enhance API] FormData Buffer转换完成,大小:", formDataBuffer.length, "bytes");
// 将Buffer转换为Uint8Array用于fetch(更兼容)
const uint8Array = new Uint8Array(formDataBuffer);
console.log("[Enhance API] 发送创建任务请求...");
const createResponse = await fetch(enhanceUrl, {
method: "POST",
headers,
body: uint8Array,
signal: controller.signal,
});
console.log("[Enhance API] 创建任务响应状态:", createResponse.status, createResponse.statusText);
clearTimeout(timeoutId);
let createResponse: Response | null = null;
let lastFetchError: unknown;
for (let attempt = 1; attempt <= ENHANCE_CREATE_RETRY_COUNT + 1; attempt++) {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), ENHANCE_CREATE_TIMEOUT_MS);
try {
console.log(`[Enhance API] 发送创建任务请求... attempt=${attempt}`);
createResponse = await fetch(enhanceUrl, {
method: "POST",
headers,
body: uint8Array,
signal: controller.signal,
});
console.log("[Enhance API] 创建任务响应状态:", createResponse.status, createResponse.statusText);
clearTimeout(timeoutId);
break;
} catch (fetchError) {
clearTimeout(timeoutId);
lastFetchError = fetchError;
const canRetry = attempt <= ENHANCE_CREATE_RETRY_COUNT && isRetryableNetworkError(fetchError);
if (canRetry) {
console.warn(
`[Enhance API] 创建任务失败,准备重试: attempt=${attempt}, error=${fetchError instanceof Error ? fetchError.message : String(fetchError)}`
);
await new Promise((resolve) => setTimeout(resolve, 1500));
continue;
}
throw fetchError;
}
}
if (!createResponse) {
throw lastFetchError instanceof Error ? lastFetchError : new Error("创建任务请求失败");
}
if (!createResponse.ok) {
const errorText = await createResponse.text();
@ -641,34 +733,34 @@ export async function POST(request: NextRequest) {
// 检查是否是网络连接错误
if (fetchError instanceof Error) {
if (fetchError.name === 'AbortError') {
if (fetchError.name === "AbortError") {
return NextResponse.json(
{
error: "变清晰API请求超时",
detail: "创建任务请求超过60秒未响应,请检查API服务器状态或网络连接",
detail: "创建任务请求超过120秒未响应,请检查API服务器状态或网络连接",
},
{ status: 504 }
);
}
// 检查错误消息和cause中的网络错误
const errorMessage = fetchError.message || '';
const errorMessage = fetchError.message || "";
const errorCause = (fetchError as any).cause;
const causeMessage = errorCause?.message || '';
const causeCode = errorCause?.code || '';
const causeMessage = errorCause?.message || "";
const causeCode = errorCause?.code || "";
// 检查各种网络连接错误
if (errorMessage.includes('EHOSTUNREACH') ||
errorMessage.includes('ENOTFOUND') ||
errorMessage.includes('ECONNREFUSED') ||
errorMessage.includes('No route to host') ||
causeMessage.includes('EHOSTUNREACH') ||
causeMessage.includes('ENOTFOUND') ||
causeMessage.includes('ECONNREFUSED') ||
causeMessage.includes('No route to host') ||
causeCode === 'EHOSTUNREACH' ||
causeCode === 'ENOTFOUND' ||
causeCode === 'ECONNREFUSED') {
if (errorMessage.includes("EHOSTUNREACH") ||
errorMessage.includes("ENOTFOUND") ||
errorMessage.includes("ECONNREFUSED") ||
errorMessage.includes("No route to host") ||
causeMessage.includes("EHOSTUNREACH") ||
causeMessage.includes("ENOTFOUND") ||
causeMessage.includes("ECONNREFUSED") ||
causeMessage.includes("No route to host") ||
causeCode === "EHOSTUNREACH" ||
causeCode === "ENOTFOUND" ||
causeCode === "ECONNREFUSED") {
return NextResponse.json(
{
error: "无法连接到变清晰API服务器",
@ -690,7 +782,17 @@ export async function POST(request: NextRequest) {
// 4. 轮询获取任务结果(根据官方示例)
console.log(`[Enhance] 开始轮询任务结果,task_id: ${taskId}`);
const taskResult = await pollTaskResult(apiUrl, config.apiKey!, taskId);
let taskResult: any;
try {
taskResult = await pollTaskResult(apiUrl, config.apiKey!, taskId);
} catch (pollError) {
const detail = pollError instanceof Error ? pollError.message : "轮询失败";
const friendlyMessage = getAosFriendlyErrorMessage(detail);
return NextResponse.json(
{ error: "变清晰处理失败", detail: friendlyMessage || detail },
{ status: friendlyMessage ? 400 : 500 }
);
}
const finalProcessingTime = (Date.now() - startTime) / 1000;
@ -847,4 +949,3 @@ export async function POST(request: NextRequest) {
);
}
}

View File

@ -5,6 +5,9 @@ type RealtimeCanvasConfig = {
enabled?: boolean;
runwareApiKey?: string;
runwareModel?: string;
runwareControlModel?: string;
runwareControlPreprocessor?: "canny" | "lineart" | "depth";
renderMode?: "strict-structure" | "balanced" | "stylize";
falApiKey?: string;
apiKey?: string;
strictSketch?: boolean;
@ -163,6 +166,18 @@ export async function GET() {
},
behavior: {
strictSketch: config.strictSketch !== false,
renderMode:
config.renderMode === "strict-structure" || config.renderMode === "stylize"
? config.renderMode
: "balanced",
},
control: {
model: String(config.runwareControlModel || "runware:25@1").trim() || "runware:25@1",
preprocessor:
config.runwareControlPreprocessor === "lineart" ||
config.runwareControlPreprocessor === "depth"
? config.runwareControlPreprocessor
: "canny",
},
});
}

View File

@ -1,10 +1,18 @@
import { auth } from "@/app/(auth)/auth";
import { getSystemSetting } from "@/lib/db/queries";
import { getRunwareWebSocketClient } from "@/lib/realtime/runware-ws";
type RealtimeCanvasConfig = {
enabled?: boolean;
runwareApiKey?: string;
runwareModel?: string;
runwareStylizeModel?: string;
runwareControlModel?: string;
runwareControlPreprocessor?: "canny" | "lineart" | "depth";
renderMode?: "strict-structure" | "balanced" | "stylize";
runwareUseWebSocket?: boolean;
falRealtimeEnabled?: boolean;
falRealtimeModel?: string;
falApiKey?: string;
apiKey?: string;
negativePrompt?: string;
@ -17,9 +25,23 @@ type RealtimeCanvasConfig = {
};
};
type AngleEditConfig = {
falApiKey?: string;
};
type GeneratePayload = {
requestId?: string;
imageUrl?: string;
maskImageUrl?: string;
qualityProfile?: "preview" | "refine";
roi?:
| {
x?: number;
y?: number;
width?: number;
height?: number;
}
| null;
prompt?: string;
seed?: number;
strictSketch?: boolean;
@ -29,6 +51,12 @@ type GeneratePayload = {
strength?: number;
};
const FALLBACK_REALTIME_PROMPT =
"严格按照输入草图生成:保持主体轮廓、位置、比例和数量一致,只做材质与光影细化;禁止改变构图,禁止新增无关背景纹理。";
const FALLBACK_FAL_REALTIME_MODEL = "fal-ai/flux-2/klein/realtime";
const FALLBACK_RUNWARE_STYLIZE_MODEL = "runware:106@1";
const FALLBACK_RUNWARE_CONTROL_MODEL = "runware:25@1";
function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(min, value));
}
@ -63,6 +91,11 @@ function parseRunwareError(payload: any) {
}
if (typeof payload?.error === "string") return payload.error;
if (typeof payload?.message === "string") return payload.message;
if (Array.isArray(payload?.errors) && payload.errors.length > 0) {
const firstError = payload.errors[0];
if (typeof firstError?.message === "string") return firstError.message;
if (typeof firstError?.code === "string") return firstError.code;
}
return "";
}
@ -74,6 +107,128 @@ function resolveSeed(seed: unknown) {
return normalized;
}
function shouldRetryWithoutMask(errorMessage: string) {
const normalized = errorMessage.toLowerCase();
return (
normalized.includes("unsupported use of 'maskimage'") ||
normalized.includes("unsupported use of \"maskimage\"") ||
normalized.includes("unsupported use of maskimage") ||
normalized.includes("maskimage is not supported")
);
}
function shouldRetryWithoutNegativePrompt(errorMessage: string) {
const normalized = errorMessage.toLowerCase();
return (
normalized.includes("unsupportedarchitecturenegativeprompt") ||
normalized.includes("negative prompt is not supported") ||
normalized.includes("unsupported use of 'negativeprompt'") ||
normalized.includes("unsupported use of \"negativeprompt\"")
);
}
function shouldRetryWithoutControlNet(errorMessage: string) {
const normalized = errorMessage.toLowerCase();
return (
normalized.includes("unsupportedcontrolnetmodel") ||
normalized.includes("unsupported use of 'controlnet'") ||
normalized.includes("unsupported use of \"controlnet\"") ||
normalized.includes("controlnet is not supported")
);
}
function resolveRenderModeSettings(
mode: "strict-structure" | "balanced" | "stylize"
) {
if (mode === "strict-structure") {
return {
controlWeight: 1,
seedStrengthMin: 0.14,
seedStrengthMax: 0.58,
};
}
if (mode === "stylize") {
return {
controlWeight: 0.72,
seedStrengthMin: 0.28,
seedStrengthMax: 0.8,
};
}
return {
controlWeight: 0.86,
seedStrengthMin: 0.2,
seedStrengthMax: 0.7,
};
}
function extractFalImageUrl(payload: any) {
const direct =
payload?.images?.[0]?.url ||
payload?.image?.url ||
payload?.output?.[0]?.url ||
payload?.data?.images?.[0]?.url;
if (typeof direct === "string" && direct.trim()) {
return direct.trim();
}
const queue: any[] = [payload];
const visited = new Set<any>();
while (queue.length > 0) {
const current = queue.shift();
if (!current || typeof current !== "object" || visited.has(current)) continue;
visited.add(current);
for (const value of Object.values(current)) {
if (typeof value === "string" && /^https?:\/\//i.test(value)) {
return value;
}
if (value && typeof value === "object") {
queue.push(value);
}
}
}
return "";
}
function extractRunwareOutputImage(payload: any, taskUUID: string) {
const resultList = Array.isArray(payload)
? payload
: Array.isArray(payload?.data)
? payload.data
: [];
const imageResult =
resultList.find(
(item: any) =>
item?.taskUUID === taskUUID && (item?.imageDataURI || item?.imageURL)
) ||
resultList.find((item: any) => item?.imageDataURI || item?.imageURL);
const imageDataURI = String(imageResult?.imageDataURI || "").trim();
const imageURL = String(imageResult?.imageURL || "").trim();
const outputImage = imageDataURI || imageURL;
return {
imageResult,
imageDataURI,
imageURL,
outputImage,
};
}
function extractControlGuideImage(payload: any, taskUUID: string) {
const resultList = Array.isArray(payload)
? payload
: Array.isArray(payload?.data)
? payload.data
: [];
const guideResult =
resultList.find(
(item: any) =>
item?.taskUUID === taskUUID && (item?.guideImageURL || item?.guideImageDataURI)
) ||
resultList.find((item: any) => item?.guideImageURL || item?.guideImageDataURI);
const guideImageDataURI = String(guideResult?.guideImageDataURI || "").trim();
const guideImageURL = String(guideResult?.guideImageURL || "").trim();
return guideImageDataURI || guideImageURL;
}
export async function POST(request: Request) {
const session = await auth();
if (!session?.user?.id) {
@ -91,9 +246,14 @@ export async function POST(request: Request) {
if (!imageUrl) {
return Response.json({ code: "BAD_REQUEST", message: "缺少 imageUrl" }, { status: 400 });
}
const maskImageUrl = String(body.maskImageUrl || "").trim();
const setting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null);
const [setting, angleEditSetting] = await Promise.all([
getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null),
getSystemSetting({ key: "angle_edit_config" }).catch(() => null),
]);
const config = (setting?.value || {}) as RealtimeCanvasConfig;
const angleEditConfig = (angleEditSetting?.value || {}) as AngleEditConfig;
if (config.enabled === false) {
return Response.json(
@ -110,6 +270,13 @@ export async function POST(request: Request) {
process.env.RUNWARE_KEY ||
""
).trim();
const falApiKey = String(
config.falApiKey ||
angleEditConfig.falApiKey ||
process.env.FAL_KEY ||
process.env.FAL_API_KEY ||
""
).trim();
if (!runwareApiKey) {
return Response.json(
{
@ -121,11 +288,43 @@ export async function POST(request: Request) {
}
const fallbackSeedImageModel = "runware:101@1";
const model = String(
const configuredModel = String(
config.runwareModel || process.env.RUNWARE_MODEL || fallbackSeedImageModel
).trim();
const model = configuredModel || fallbackSeedImageModel;
const configuredStylizeModel = String(
config.runwareStylizeModel ||
process.env.RUNWARE_REALTIME_STYLIZE_MODEL ||
FALLBACK_RUNWARE_STYLIZE_MODEL
).trim();
const stylizeModel = configuredStylizeModel || FALLBACK_RUNWARE_STYLIZE_MODEL;
const configuredControlModel = String(
config.runwareControlModel ||
process.env.RUNWARE_REALTIME_CONTROL_MODEL ||
FALLBACK_RUNWARE_CONTROL_MODEL
).trim();
const controlModel = configuredControlModel || FALLBACK_RUNWARE_CONTROL_MODEL;
const controlPreprocessor =
config.runwareControlPreprocessor === "lineart" ||
config.runwareControlPreprocessor === "depth"
? config.runwareControlPreprocessor
: "canny";
const renderMode =
config.renderMode === "strict-structure" ||
config.renderMode === "stylize" ||
config.renderMode === "balanced"
? config.renderMode
: "balanced";
const renderModeSettings = resolveRenderModeSettings(renderMode);
const falRealtimeModel = String(
config.falRealtimeModel || process.env.FAL_REALTIME_MODEL || FALLBACK_FAL_REALTIME_MODEL
).trim();
const preferWebSocket =
config.runwareUseWebSocket !== false &&
process.env.RUNWARE_REALTIME_USE_WS !== "0";
const maxInputResolution = clamp(Number(config.maxInputResolution || 1024), 512, 1536);
const size = resolveSquareSize(body.imageSize, maxInputResolution);
const qualityProfile = body.qualityProfile === "preview" ? "preview" : "refine";
const steps = clamp(
Number(body.steps || config.defaults?.steps || 8),
1,
@ -144,64 +343,447 @@ export async function POST(request: Request) {
const strictSketch = Boolean(body.strictSketch ?? config.strictSketch ?? true);
const seed = resolveSeed(body.seed);
const positivePrompt =
String(body.prompt || "").trim() ||
"严格按照输入草图生成:保持主体轮廓、位置、比例和数量一致,只做材质与光影细化;禁止改变构图,禁止新增无关背景纹理。";
const rawPrompt = String(body.prompt || "").trim();
const hasUserPrompt =
rawPrompt.length > 0 && rawPrompt !== FALLBACK_REALTIME_PROMPT;
const positivePrompt = hasUserPrompt
? [
`User prompt: ${rawPrompt}`,
"Keep the object layout and perspective from the sketch.",
"Keep the exact silhouette and part topology from the sketch (same count of legs/supports and same frame structure).",
"The main object category must exactly match the user prompt.",
"Do not swap object class (for example, chair must remain chair, never animal).",
"Do not add bulky cushions, thick armrests, or boxy masses unless they clearly exist in the sketch.",
"Render as a full-color photorealistic product image with realistic material, lighting, and texture.",
"No sketch style, no line art outlines, no monochrome drawing.",
].join(" ")
: FALLBACK_REALTIME_PROMPT;
const negativePrompt =
String(config.negativePrompt || "").trim() ||
"改变构图, 改变主体位置, 多余人物, 多余肢体, 重复物体, 镜像翻转, 错误透视, 模糊轮廓, 文字水印, 大面积背景噪点";
"改变构图, 改变主体位置, 多余人物, 多余肢体, 重复物体, 镜像翻转, 错误透视, 模糊轮廓, 文字水印, 大面积背景噪点, 线稿, 线描, 草图, 黑白素描, 轮廓线, line art, sketch, outline drawing, monochrome drawing, wrong object category, unrelated subject, animal when furniture requested, furniture when animal requested";
let fallbackWarning = "";
const requestInference = async (modelId: string) => {
const useFalRealtime =
config.falRealtimeEnabled === true || process.env.FAL_REALTIME_ENABLED === "1";
if (hasUserPrompt && qualityProfile === "refine" && falApiKey && useFalRealtime) {
try {
const fal = await import("@fal-ai/serverless-client");
fal.config({
credentials: falApiKey,
});
const falInput: Record<string, unknown> = {
prompt: rawPrompt,
negative_prompt: negativePrompt,
image_url: imageUrl,
image_size: body.imageSize === "square_hd" ? "square_hd" : "square",
num_inference_steps: 6,
schedule_mu: 2.3,
output_feedback_strength: 0.95,
sync_mode: true,
};
const falResult: any = await fal.subscribe(falRealtimeModel, {
input: falInput,
logs: false,
timeout: 90_000,
});
const falImageUrl = extractFalImageUrl(falResult);
if (falImageUrl) {
console.log("[RealtimeFal] frame success", {
userId: session.user.id,
requestId: body.requestId || null,
model: falRealtimeModel,
promptLength: rawPrompt.length,
qualityProfile,
});
return Response.json({
ok: true,
provider: "fal",
model: falRealtimeModel,
transport: "fal-http",
imageUrl: falImageUrl,
cost: 0,
});
}
console.warn("[RealtimeFal] empty output, fallback to Runware", {
userId: session.user.id,
requestId: body.requestId || null,
model: falRealtimeModel,
});
} catch (error) {
fallbackWarning =
error instanceof Error && /unauthorized|forbidden/i.test(error.message)
? "FAL 实时模型鉴权失败,当前已回退为 Runware 草图模式;请在系统设置更新有效的 FAL API Key。"
: "FAL 实时模型调用失败,当前已回退为 Runware 草图模式。";
console.warn("[RealtimeFal] request failed, fallback to Runware", {
userId: session.user.id,
requestId: body.requestId || null,
model: falRealtimeModel,
error: error instanceof Error ? error.message : String(error),
});
}
}
type InferenceRequestResult = {
taskUUID: string;
modelId: string;
runtimeStrength: number;
runtimeCfgScale: number;
runtimeSteps: number;
effectiveStrictSketch: boolean;
runtimeSeed?: number;
usedMaskImage: boolean;
controlGuideImage: string;
controlNetModel: string;
path: "seed-image" | "controlnet";
transport: "ws" | "http";
upstreamStatus: number;
upstreamOk: boolean;
upstreamJson: any;
upstreamError: string;
wsFallbackError: string;
};
type RunwareTaskPayload = {
taskType: string;
taskUUID?: string;
[key: string]: unknown;
};
const sendRunwareTask = async (runwareTask: RunwareTaskPayload, timeoutMs = 90_000) => {
let transport: "ws" | "http" = "http";
let upstreamStatus = 0;
let upstreamOk = false;
let upstreamJson: any = null;
let upstreamError = "";
let wsFallbackError = "";
if (preferWebSocket) {
try {
const wsClient = getRunwareWebSocketClient(runwareApiKey);
upstreamJson = await wsClient.sendTask(runwareTask, timeoutMs);
upstreamError = parseRunwareError(upstreamJson);
upstreamStatus = upstreamError ? 502 : 200;
upstreamOk = !upstreamError;
transport = "ws";
} catch (error) {
wsFallbackError = error instanceof Error ? error.message : "Runware WebSocket 调用失败";
}
}
if (!upstreamJson) {
const upstreamResponse = await fetch("https://api.runware.ai/v1", {
method: "POST",
headers: {
Authorization: `Bearer ${runwareApiKey}`,
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify([runwareTask]),
cache: "no-store",
});
upstreamJson = await upstreamResponse.json().catch(() => null);
upstreamError = parseRunwareError(upstreamJson);
upstreamStatus = upstreamResponse.status;
upstreamOk = upstreamResponse.ok && !upstreamError;
transport = "http";
}
return {
transport,
upstreamStatus,
upstreamOk,
upstreamJson,
upstreamError,
wsFallbackError,
};
};
const requestInference = async (
modelId: string,
useMaskImage: boolean,
sourceImageUrl: string
): Promise<InferenceRequestResult> => {
const taskUUID = crypto.randomUUID();
const runtimeStrength = strictSketch ? clamp(strength, 0.12, 0.55) : strength;
const runtimeCfgScale = strictSketch ? clamp(cfgScale, 6, 14) : cfgScale;
const runwareRequest = [
{
taskType: "imageInference",
taskUUID,
model: modelId,
positivePrompt,
negativePrompt,
seedImage: imageUrl,
width: size,
height: size,
strength: runtimeStrength,
CFGScale: runtimeCfgScale,
steps,
includeCost: true,
numberResults: 1,
outputType: "dataURI",
outputFormat: "JPG",
...(typeof seed === "number" ? { seed } : {}),
},
];
const usePromptStylize = hasUserPrompt;
const previewMode = qualityProfile === "preview";
const isStylizePass =
usePromptStylize &&
qualityProfile === "refine" &&
Boolean(stylizeModel) &&
modelId === stylizeModel;
const relaxStructureForPrompt = usePromptStylize && !isStylizePass;
const effectiveStrictSketch = strictSketch && !relaxStructureForPrompt;
const profileStrength = usePromptStylize
? previewMode
? clamp(Math.max(strength, 0.58), 0.05, 1)
: isStylizePass
? clamp(Math.max(strength, 0.72), 0.05, 1)
: clamp(Math.max(strength, 0.9), 0.05, 1)
: previewMode
? clamp(strength * 0.85, 0.05, 1)
: strength;
const profileCfgScale = usePromptStylize
? previewMode
? clamp(Math.max(5.5, Math.min(cfgScale, 8.5)), 1, 14)
: isStylizePass
? clamp(Math.max(6.5, Math.min(cfgScale, 8.5)), 1, 14)
: clamp(Math.max(6, Math.min(cfgScale, 8.5)), 1, 14)
: previewMode
? clamp(cfgScale * 0.9, 1, 14)
: cfgScale;
const profileSteps = usePromptStylize
? previewMode
? clamp(Math.max(steps, 6), 1, 12)
: isStylizePass
? clamp(Math.max(steps, 8), 1, 10)
: clamp(Math.max(steps, 12), 1, 12)
: previewMode
? clamp(Math.min(steps, 6), 1, 12)
: clamp(Math.max(steps, 6), 1, 12);
const runtimeStrength = effectiveStrictSketch
? (() => {
const minStrength = usePromptStylize && !previewMode ? 0.2 : 0.12;
const maxStrength =
isStylizePass ? 0.62 : usePromptStylize && !previewMode ? 0.72 : 0.55;
return clamp(profileStrength, minStrength, maxStrength);
})()
: profileStrength;
const runtimeCfgScale = effectiveStrictSketch
? clamp(
profileCfgScale,
isStylizePass ? 6.5 : usePromptStylize && !previewMode ? 5 : 6,
14
)
: profileCfgScale;
const runtimeSeed = relaxStructureForPrompt ? undefined : seed;
const isRunware106 = modelId.toLowerCase().startsWith("runware:106@");
const inferenceSize = isRunware106 ? 1024 : size;
const runtimeStrengthLimited = clamp(
runtimeStrength,
renderModeSettings.seedStrengthMin,
renderModeSettings.seedStrengthMax
);
const runwareTask = {
taskType: "imageInference",
taskUUID,
model: modelId,
positivePrompt,
negativePrompt,
seedImage: sourceImageUrl,
width: inferenceSize,
height: inferenceSize,
strength: runtimeStrengthLimited,
CFGScale: runtimeCfgScale,
steps: profileSteps,
includeCost: true,
numberResults: 1,
outputType: "dataURI",
outputFormat: "JPG",
...(useMaskImage && maskImageUrl ? { maskImage: maskImageUrl } : {}),
...(typeof runtimeSeed === "number" ? { seed: runtimeSeed } : {}),
};
const upstreamResponse = await fetch("https://api.runware.ai/v1", {
method: "POST",
headers: {
Authorization: `Bearer ${runwareApiKey}`,
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify(runwareRequest),
cache: "no-store",
});
const upstreamJson = await upstreamResponse.json().catch(() => null);
const upstreamError = parseRunwareError(upstreamJson);
const upstream = await sendRunwareTask(runwareTask);
return {
taskUUID,
modelId,
runtimeStrength,
runtimeStrength: runtimeStrengthLimited,
runtimeCfgScale,
upstreamResponse,
upstreamJson,
upstreamError,
runtimeSteps: profileSteps,
effectiveStrictSketch,
runtimeSeed,
usedMaskImage: useMaskImage && Boolean(maskImageUrl),
controlGuideImage: "",
controlNetModel: "",
path: "seed-image" as const,
...upstream,
};
};
let requestResult = await requestInference(model);
const requestWithMaskFallback = async (
modelId: string,
useMaskImage: boolean,
sourceImageUrl: string
) => {
let requestResult = await requestInference(modelId, useMaskImage, sourceImageUrl);
let maskEnabled = useMaskImage;
if (
maskEnabled &&
shouldRetryWithoutMask(requestResult.upstreamError || "")
) {
requestResult = await requestInference(modelId, false, sourceImageUrl);
maskEnabled = false;
}
return { requestResult, maskEnabled };
};
const requestControlNetGuide = async (sourceImageUrl: string) => {
const taskUUID = crypto.randomUUID();
const controlTask = {
taskType: "imageControlNetPreProcess",
taskUUID,
inputImage: sourceImageUrl,
preProcessorType: controlPreprocessor,
};
const upstream = await sendRunwareTask(controlTask, 45_000);
return {
taskUUID,
guideImage: extractControlGuideImage(upstream.upstreamJson, taskUUID),
...upstream,
};
};
const requestControlNetInference = async (
modelId: string,
guideImage: string,
includeNegativePrompt: boolean
): Promise<InferenceRequestResult> => {
const taskUUID = crypto.randomUUID();
const previewMode = qualityProfile === "preview";
const profileSteps = previewMode
? clamp(Math.max(steps, 6), 1, 12)
: clamp(Math.max(steps, 10), 1, 12);
const profileCfgScale = previewMode
? clamp(Math.max(5, Math.min(cfgScale, 8)), 1, 14)
: clamp(Math.max(6, Math.min(cfgScale, 9)), 1, 14);
const controlWeight = previewMode
? clamp(renderModeSettings.controlWeight - 0.06, 0.55, 1)
: renderModeSettings.controlWeight;
const runwareTask: RunwareTaskPayload = {
taskType: "imageInference",
taskUUID,
model: modelId,
positivePrompt,
width: size,
height: size,
CFGScale: profileCfgScale,
steps: profileSteps,
includeCost: true,
numberResults: 1,
outputType: "dataURI",
outputFormat: "JPG",
...(typeof seed === "number" ? { seed } : {}),
...(includeNegativePrompt ? { negativePrompt } : {}),
controlNet: [
{
model: controlModel,
guideImage,
weight: controlWeight,
startStep: 0,
endStep: profileSteps,
controlMode: "balanced",
},
],
};
const upstream = await sendRunwareTask(runwareTask, 120_000);
return {
taskUUID,
modelId,
runtimeStrength: 0,
runtimeCfgScale: profileCfgScale,
runtimeSteps: profileSteps,
effectiveStrictSketch: strictSketch,
runtimeSeed: typeof seed === "number" ? seed : undefined,
usedMaskImage: false,
controlGuideImage: guideImage,
controlNetModel: controlModel,
path: "controlnet" as const,
...upstream,
};
};
const runTwoPassStylize = false;
let maskEnabled = Boolean(maskImageUrl);
let requestResult: InferenceRequestResult;
let pathLabel: "seed-image" | "controlnet" = "seed-image";
let controlGuideUsed = "";
let controlPreprocessUsed = false;
let controlGuideFallback = false;
const shouldUseControlNetPath =
hasUserPrompt && qualityProfile === "refine" && strictSketch;
if (shouldUseControlNetPath) {
const guideResult = await requestControlNetGuide(imageUrl);
const preprocessOk = guideResult.upstreamOk && !guideResult.upstreamError;
const guideImage = preprocessOk && guideResult.guideImage ? guideResult.guideImage : imageUrl;
controlGuideUsed = guideImage;
controlPreprocessUsed = preprocessOk;
controlGuideFallback = !preprocessOk || !guideResult.guideImage;
if (!preprocessOk && (guideResult.upstreamError || "").trim()) {
fallbackWarning = "ControlNet 预处理失败,已回退使用原始线稿引导。";
}
let controlResult = await requestControlNetInference(model, guideImage, true);
if (shouldRetryWithoutNegativePrompt(controlResult.upstreamError || "")) {
controlResult = await requestControlNetInference(model, guideImage, false);
}
if (controlResult.upstreamOk && !controlResult.upstreamError) {
requestResult = controlResult;
maskEnabled = false;
pathLabel = "controlnet";
} else if (shouldRetryWithoutControlNet(controlResult.upstreamError || "")) {
fallbackWarning = "当前模型不支持 ControlNet,已回退到 seedImage 模式。";
const fallbackPass = await requestWithMaskFallback(stylizeModel || model, maskEnabled, imageUrl);
requestResult = fallbackPass.requestResult;
maskEnabled = fallbackPass.maskEnabled;
pathLabel = "seed-image";
} else {
requestResult = controlResult;
maskEnabled = false;
pathLabel = "controlnet";
}
} else if (runTwoPassStylize) {
const basePass = await requestWithMaskFallback(model, maskEnabled, imageUrl);
requestResult = basePass.requestResult;
maskEnabled = basePass.maskEnabled;
if (requestResult.upstreamOk && !requestResult.upstreamError) {
const baseImage = extractRunwareOutputImage(
requestResult.upstreamJson,
requestResult.taskUUID
).outputImage;
if (baseImage) {
console.log("[RealtimeRunware] two-pass base success", {
userId: session.user.id,
requestId: body.requestId || null,
baseModel: model,
stylizeModel,
baseImageType: baseImage.startsWith("data:") ? "dataURI" : "url",
});
const stylizePass = await requestWithMaskFallback(stylizeModel, false, baseImage);
requestResult = stylizePass.requestResult;
maskEnabled = stylizePass.maskEnabled;
}
}
} else {
const initialModel =
hasUserPrompt && qualityProfile === "refine" ? stylizeModel || model : model;
const initialPass = await requestWithMaskFallback(initialModel, maskEnabled, imageUrl);
requestResult = initialPass.requestResult;
maskEnabled = initialPass.maskEnabled;
pathLabel = "seed-image";
}
if (pathLabel === "controlnet" && (!requestResult.upstreamOk || requestResult.upstreamError)) {
fallbackWarning = "ControlNet 推理失败,已自动回退为 seedImage 模式。";
const fallbackPass = await requestWithMaskFallback(stylizeModel || model, maskEnabled, imageUrl);
requestResult = fallbackPass.requestResult;
maskEnabled = fallbackPass.maskEnabled;
pathLabel = "seed-image";
}
const shouldRetryWithSeedImageModel =
(requestResult.upstreamError || "")
.toLowerCase()
@ -209,36 +791,34 @@ export async function POST(request: Request) {
model !== fallbackSeedImageModel;
if (shouldRetryWithSeedImageModel) {
requestResult = await requestInference(fallbackSeedImageModel);
fallbackWarning =
"当前模型不支持草图引导,已自动切换到兼容的实时草图模型。";
const fallbackResult = await requestWithMaskFallback(
fallbackSeedImageModel,
maskEnabled,
imageUrl
);
requestResult = fallbackResult.requestResult;
maskEnabled = fallbackResult.maskEnabled;
}
if (!requestResult.upstreamResponse.ok || requestResult.upstreamError) {
if (!requestResult.upstreamOk || requestResult.upstreamError) {
return Response.json(
{
code: "RUNWARE_REQUEST_FAILED",
message:
requestResult.upstreamError ||
`Runware 请求失败(${requestResult.upstreamResponse.status})`,
requestResult.wsFallbackError ||
`Runware 请求失败(${requestResult.upstreamStatus})`,
},
{ status: 502 }
);
}
const resultList = Array.isArray(requestResult.upstreamJson)
? requestResult.upstreamJson
: Array.isArray(requestResult.upstreamJson?.data)
? requestResult.upstreamJson.data
: [];
const imageResult =
resultList.find(
(item: any) =>
item?.taskUUID === requestResult.taskUUID && (item?.imageDataURI || item?.imageURL)
) ||
resultList.find((item: any) => item?.imageDataURI || item?.imageURL);
const imageDataURI = String(imageResult?.imageDataURI || "").trim();
const imageURL = String(imageResult?.imageURL || "").trim();
const outputImage = imageDataURI || imageURL;
const { imageResult, imageDataURI, imageURL, outputImage } = extractRunwareOutputImage(
requestResult.upstreamJson,
requestResult.taskUUID
);
if (!outputImage) {
return Response.json(
@ -253,20 +833,44 @@ export async function POST(request: Request) {
requestId: body.requestId || null,
taskUUID: requestResult.taskUUID,
model: requestResult.modelId,
seed: typeof seed === "number" ? seed : null,
steps,
transport: requestResult.transport,
seed: typeof requestResult.runtimeSeed === "number" ? requestResult.runtimeSeed : null,
hasUserPrompt,
promptLength: rawPrompt.length,
promptPreview: hasUserPrompt ? rawPrompt.slice(0, 32) : null,
qualityProfile,
steps: requestResult.runtimeSteps,
cfgScale: requestResult.runtimeCfgScale,
strength: requestResult.runtimeStrength,
strictSketchApplied: requestResult.effectiveStrictSketch,
maskEnabled,
roi: body.roi || null,
cost,
renderMode,
path: pathLabel,
controlNetModel: pathLabel === "controlnet" ? requestResult.controlNetModel : null,
controlGuideType:
pathLabel === "controlnet" && controlGuideUsed
? controlGuideUsed.startsWith("data:")
? "dataURI"
: "url"
: null,
controlGuideFallback,
controlPreprocessUsed,
twoPassStylize: runTwoPassStylize,
fallbackWarning: fallbackWarning || null,
});
return Response.json({
ok: true,
provider: "runware",
model: requestResult.modelId,
transport: requestResult.transport,
taskUUID: requestResult.taskUUID,
path: pathLabel,
imageDataURI: imageDataURI || undefined,
imageUrl: imageURL || undefined,
cost,
warning: fallbackWarning || undefined,
});
}

View File

@ -89,6 +89,7 @@ async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/pn
}
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
const AOS_MAX_RESOLUTION = 4096;
const AOS_CREATE_TIMEOUT_MS = 120_000;
const AOS_POLL_TIMEOUT_MS = 60_000;
const AOS_DOWNLOAD_TIMEOUT_MS = 60_000;
@ -106,6 +107,71 @@ const withTimeout = async <T,>(
}
};
function getAosFriendlyErrorMessage(detail: string): string | null {
const resolutionMatch = detail.match(/resolution excess,\s*width=(\d+),\s*height=(\d+)/i);
if (resolutionMatch) {
const [, width, height] = resolutionMatch;
return `图片分辨率超限(${width}x${height}),AOS要求宽高都不超过 ${AOS_MAX_RESOLUTION}px。请缩小后重试。`;
}
if (/invalid file/i.test(detail)) {
return "图片文件无法识别,请尝试转换为 JPG/PNG 后重试。";
}
return null;
}
async function prepareImageForAos(
imageBuffer: Buffer
): Promise<{ buffer: Buffer; mimeType: string; filename: string }> {
try {
const sharp = await import("sharp");
const image = sharp.default(imageBuffer);
const metadata = await image.metadata();
const width = metadata.width;
const height = metadata.height;
const format = (metadata.format || "png").toLowerCase();
const ext = format === "jpeg" ? "jpg" : format;
const filenameExt =
ext === "jpg" || ext === "png" || ext === "webp" || ext === "tiff" ? ext : "png";
const mimeType =
ext === "jpg" ? "image/jpeg" :
ext === "webp" ? "image/webp" :
ext === "tiff" ? "image/tiff" :
"image/png";
if (!width || !height || (width <= AOS_MAX_RESOLUTION && height <= AOS_MAX_RESOLUTION)) {
return {
buffer: imageBuffer,
mimeType,
filename: `image.${filenameExt}`,
};
}
const resizedBuffer = await image
.resize(AOS_MAX_RESOLUTION, AOS_MAX_RESOLUTION, {
fit: "inside",
withoutEnlargement: true,
})
.toBuffer();
const resizedMeta = await sharp.default(resizedBuffer).metadata();
console.warn(
`[AOS RMBG] 输入图片分辨率超限,已自动缩放: ${width}x${height} -> ${resizedMeta.width}x${resizedMeta.height}`
);
return {
buffer: resizedBuffer,
mimeType,
filename: `image.${filenameExt}`,
};
} catch (error) {
console.error("[AOS RMBG] 图片预处理失败,继续使用原图:", error);
return { buffer: imageBuffer, mimeType: "image/png", filename: "image.png" };
}
}
/**
* 使用 RMBG-2.0 API 进行抠图
*/
@ -254,9 +320,10 @@ async function processWithAosSegmentation(
const headers = { "X-API-KEY": config.apiKey };
try {
const normalizedImage = await prepareImageForAos(imageBuffer);
const createForm = new FormData();
const imageBlob = new Blob([new Uint8Array(imageBuffer)], { type: "image/png" });
createForm.append("image_file", imageBlob, "image.png");
const imageBlob = new Blob([new Uint8Array(normalizedImage.buffer)], { type: normalizedImage.mimeType });
createForm.append("image_file", imageBlob, normalizedImage.filename);
createForm.append("sync", "0");
createForm.append("return_type", "1");
createForm.append("output_type", "2");
@ -527,9 +594,11 @@ export async function POST(request: NextRequest) {
});
if (!result.success || !result.imageBuffer) {
const detail = result.error || "AOS处理失败";
const friendlyMessage = getAosFriendlyErrorMessage(detail);
return NextResponse.json(
{ error: "抠图处理失败", detail: result.error },
{ status: 500 }
{ error: "抠图处理失败", detail: friendlyMessage || detail },
{ status: friendlyMessage ? 400 : 500 }
);
}

File diff suppressed because it is too large Load Diff

View File

@ -60,7 +60,6 @@ import { CanvasWorkspace } from "@/components/canvas";
import { CanvasChatPanel } from "@/components/canvas/CanvasChatPanel";
import { CanvasFileListPanel } from "@/components/canvas/CanvasFileListPanel";
import { ObjectPickerOverlay } from "@/components/canvas/ObjectPickerOverlay";
import { RealtimeCanvasDialog } from "@/components/canvas/RealtimeCanvasDialog";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { cn, fetcher } from "@/lib/utils";
import { MessageSquare, FolderOpen, Grid2X2 } from "lucide-react";
@ -124,7 +123,6 @@ export default function CanvasEditorPage({ params }: PageProps) {
const [isPanelOpen, setIsPanelOpen] = useState(true);
// 面板模式: 'chat' 聊天面板, 'fileList' 文件列表
const [panelMode, setPanelMode] = useState<'chat' | 'fileList'>('chat');
const [isRealtimeCanvasOpen, setIsRealtimeCanvasOpen] = useState(false);
const { data: siteSettingsData } = useSWR<{
siteName?: string;
@ -359,38 +357,6 @@ export default function CanvasEditorPage({ params }: PageProps) {
});
};
const handleApplyRealtimeResult = useCallback(
({ imageUrl, prompt: imagePrompt }: { imageUrl: string; prompt: string }) => {
const viewportX = viewport.x || 0;
const viewportY = viewport.y || 0;
const viewportZoom = viewport.zoom || 1;
const stageWidth = typeof window !== "undefined" ? window.innerWidth : 1280;
const stageHeight = typeof window !== "undefined" ? window.innerHeight : 720;
const targetWidth = 360;
const targetHeight = 360;
const x = (stageWidth * 0.45 - viewportX) / viewportZoom - targetWidth / 2;
const y = (stageHeight * 0.45 - viewportY) / viewportZoom - targetHeight / 2;
addNode({
type: "image",
position: { x, y },
size: { width: targetWidth, height: targetHeight },
data: {
src: imageUrl,
status: "completed",
title: "实时画布结果",
prompt: imagePrompt,
model: "fal-ai/flux-2/klein/realtime",
},
connections: [],
});
toast.success("实时画布结果已添加");
},
[addNode, viewport.x, viewport.y, viewport.zoom]
);
// 处理图片操作(通过事件分发到选中图片节点执行)
const handleAction = (action: string) => {
if (!selectedImageNode) {
@ -762,7 +728,7 @@ export default function CanvasEditorPage({ params }: PageProps) {
<Button
variant="ghost"
size="icon"
onClick={() => setIsRealtimeCanvasOpen(true)}
onClick={() => router.push(`/canvas/${id}/realtime`)}
className="rounded-xl h-9 w-9"
>
<PenTool className="h-4 w-4" />
@ -825,13 +791,6 @@ export default function CanvasEditorPage({ params }: PageProps) {
{/* 6. 物品拾取模式覆盖层 */}
<ObjectPickerOverlay />
<RealtimeCanvasDialog
open={isRealtimeCanvasOpen}
onOpenChange={setIsRealtimeCanvasOpen}
initialImageUrl={selectedImageNode ? ((selectedImageNode.data as any)?.src as string) : null}
onApplyResult={handleApplyRealtimeResult}
/>
</div>
</TooltipProvider>
);

View File

@ -0,0 +1,82 @@
"use client";
import { useCallback } from "react";
import { useParams, useRouter } from "next/navigation";
import { toast } from "sonner";
import { RealtimeCanvasDialog } from "@/components/canvas/RealtimeCanvasDialog";
import { canvasService } from "@/services/canvas";
import { useCanvasStore, canvasSelectors } from "@/store/canvas";
export default function CanvasRealtimePage() {
const router = useRouter();
const routeParams = useParams();
const id = String(routeParams?.id || "");
const viewport = useCanvasStore(canvasSelectors.viewport);
const selectedNodeIds = useCanvasStore(canvasSelectors.selectedNodeIds);
const nodes = useCanvasStore(canvasSelectors.nodes);
const addNode = useCanvasStore((state) => state.addNode);
const getCanvasState = useCanvasStore((state) => state.getCanvasState);
const selectedImageNode =
selectedNodeIds.length === 1
? nodes.find((node) => node.id === selectedNodeIds[0] && node.type === "image")
: null;
const handleOpenChange = useCallback(
(nextOpen: boolean) => {
if (!nextOpen) {
router.push(`/canvas/${id}`);
}
},
[id, router]
);
const handleApplyResult = useCallback(
async ({ imageUrl, prompt }: { imageUrl: string; prompt: string }) => {
const viewportX = viewport.x || 0;
const viewportY = viewport.y || 0;
const viewportZoom = viewport.zoom || 1;
const stageWidth = typeof window !== "undefined" ? window.innerWidth : 1280;
const stageHeight = typeof window !== "undefined" ? window.innerHeight : 720;
const targetWidth = 360;
const targetHeight = 360;
const x = (stageWidth * 0.45 - viewportX) / viewportZoom - targetWidth / 2;
const y = (stageHeight * 0.45 - viewportY) / viewportZoom - targetHeight / 2;
addNode({
type: "image",
position: { x, y },
size: { width: targetWidth, height: targetHeight },
data: {
src: imageUrl,
status: "completed",
title: "实时画布结果",
prompt,
model: "fal-ai/flux-2/klein/realtime",
},
connections: [],
});
try {
await canvasService.saveCanvasState(id, getCanvasState());
} catch (error) {
console.error("[RealtimeCanvasPage] 保存结果失败:", error);
}
toast.success("实时画布结果已添加");
},
[addNode, getCanvasState, id, viewport.x, viewport.y, viewport.zoom]
);
return (
<RealtimeCanvasDialog
standalone
open
onOpenChange={handleOpenChange}
initialImageUrl={selectedImageNode ? ((selectedImageNode.data as any)?.src as string) : null}
onApplyResult={handleApplyResult}
/>
);
}

View File

@ -35,6 +35,7 @@ import {
Undo2,
Redo2,
Send,
X,
} from "lucide-react";
import { toast } from "sonner";
@ -68,6 +69,7 @@ type RealtimeCanvasDialogProps = {
onOpenChange: (open: boolean) => void;
initialImageUrl?: string | null;
onApplyResult: (payload: { imageUrl: string; prompt: string }) => void;
standalone?: boolean;
};
const FALLBACK_PROMPT =
@ -122,7 +124,7 @@ async function canvasToJpegDataUri(
options?: {
autoCrop?: boolean;
}
) {
): Promise<{ dataUri: string; width: number; height: number }> {
type CropBox = {
x: number;
y: number;
@ -455,12 +457,114 @@ async function canvasToJpegDataUri(
throw new Error("导出图片失败");
}
return await new Promise<string>((resolve, reject) => {
const dataUri = await new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(String(reader.result || ""));
reader.onerror = () => reject(new Error("读取导出图片失败"));
reader.readAsDataURL(blob);
});
return {
dataUri,
width: exportWidth,
height: exportHeight,
};
}
type FlushMode = "preview" | "refine";
type NormalizedRegion = {
x: number;
y: number;
width: number;
height: number;
};
function clamp01(value: number) {
return Math.min(1, Math.max(0, value));
}
function mergeRegion(
current: NormalizedRegion | null,
next: NormalizedRegion
): NormalizedRegion {
if (!current) return next;
const x1 = Math.min(current.x, next.x);
const y1 = Math.min(current.y, next.y);
const x2 = Math.max(current.x + current.width, next.x + next.width);
const y2 = Math.max(current.y + current.height, next.y + next.height);
return {
x: clamp01(x1),
y: clamp01(y1),
width: clamp01(x2 - x1),
height: clamp01(y2 - y1),
};
}
function toNormalizedRegion(
canvas: HTMLCanvasElement,
rect: {
minX: number;
minY: number;
maxX: number;
maxY: number;
}
): NormalizedRegion | null {
const bounds = canvas.getBoundingClientRect();
const width = Math.max(1, bounds.width);
const height = Math.max(1, bounds.height);
const x = clamp01(rect.minX / width);
const y = clamp01(rect.minY / height);
const right = clamp01(rect.maxX / width);
const bottom = clamp01(rect.maxY / height);
const regionWidth = right - x;
const regionHeight = bottom - y;
if (regionWidth <= 0 || regionHeight <= 0) return null;
return { x, y, width: regionWidth, height: regionHeight };
}
async function createMaskDataUri(
width: number,
height: number,
region: NormalizedRegion
) {
const canvas = document.createElement("canvas");
canvas.width = Math.max(1, width);
canvas.height = Math.max(1, height);
const ctx = canvas.getContext("2d");
if (!ctx) {
throw new Error("无法初始化遮罩画布");
}
const basePad = Math.max(12, Math.floor(Math.max(width, height) * 0.02));
let x = Math.floor(region.x * width) - basePad;
let y = Math.floor(region.y * height) - basePad;
let regionWidth = Math.ceil(region.width * width) + basePad * 2;
let regionHeight = Math.ceil(region.height * height) + basePad * 2;
const minPatch = Math.max(40, Math.floor(Math.max(width, height) * 0.1));
if (regionWidth < minPatch) {
const delta = Math.ceil((minPatch - regionWidth) / 2);
x -= delta;
regionWidth += delta * 2;
}
if (regionHeight < minPatch) {
const delta = Math.ceil((minPatch - regionHeight) / 2);
y -= delta;
regionHeight += delta * 2;
}
x = Math.max(0, x);
y = Math.max(0, y);
const right = Math.min(width, x + regionWidth);
const bottom = Math.min(height, y + regionHeight);
ctx.fillStyle = "#000000";
ctx.fillRect(0, 0, width, height);
ctx.fillStyle = "#ffffff";
ctx.fillRect(x, y, Math.max(1, right - x), Math.max(1, bottom - y));
return canvas.toDataURL("image/png");
}
export function RealtimeCanvasDialog({
@ -468,6 +572,7 @@ export function RealtimeCanvasDialog({
onOpenChange,
initialImageUrl,
onApplyResult,
standalone = false,
}: RealtimeCanvasDialogProps) {
const canvasRef = useRef<HTMLCanvasElement | null>(null);
const previewCanvasRef = useRef<HTMLCanvasElement | null>(null);
@ -479,8 +584,15 @@ export function RealtimeCanvasDialog({
const pendingFlushTimerRef = useRef<number | null>(null);
const flushInFlightRef = useRef(false);
const dirtyRef = useRef(false);
const dirtyRegionRef = useRef<NormalizedRegion | null>(null);
const draftVersionRef = useRef(0);
const pendingFlushModeRef = useRef<FlushMode | null>(null);
const pendingFlushForceRef = useRef(false);
const latencyEmaRef = useRef<number | null>(null);
const baseThrottleMsRef = useRef<number>(defaultRealtimeConfig.transport.throttleInterval);
const pumpFlushRef = useRef<(() => void) | null>(null);
const warningMessageRef = useRef("");
const seqRef = useRef(0);
const requestSeqMapRef = useRef<Map<string, number>>(new Map());
const requestStartMapRef = useRef<Map<string, number>>(new Map());
const lastRenderedSeqRef = useRef(0);
const metricsQueueRef = useRef<any[]>([]);
@ -574,8 +686,42 @@ export function RealtimeCanvasDialog({
setCanRedo(false);
}, []);
const markCanvasDirty = useCallback((region?: NormalizedRegion | null) => {
dirtyRef.current = true;
draftVersionRef.current += 1;
if (!region) {
dirtyRegionRef.current = { x: 0, y: 0, width: 1, height: 1 };
return;
}
dirtyRegionRef.current = mergeRegion(dirtyRegionRef.current, region);
}, []);
const markCanvasDirtyRect = useCallback(
(rect: { minX: number; minY: number; maxX: number; maxY: number }) => {
const canvas = canvasRef.current;
if (!canvas) return;
const region = toNormalizedRegion(canvas, rect);
if (!region) return;
markCanvasDirty(region);
},
[markCanvasDirty]
);
const markCanvasDirtyStroke = useCallback(
(from: { x: number; y: number }, to: { x: number; y: number }, lineWidth: number) => {
const pad = Math.max(6, Math.ceil(lineWidth * 1.2));
markCanvasDirtyRect({
minX: Math.min(from.x, to.x) - pad,
minY: Math.min(from.y, to.y) - pad,
maxX: Math.max(from.x, to.x) + pad,
maxY: Math.max(from.y, to.y) + pad,
});
},
[markCanvasDirtyRect]
);
const flushFrame = useCallback(
async (force = false) => {
async (mode: FlushMode, force = false) => {
if (!open) return;
if (!connectionRef.current) return;
if (flushInFlightRef.current) return;
@ -588,20 +734,37 @@ export function RealtimeCanvasDialog({
const seq = seqRef.current + 1;
seqRef.current = seq;
const requestId = `${sessionIdRef.current}-${seq}`;
requestSeqMapRef.current.set(requestId, seq);
requestStartMapRef.current.set(requestId, Date.now());
const draftVersionAtSend = draftVersionRef.current;
const dirtyRegionAtSend = dirtyRegionRef.current
? { ...dirtyRegionRef.current }
: null;
try {
const imageDataUri = await canvasToJpegDataUri(
const exportedImage = await canvasToJpegDataUri(
canvas,
config.image.maxInputResolution,
config.image.jpegQuality,
canvasBackgroundRef.current,
{ autoCrop: true }
{ autoCrop: false }
);
const imageDataUri = exportedImage.dataUri;
const shouldUseMask =
Boolean(dirtyRegionAtSend) &&
mode !== "refine" &&
(dirtyRegionAtSend?.width || 0) * (dirtyRegionAtSend?.height || 0) < 0.92;
const maskImageUrl =
shouldUseMask && dirtyRegionAtSend
? await createMaskDataUri(
exportedImage.width,
exportedImage.height,
dirtyRegionAtSend
)
: undefined;
const activeConnection = connectionRef.current;
if (!activeConnection) {
requestStartMapRef.current.delete(requestId);
setStatus("connecting");
setStatusMessage("连接重建中,正在重试...");
return;
@ -623,6 +786,9 @@ export function RealtimeCanvasDialog({
steps: config.model.defaults.steps,
cfgScale: config.model.defaults.cfgScale,
strength: config.model.defaults.strength,
qualityProfile: mode,
maskImageUrl,
roi: dirtyRegionAtSend || undefined,
}),
});
@ -630,6 +796,8 @@ export function RealtimeCanvasDialog({
imageUrl?: string;
imageDataURI?: string;
cost?: number;
transport?: string;
warning?: string;
message?: string;
error?: string;
};
@ -646,7 +814,6 @@ export function RealtimeCanvasDialog({
const startedAt = requestStartMapRef.current.get(requestId) || Date.now();
const latencyMs = Date.now() - startedAt;
requestStartMapRef.current.delete(requestId);
requestSeqMapRef.current.delete(requestId);
if (seq > 0 && seq < lastRenderedSeqRef.current) {
return;
@ -655,8 +822,29 @@ export function RealtimeCanvasDialog({
lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq);
setPreviewUrl(nextPreviewUrl);
setStatus("live");
setStatusMessage("实时渲染中");
dirtyRef.current = false;
setStatusMessage(
payload.transport === "ws" ? "实时渲染中(WS)" : "实时渲染中(HTTP)"
);
const upstreamWarning = String(payload.warning || "").trim();
if (upstreamWarning && upstreamWarning !== warningMessageRef.current) {
warningMessageRef.current = upstreamWarning;
toast.warning(upstreamWarning);
}
if (draftVersionAtSend === draftVersionRef.current) {
dirtyRef.current = false;
dirtyRegionRef.current = null;
}
const previousLatency = latencyEmaRef.current;
const nextLatency =
previousLatency === null ? latencyMs : previousLatency * 0.72 + latencyMs * 0.28;
latencyEmaRef.current = nextLatency;
const computedThrottle = Math.round(
Math.max(baseThrottleMsRef.current, Math.min(520, nextLatency * 0.45))
);
setAdaptiveThrottleMs((prev) =>
Math.abs(prev - computedThrottle) >= 8 ? computedThrottle : prev
);
pushMetric({
name: "frame_send",
@ -669,41 +857,81 @@ export function RealtimeCanvasDialog({
seq,
latencyMs,
queueDepth: Number(payload.cost || 0),
qualityProfile: mode,
hasMask: Boolean(maskImageUrl),
transport: payload.transport || "unknown",
});
} catch (error) {
requestStartMapRef.current.delete(requestId);
requestSeqMapRef.current.delete(requestId);
console.error("[RealtimeCanvas] flush frame failed:", error);
setStatus("error");
setStatusMessage(error instanceof Error ? error.message : "发送失败");
} finally {
flushInFlightRef.current = false;
if (pendingFlushModeRef.current) {
if (pendingFlushTimerRef.current) {
window.clearTimeout(pendingFlushTimerRef.current);
pendingFlushTimerRef.current = null;
}
const shouldDelay = pendingFlushModeRef.current === "preview";
if (shouldDelay) {
pendingFlushTimerRef.current = window.setTimeout(() => {
pendingFlushTimerRef.current = null;
pumpFlushRef.current?.();
}, adaptiveThrottleMs);
} else {
pumpFlushRef.current?.();
}
}
}
},
[config, open, prompt, pushMetric]
[adaptiveThrottleMs, config, open, prompt, pushMetric]
);
const pumpFlush = useCallback(() => {
if (flushInFlightRef.current) return;
const mode = pendingFlushModeRef.current;
const force = pendingFlushForceRef.current;
if (!mode) return;
pendingFlushModeRef.current = null;
pendingFlushForceRef.current = false;
void flushFrame(mode, force);
}, [flushFrame]);
const scheduleFlush = useCallback(
(force = false, markDirty = true) => {
(force = false, markDirty = true, mode?: FlushMode) => {
if (markDirty) {
dirtyRef.current = true;
markCanvasDirty(null);
}
if (!dirtyRef.current) return;
if (force) {
const nextMode: FlushMode = mode || (force ? "refine" : "preview");
if (pendingFlushModeRef.current === "refine" || nextMode === "refine") {
pendingFlushModeRef.current = "refine";
} else {
pendingFlushModeRef.current = "preview";
}
pendingFlushForceRef.current = pendingFlushForceRef.current || force;
if (!dirtyRef.current && !pendingFlushForceRef.current) {
pendingFlushModeRef.current = null;
pendingFlushForceRef.current = false;
return;
}
if (force || pendingFlushModeRef.current === "refine") {
if (pendingFlushTimerRef.current) {
window.clearTimeout(pendingFlushTimerRef.current);
pendingFlushTimerRef.current = null;
}
void flushFrame(true);
pumpFlush();
return;
}
if (pendingFlushTimerRef.current) return;
pendingFlushTimerRef.current = window.setTimeout(() => {
pendingFlushTimerRef.current = null;
void flushFrame();
pumpFlush();
}, adaptiveThrottleMs);
},
[adaptiveThrottleMs, flushFrame]
[adaptiveThrottleMs, markCanvasDirty, pumpFlush]
);
const scheduleFlushRef = useRef(scheduleFlush);
@ -713,6 +941,10 @@ export function RealtimeCanvasDialog({
scheduleFlushRef.current = scheduleFlush;
}, [scheduleFlush]);
useEffect(() => {
pumpFlushRef.current = pumpFlush;
}, [pumpFlush]);
useEffect(() => {
flushMetricsRef.current = flushMetrics;
}, [flushMetrics]);
@ -797,10 +1029,10 @@ export function RealtimeCanvasDialog({
ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight);
hasLoadedInitialImageRef.current = true;
dirtyRef.current = true;
markCanvasDirty(null);
saveHistory(); // Save initial image state
return true;
}, [initialImageUrl, open, saveHistory]);
}, [initialImageUrl, markCanvasDirty, open, saveHistory]);
const initializeCanvas = useCallback(() => {
const canvas = canvasRef.current;
@ -923,11 +1155,21 @@ export function RealtimeCanvasDialog({
clearPreviewCanvas();
} else {
drawStroke(point, point);
scheduleFlush();
const lineWidth = brushMode === "eraser" ? brushSize * 1.45 : brushSize;
markCanvasDirtyStroke(point, point, lineWidth);
scheduleFlush(false, false, "preview");
}
event.currentTarget.setPointerCapture(event.pointerId);
},
[brushMode, clearPreviewCanvas, drawStroke, getCanvasPoint, scheduleFlush]
[
brushMode,
brushSize,
clearPreviewCanvas,
drawStroke,
getCanvasPoint,
markCanvasDirtyStroke,
scheduleFlush,
]
);
const handlePointerLeave = useCallback(() => {
@ -940,7 +1182,7 @@ export function RealtimeCanvasDialog({
shapeStartPointRef.current = null;
clearPreviewCanvas();
if (wasDrawing) {
scheduleFlush(true, false);
scheduleFlush(true, false, "refine");
}
}, [clearPreviewCanvas, scheduleFlush]);
@ -978,11 +1220,21 @@ export function RealtimeCanvasDialog({
}
} else {
drawStroke(lastPointRef.current, point);
const lineWidth = brushMode === "eraser" ? brushSize * 1.45 : brushSize;
markCanvasDirtyStroke(lastPointRef.current, point, lineWidth);
}
lastPointRef.current = point;
scheduleFlush();
scheduleFlush(false, false, "preview");
},
[drawShape, drawStroke, getCanvasPoint, scheduleFlush, brushMode, brushSize]
[
drawShape,
drawStroke,
getCanvasPoint,
scheduleFlush,
brushMode,
brushSize,
markCanvasDirtyStroke,
]
);
const handlePointerUp = useCallback(
@ -993,6 +1245,14 @@ export function RealtimeCanvasDialog({
const shapeStart = shapeStartPointRef.current;
if (isShapeMode(brushMode) && endPoint && shapeStart) {
drawShape(shapeStart, endPoint);
const lineWidth = Math.max(1, brushSize * 0.9);
const pad = Math.max(6, Math.ceil(lineWidth * 1.2));
markCanvasDirtyRect({
minX: Math.min(shapeStart.x, endPoint.x) - pad,
minY: Math.min(shapeStart.y, endPoint.y) - pad,
maxX: Math.max(shapeStart.x, endPoint.x) + pad,
maxY: Math.max(shapeStart.y, endPoint.y) + pad,
});
clearPreviewCanvas();
}
lastPointRef.current = null;
@ -1003,9 +1263,18 @@ export function RealtimeCanvasDialog({
// ignore
}
saveHistory(); // Save state after stroke
scheduleFlush(true, false);
scheduleFlush(true, false, "refine");
},
[brushMode, clearPreviewCanvas, drawShape, getCanvasPoint, saveHistory, scheduleFlush]
[
brushMode,
brushSize,
clearPreviewCanvas,
drawShape,
getCanvasPoint,
markCanvasDirtyRect,
saveHistory,
scheduleFlush,
]
);
useEffect(() => {
@ -1043,8 +1312,14 @@ export function RealtimeCanvasDialog({
sessionIdRef.current = `rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
seqRef.current = 0;
lastRenderedSeqRef.current = 0;
requestSeqMapRef.current.clear();
requestStartMapRef.current.clear();
draftVersionRef.current = 0;
dirtyRef.current = false;
dirtyRegionRef.current = null;
pendingFlushModeRef.current = null;
pendingFlushForceRef.current = false;
latencyEmaRef.current = null;
warningMessageRef.current = "";
metricsQueueRef.current = [];
}, [open]);
@ -1132,6 +1407,7 @@ export function RealtimeCanvasDialog({
};
setConfig(resolvedConfig);
baseThrottleMsRef.current = resolvedConfig.transport.throttleInterval;
setAdaptiveThrottleMs(resolvedConfig.transport.throttleInterval);
if (cancelled) return;
@ -1169,6 +1445,8 @@ export function RealtimeCanvasDialog({
setStatusMessage("等待连接");
pointerDownRef.current = false;
lastPointRef.current = null;
pendingFlushModeRef.current = null;
pendingFlushForceRef.current = false;
};
}, [open, pushMetric, refreshToken]);
@ -1182,8 +1460,7 @@ export function RealtimeCanvasDialog({
const clearSketch = useCallback(() => {
resetCanvas();
clearPreviewCanvas();
dirtyRef.current = true;
scheduleFlush(true, false);
scheduleFlush(true, true, "refine");
}, [clearPreviewCanvas, resetCanvas, scheduleFlush]);
const handleSketchUpload = useCallback(
@ -1268,9 +1545,14 @@ export function RealtimeCanvasDialog({
const canvas = canvasRef.current;
if (!canvas) return;
try {
const dataUrl = await canvasToJpegDataUri(canvas, 2048, 0.92, canvasBackgroundRef.current);
const exportedImage = await canvasToJpegDataUri(
canvas,
2048,
0.92,
canvasBackgroundRef.current
);
const anchor = document.createElement("a");
anchor.href = dataUrl;
anchor.href = exportedImage.dataUri;
anchor.download = `realtime-canvas-${Date.now()}.jpg`;
anchor.click();
toast.success("已保存当前画布");
@ -1292,13 +1574,31 @@ export function RealtimeCanvasDialog({
onOpenChange(false);
}, [onApplyResult, onOpenChange, previewUrl, prompt]);
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="h-[95vh] w-[98vw] max-w-[1800px] overflow-hidden rounded-[20px] border-0 bg-background p-0 shadow-2xl sm:rounded-[20px] md:rounded-[24px]">
<DialogHeader className="sr-only">
<DialogTitle>实时画布</DialogTitle>
<DialogDescription>左侧绘制草图,右侧实时预览。</DialogDescription>
</DialogHeader>
const content = (
<>
{standalone ? (
<div className="sr-only">
<h2>实时画布</h2>
<p>左侧绘制草图,右侧实时预览。</p>
</div>
) : (
<DialogHeader className="sr-only">
<DialogTitle>实时画布</DialogTitle>
<DialogDescription>左侧绘制草图,右侧实时预览。</DialogDescription>
</DialogHeader>
)}
{standalone ? (
<Button
type="button"
variant="ghost"
size="icon"
className="absolute right-4 top-4 z-[60] h-9 w-9 rounded-full bg-background/85 shadow-sm"
onClick={() => onOpenChange(false)}
title="返回画布"
>
<X className="h-5 w-5" />
</Button>
) : null}
{/* Top Header - Minimal */}
<div className="absolute left-0 right-0 top-0 z-50 flex h-14 items-center justify-start px-6 bg-gradient-to-b from-black/20 to-transparent pointer-events-none">
<div className="pointer-events-auto flex items-center gap-3 rounded-full bg-background/95 backdrop-blur-sm px-4 py-1.5 border border-border/50 shadow-sm">
@ -1622,7 +1922,21 @@ export function RealtimeCanvasDialog({
</span>
</div>
</div>
</>
);
if (standalone) {
return (
<div className="h-dvh w-full overflow-hidden bg-background">
<div className="relative h-full w-full overflow-hidden bg-background">{content}</div>
</div>
);
}
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="h-[95vh] w-[98vw] max-w-[1800px] overflow-hidden rounded-[20px] border-0 bg-background p-0 shadow-2xl sm:rounded-[20px] md:rounded-[24px]">
{content}
</DialogContent>
</Dialog>
);

View File

@ -1,14 +1,50 @@
"use client";
import React, { Suspense, useMemo } from "react";
import React, { Suspense, useEffect, useMemo } from "react";
import { Canvas } from "@react-three/fiber";
import { Environment, OrbitControls, useGLTF } from "@react-three/drei";
import { Environment, Grid, OrbitControls, useGLTF } from "@react-three/drei";
import { AlertCircle, Box, ExternalLink, Loader2 } from "lucide-react";
import { Button } from "@/components/ui/button";
import type { Mesh, Object3D } from "three";
export interface Model3DLayer {
id: string;
name: string;
triangles: number;
vertices: number;
}
export interface Model3DStats {
triangles: number;
vertices: number;
meshCount: number;
}
export interface Model3DViewerSettings {
envPreset:
| "city"
| "sunset"
| "studio"
| "dawn"
| "night"
| "warehouse"
| "forest"
| "apartment"
| "park"
| "lobby";
lightIntensity: number;
autoRotate: boolean;
showGrid: boolean;
wireframe: boolean;
}
interface Model3DViewerProps {
modelUrl?: string | null;
previewImageUrl?: string | null;
settings: Model3DViewerSettings;
layerVisibility?: Record<string, boolean>;
onLayersChange?: (layers: Model3DLayer[]) => void;
onStatsChange?: (stats: Model3DStats) => void;
}
class ViewerErrorBoundary extends React.Component<
@ -38,9 +74,100 @@ class ViewerErrorBoundary extends React.Component<
}
}
function ModelScene({ url }: { url: string }) {
function isMesh(node: Object3D): node is Mesh {
return (node as Mesh).isMesh === true;
}
function collectSceneMeta(scene: Object3D): {
layers: Model3DLayer[];
stats: Model3DStats;
meshRefs: Array<{ layerId: string; mesh: Mesh }>;
} {
const layers: Model3DLayer[] = [];
const meshRefs: Array<{ layerId: string; mesh: Mesh }> = [];
let triangles = 0;
let vertices = 0;
let meshCount = 0;
let meshIndex = 0;
scene.traverse((node) => {
if (!isMesh(node)) return;
const geometry = node.geometry;
if (!geometry) return;
const id = `${node.name || "Mesh"}__${meshIndex}`;
meshIndex += 1;
const indexCount = geometry.index?.count || 0;
const positionCount = geometry.attributes.position?.count || 0;
const meshTriangles = indexCount > 0 ? Math.floor(indexCount / 3) : Math.floor(positionCount / 3);
const meshVertices = positionCount;
triangles += meshTriangles;
vertices += meshVertices;
meshCount += 1;
layers.push({
id,
name: node.name?.trim() || `mesh_${meshCount}`,
triangles: meshTriangles,
vertices: meshVertices,
});
meshRefs.push({ layerId: id, mesh: node });
});
return {
layers,
stats: {
triangles,
vertices,
meshCount,
},
meshRefs,
};
}
function ModelScene({
url,
settings,
layerVisibility,
onLayersChange,
onStatsChange,
}: {
url: string;
settings: Model3DViewerSettings;
layerVisibility?: Record<string, boolean>;
onLayersChange?: (layers: Model3DLayer[]) => void;
onStatsChange?: (stats: Model3DStats) => void;
}) {
const model = useGLTF(url);
const scene = useMemo(() => model.scene.clone(), [model.scene]);
const meta = useMemo(() => collectSceneMeta(scene), [scene]);
useEffect(() => {
onLayersChange?.(meta.layers);
onStatsChange?.(meta.stats);
}, [meta.layers, meta.stats, onLayersChange, onStatsChange]);
useEffect(() => {
for (const { layerId, mesh } of meta.meshRefs) {
const nextVisible = layerVisibility?.[layerId];
mesh.visible = nextVisible !== false;
}
}, [layerVisibility, meta.meshRefs]);
useEffect(() => {
for (const { mesh } of meta.meshRefs) {
const materials = Array.isArray(mesh.material) ? mesh.material : [mesh.material];
for (const material of materials) {
if (!material) continue;
if ("wireframe" in material) {
(material as any).wireframe = settings.wireframe;
material.needsUpdate = true;
}
}
}
}, [meta.meshRefs, settings.wireframe]);
return <primitive object={scene} scale={1} />;
}
@ -66,7 +193,14 @@ function EmptyState({ previewImageUrl }: { previewImageUrl?: string | null }) {
);
}
export function Model3DViewer({ modelUrl, previewImageUrl }: Model3DViewerProps) {
export function Model3DViewer({
modelUrl,
previewImageUrl,
settings,
layerVisibility,
onLayersChange,
onStatsChange,
}: Model3DViewerProps) {
if (!modelUrl) {
return <EmptyState previewImageUrl={previewImageUrl} />;
}
@ -104,8 +238,20 @@ export function Model3DViewer({ modelUrl, previewImageUrl }: Model3DViewerProps)
}
>
<Canvas camera={{ position: [2, 1.5, 2], fov: 45 }}>
<ambientLight intensity={0.8} />
<directionalLight position={[5, 8, 5]} intensity={1.2} />
<ambientLight intensity={0.5 * settings.lightIntensity} />
<directionalLight position={[5, 8, 5]} intensity={1.2 * settings.lightIntensity} />
{settings.showGrid ? (
<Grid
position={[0, -0.01, 0]}
args={[12, 12]}
cellColor="#8c8c8c"
sectionColor="#525252"
cellSize={0.5}
sectionSize={2}
fadeDistance={40}
fadeStrength={1.5}
/>
) : null}
<Suspense
fallback={
<mesh>
@ -114,10 +260,16 @@ export function Model3DViewer({ modelUrl, previewImageUrl }: Model3DViewerProps)
</mesh>
}
>
<ModelScene url={modelUrl} />
<Environment preset="city" />
<ModelScene
url={modelUrl}
settings={settings}
layerVisibility={layerVisibility}
onLayersChange={onLayersChange}
onStatsChange={onStatsChange}
/>
<Environment preset={settings.envPreset} />
</Suspense>
<OrbitControls enablePan enableZoom enableRotate />
<OrbitControls enablePan enableZoom enableRotate autoRotate={settings.autoRotate} />
</Canvas>
<div className="pointer-events-none absolute left-3 top-3 rounded-md bg-background/80 px-2 py-1 text-xs text-muted-foreground">
<span className="inline-flex items-center gap-1">

View File

@ -0,0 +1,302 @@
const RUNWARE_ENDPOINT = "https://api.runware.ai/v1";
const DEFAULT_TIMEOUT_MS = 12_000;
const PROBE_IMAGE_URL =
"https://upload.wikimedia.org/wikipedia/commons/thumb/a/a9/Example.jpg/512px-Example.jpg";
const DEFAULT_MODEL_ID = "runware:101@1";
export type RunwareCapabilityCode =
| "SUPPORTED"
| "UNSUPPORTED_SEED_IMAGE"
| "MODEL_REQUIRES_MASK"
| "AUTH_FAILED"
| "MODEL_INVALID"
| "UPSTREAM_ERROR";
export type RunwareSeedImageProbeResult = {
supported: boolean;
code: RunwareCapabilityCode;
message: string;
status?: number;
};
type RunwareErrorInfo = {
code: string;
message: string;
parameter: string;
};
function toText(value: unknown) {
return typeof value === "string" ? value.trim() : "";
}
function collectErrorInfos(payload: unknown): RunwareErrorInfo[] {
const infos: RunwareErrorInfo[] = [];
const visited = new Set<unknown>();
const queue: unknown[] = [payload];
while (queue.length > 0) {
const current = queue.shift();
if (!current || typeof current !== "object" || visited.has(current)) {
continue;
}
visited.add(current);
if (Array.isArray(current)) {
for (const item of current) {
queue.push(item);
}
continue;
}
const objectValue = current as Record<string, unknown>;
const message = toText(objectValue.message || objectValue.error);
const code = toText(objectValue.code);
const parameter = toText(objectValue.parameter);
const lowerCode = code.toLowerCase();
const looksLikeErrorCode = containsAny(lowerCode, [
"error",
"invalid",
"unsupported",
"unauthorized",
"forbidden",
"notfound",
"not_found",
"failed",
]);
if (message || parameter || looksLikeErrorCode) {
infos.push({ code, message, parameter });
}
const nestedErrors = objectValue.errors;
const nestedData = objectValue.data;
if (nestedErrors) {
queue.push(nestedErrors);
}
if (nestedData) {
queue.push(nestedData);
}
}
return infos;
}
function containsAny(text: string, patterns: string[]) {
return patterns.some((pattern) => text.includes(pattern));
}
function normalizeErrorText(errorInfo: RunwareErrorInfo) {
return `${errorInfo.code} ${errorInfo.parameter} ${errorInfo.message}`.toLowerCase();
}
function isUnsupportedSeedImage(errorInfo: RunwareErrorInfo) {
const text = normalizeErrorText(errorInfo);
return (
containsAny(text, [
"unsupported use of 'seedimage'",
'unsupported use of "seedimage"',
"unsupported use of seedimage",
"seedimage is not supported",
"unsupportedparameter seedimage",
"unsupported parameter seedimage",
]) ||
(text.includes("seedimage") && text.includes("unsupported"))
);
}
function isAuthFailure(status: number, errorInfo: RunwareErrorInfo) {
const text = normalizeErrorText(errorInfo);
if (status === 401 || status === 403) return true;
return containsAny(text, [
"unauthorized",
"forbidden",
"invalid api key",
"invalidapikey",
"authentication failed",
"auth failed",
"invalid credentials",
]);
}
function isModelInvalid(errorInfo: RunwareErrorInfo) {
const text = normalizeErrorText(errorInfo);
return containsAny(text, [
"invalid model",
"model not found",
"unknown model",
"model does not exist",
"invalidmodel",
]);
}
function modelRequiresMask(errorInfo: RunwareErrorInfo) {
const text = normalizeErrorText(errorInfo);
return containsAny(text, [
"missingmaskimage",
"missing maskimage",
"requires a mask image",
"requires mask image",
"requires a mask",
]);
}
function responseHasImage(payload: unknown, taskUUID: string) {
if (!payload) return false;
const list = Array.isArray(payload)
? payload
: Array.isArray((payload as Record<string, unknown>)?.data)
? ((payload as Record<string, unknown>).data as unknown[])
: [];
return list.some((item) => {
if (!item || typeof item !== "object") return false;
const row = item as Record<string, unknown>;
const image = toText(row.imageDataURI || row.imageURL || row.imageUrl);
if (!image) return false;
const rowTaskUUID = toText(row.taskUUID);
return !rowTaskUUID || rowTaskUUID === taskUUID;
});
}
function parseProbeFailure(
status: number,
modelId: string,
infos: RunwareErrorInfo[]
): RunwareSeedImageProbeResult {
const firstMessage = infos.find((info) => info.message)?.message;
if (status === 401 || status === 403) {
return {
supported: false,
code: "AUTH_FAILED",
message: "Runware API Key 无效或无权限,请检查后重试。",
status,
};
}
if (infos.some((info) => isUnsupportedSeedImage(info))) {
return {
supported: false,
code: "UNSUPPORTED_SEED_IMAGE",
message: `模型 ${modelId} 不支持 seedImage,已阻止保存。`,
status,
};
}
if (infos.some((info) => modelRequiresMask(info))) {
return {
supported: false,
code: "MODEL_REQUIRES_MASK",
message: `模型 ${modelId} 依赖 maskImage,当前实时画布链路不兼容。`,
status,
};
}
if (infos.some((info) => isAuthFailure(status, info))) {
return {
supported: false,
code: "AUTH_FAILED",
message: "Runware API Key 无效或无权限,请检查后重试。",
status,
};
}
if (infos.some((info) => isModelInvalid(info))) {
return {
supported: false,
code: "MODEL_INVALID",
message: `模型 ${modelId} 不存在或不可用,请更换模型后重试。`,
status,
};
}
return {
supported: false,
code: "UPSTREAM_ERROR",
message: firstMessage || `Runware 探针请求失败(${status || 0})`,
status,
};
}
export async function probeRunwareSeedImageSupport(params: {
apiKey: string;
modelId?: string;
timeoutMs?: number;
}): Promise<RunwareSeedImageProbeResult> {
const apiKey = params.apiKey.trim();
const modelId = (params.modelId || DEFAULT_MODEL_ID).trim() || DEFAULT_MODEL_ID;
const timeoutMs = Number.isFinite(params.timeoutMs) ? Number(params.timeoutMs) : DEFAULT_TIMEOUT_MS;
if (!apiKey) {
return {
supported: false,
code: "AUTH_FAILED",
message: "缺少 Runware API Key。",
};
}
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), Math.max(2_000, timeoutMs));
let status = 0;
let payload: unknown = null;
try {
const taskUUID = crypto.randomUUID();
const response = await fetch(RUNWARE_ENDPOINT, {
method: "POST",
headers: {
Authorization: `Bearer ${apiKey}`,
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify([
{
taskType: "imageInference",
taskUUID,
model: modelId,
positivePrompt: "probe",
seedImage: PROBE_IMAGE_URL,
width: 512,
height: 512,
strength: 0.2,
CFGScale: 7,
steps: 1,
numberResults: 1,
outputType: "dataURI",
},
]),
cache: "no-store",
signal: controller.signal,
});
status = response.status;
payload = await response.json().catch(() => null);
const infos = collectErrorInfos(payload);
if (response.ok && infos.length === 0 && responseHasImage(payload, taskUUID)) {
return {
supported: true,
code: "SUPPORTED",
message: "模型支持 seedImage。",
status,
};
}
return parseProbeFailure(status, modelId, infos);
} catch (error) {
if (error instanceof Error && error.name === "AbortError") {
return {
supported: false,
code: "UPSTREAM_ERROR",
message: "Runware 探针请求超时,请稍后重试。",
};
}
return {
supported: false,
code: "UPSTREAM_ERROR",
message: error instanceof Error ? error.message : "Runware 探针请求失败。",
};
} finally {
clearTimeout(timer);
}
}

279
lib/realtime/runware-ws.ts Normal file
View File

@ -0,0 +1,279 @@
type RunwareTask = {
taskType: string;
taskUUID?: string;
[key: string]: unknown;
};
type RunwarePayload = {
data?: Array<Record<string, unknown>>;
errors?: Array<Record<string, unknown>>;
[key: string]: unknown;
};
type PendingTask = {
timeout: ReturnType<typeof setTimeout>;
resolve: (value: RunwarePayload) => void;
reject: (reason?: unknown) => void;
};
class RunwareWebSocketClient {
private readonly endpoint = "wss://ws-api.runware.ai/v1";
private socket: WebSocket | null = null;
private connectionSessionUUID = "";
private authenticated = false;
private connectPromise: Promise<void> | null = null;
private pingTimer: ReturnType<typeof setInterval> | null = null;
private authWaiter:
| {
timeout: ReturnType<typeof setTimeout>;
resolve: () => void;
reject: (reason?: unknown) => void;
}
| null = null;
private pending = new Map<string, PendingTask>();
constructor(private readonly apiKey: string) {}
private closeSocket() {
if (!this.socket) return;
try {
this.socket.close();
} catch {}
this.socket = null;
}
private clearPingTimer() {
if (!this.pingTimer) return;
clearInterval(this.pingTimer);
this.pingTimer = null;
}
private rejectPending(reason: unknown) {
for (const [, task] of this.pending) {
clearTimeout(task.timeout);
task.reject(reason);
}
this.pending.clear();
}
private rejectAuthWaiter(reason: unknown) {
if (!this.authWaiter) return;
clearTimeout(this.authWaiter.timeout);
this.authWaiter.reject(reason);
this.authWaiter = null;
}
private startPing() {
this.clearPingTimer();
this.pingTimer = setInterval(() => {
const socket = this.socket;
if (!socket || socket.readyState !== WebSocket.OPEN) return;
try {
socket.send(
JSON.stringify([
{
taskType: "ping",
ping: true,
},
])
);
} catch {}
}, 55_000);
}
private sendAuth() {
const socket = this.socket;
if (!socket || socket.readyState !== WebSocket.OPEN) {
throw new Error("Runware WebSocket 未连接");
}
const authTask: Record<string, unknown> = {
taskType: "authentication",
apiKey: this.apiKey,
};
if (this.connectionSessionUUID) {
authTask.connectionSessionUUID = this.connectionSessionUUID;
}
socket.send(JSON.stringify([authTask]));
}
private parseMessage(raw: unknown) {
if (typeof raw !== "string") return null;
try {
const parsed = JSON.parse(raw) as RunwarePayload;
return parsed;
} catch {
return null;
}
}
private handleMessage(raw: unknown) {
const payload = this.parseMessage(raw);
if (!payload) return;
const dataList = Array.isArray(payload.data) ? payload.data : [];
const errorList = Array.isArray(payload.errors) ? payload.errors : [];
if (this.authWaiter) {
const authError = errorList.find((item) => item?.taskType === "authentication");
const authData = dataList.find((item) => item?.taskType === "authentication");
if (authError) {
const message = String(authError.message || "Runware WebSocket 鉴权失败");
this.rejectAuthWaiter(new Error(message));
} else if (authData) {
const sessionId = String(authData.connectionSessionUUID || "").trim();
if (sessionId) {
this.connectionSessionUUID = sessionId;
}
this.authenticated = true;
this.startPing();
this.authWaiter.resolve();
clearTimeout(this.authWaiter.timeout);
this.authWaiter = null;
}
}
for (const item of dataList) {
const taskUUID = String(item?.taskUUID || "").trim();
if (!taskUUID) continue;
const pendingTask = this.pending.get(taskUUID);
if (!pendingTask) continue;
clearTimeout(pendingTask.timeout);
this.pending.delete(taskUUID);
pendingTask.resolve(payload);
}
for (const item of errorList) {
const taskUUID = String(item?.taskUUID || "").trim();
if (!taskUUID) continue;
const pendingTask = this.pending.get(taskUUID);
if (!pendingTask) continue;
clearTimeout(pendingTask.timeout);
this.pending.delete(taskUUID);
const message = String(item?.message || item?.code || "Runware WebSocket 任务失败");
pendingTask.reject(new Error(message));
}
}
private handleSocketClose() {
this.authenticated = false;
this.clearPingTimer();
this.rejectAuthWaiter(new Error("Runware WebSocket 连接已断开"));
this.rejectPending(new Error("Runware WebSocket 连接已断开"));
this.socket = null;
}
async ensureConnected(timeoutMs = 10_000) {
if (this.socket?.readyState === WebSocket.OPEN && this.authenticated) {
return;
}
if (this.connectPromise) {
return this.connectPromise;
}
this.connectPromise = new Promise<void>((resolve, reject) => {
this.closeSocket();
this.authenticated = false;
this.clearPingTimer();
const socket = new WebSocket(this.endpoint);
this.socket = socket;
const cleanupAndReject = (error: unknown) => {
this.handleSocketClose();
reject(error instanceof Error ? error : new Error("Runware WebSocket 连接失败"));
};
socket.onmessage = (event) => {
this.handleMessage(event.data);
};
socket.onerror = () => {
if (!this.authenticated) {
cleanupAndReject(new Error("Runware WebSocket 连接错误"));
}
};
socket.onclose = () => {
this.handleSocketClose();
};
socket.onopen = () => {
this.authWaiter = {
timeout: setTimeout(() => {
this.rejectAuthWaiter(new Error("Runware WebSocket 鉴权超时"));
this.closeSocket();
reject(new Error("Runware WebSocket 鉴权超时"));
}, timeoutMs),
resolve,
reject: (reason?: unknown) => {
cleanupAndReject(reason);
},
};
try {
this.sendAuth();
} catch (error) {
cleanupAndReject(error);
}
};
}).finally(() => {
this.connectPromise = null;
});
return this.connectPromise;
}
async sendTask(task: RunwareTask, timeoutMs = 90_000) {
const taskUUID = String(task.taskUUID || "").trim();
if (!taskUUID) {
throw new Error("Runware 任务缺少 taskUUID");
}
await this.ensureConnected();
const socket = this.socket;
if (!socket || socket.readyState !== WebSocket.OPEN || !this.authenticated) {
throw new Error("Runware WebSocket 未就绪");
}
return await new Promise<RunwarePayload>((resolve, reject) => {
const timeout = setTimeout(() => {
this.pending.delete(taskUUID);
reject(new Error("Runware WebSocket 推理超时"));
}, timeoutMs);
this.pending.set(taskUUID, {
timeout,
resolve,
reject,
});
try {
socket.send(JSON.stringify([task]));
} catch (error) {
clearTimeout(timeout);
this.pending.delete(taskUUID);
reject(error instanceof Error ? error : new Error("Runware WebSocket 发送失败"));
}
});
}
}
declare global {
// eslint-disable-next-line no-var
var __runwareWebSocketPool: Map<string, RunwareWebSocketClient> | undefined;
}
function getRunwarePool() {
if (!globalThis.__runwareWebSocketPool) {
globalThis.__runwareWebSocketPool = new Map<string, RunwareWebSocketClient>();
}
return globalThis.__runwareWebSocketPool;
}
export function getRunwareWebSocketClient(apiKey: string) {
const pool = getRunwarePool();
let client = pool.get(apiKey);
if (!client) {
client = new RunwareWebSocketClient(apiKey);
pool.set(apiKey, client);
}
return client;
}

View File

@ -126,21 +126,48 @@ export async function tripoRequest<T = unknown>(
export function pickTaskModelUrl(output?: Record<string, unknown> | null): string | null {
if (!output) return null;
const candidates = [output.pbr_model, output.model, output.base_model];
const candidates = [
output.pbr_model,
output.model,
output.base_model,
output.rigged_model,
output.animated_model,
output.low_poly_model,
output.result_model,
output.glb,
output.fbx,
output.usdz,
output.obj,
output.stl,
];
for (const item of candidates) {
const value = typeof item === "string" ? item.trim() : "";
if (value) return value;
if (typeof item === "string") {
const value = item.trim();
if (value) return value;
continue;
}
if (item && typeof item === "object") {
const nested =
typeof (item as Record<string, unknown>).url === "string"
? String((item as Record<string, unknown>).url).trim()
: "";
if (nested) return nested;
}
}
return null;
}
export function pickTaskPreviewImageUrl(output?: Record<string, unknown> | null): string | null {
if (!output) return null;
const candidates = [output.generated_image, output.rendered_image];
const candidates = [
output.generated_image,
output.rendered_image,
output.preview_image,
output.thumbnail,
];
for (const item of candidates) {
const value = typeof item === "string" ? item.trim() : "";
if (value) return value;
}
return null;
}

View File

@ -1,4 +1,17 @@
export type TripoTaskType = "text_to_model" | "image_to_model" | "convert_model";
export type TripoTaskType =
| "text_to_model"
| "image_to_model"
| "multiview_to_model"
| "convert_model"
| "refine_model"
| "stylize_model"
| "texture_model"
| "animate_prerigcheck"
| "animate_rig"
| "animate_retarget"
| "mesh_segmentation"
| "mesh_completion"
| "highpoly_to_lowpoly";
export type TripoTaskStatus =
| "queued"
@ -43,4 +56,3 @@ export interface TripoApiEnvelope<T> {
message?: string;
suggestion?: string;
}