feat: 完善无限画布3D生成与实时画布能力
This commit is contained in:
parent
d272bd5ee8
commit
f6979f0196
@ -91,6 +91,8 @@ import {
|
||||
} from "@/lib/payment/dulupay-config";
|
||||
|
||||
const fetcher = (url: string) => fetch(url).then((res) => res.json());
|
||||
const DEFAULT_RUNWARE_REALTIME_MODEL = "runware:101@1";
|
||||
const DEFAULT_RUNWARE_STYLIZE_MODEL = "runware:106@1";
|
||||
|
||||
// 类型定义
|
||||
type User = {
|
||||
@ -732,6 +734,11 @@ export function AdminSettingsClient() {
|
||||
const [realtimeCanvasConfig, setRealtimeCanvasConfig] = useState({
|
||||
enabled: true,
|
||||
falApiKey: "",
|
||||
runwareModel: DEFAULT_RUNWARE_REALTIME_MODEL,
|
||||
runwareStylizeModel: DEFAULT_RUNWARE_STYLIZE_MODEL,
|
||||
runwareControlModel: "runware:25@1",
|
||||
runwareControlPreprocessor: "canny" as "canny" | "lineart" | "depth",
|
||||
renderMode: "balanced" as "strict-structure" | "balanced" | "stylize",
|
||||
throttleInterval: 100,
|
||||
maxBuffering: 2,
|
||||
maxInputResolution: 1024,
|
||||
@ -877,7 +884,13 @@ export function AdminSettingsClient() {
|
||||
if (realtimeCanvasConfigData?.setting?.value) {
|
||||
const config = realtimeCanvasConfigData.setting.value as {
|
||||
enabled?: boolean;
|
||||
runwareApiKey?: string;
|
||||
falApiKey?: string;
|
||||
runwareModel?: string;
|
||||
runwareStylizeModel?: string;
|
||||
runwareControlModel?: string;
|
||||
runwareControlPreprocessor?: "canny" | "lineart" | "depth";
|
||||
renderMode?: "strict-structure" | "balanced" | "stylize";
|
||||
throttleInterval?: number;
|
||||
maxBuffering?: number;
|
||||
maxInputResolution?: number;
|
||||
@ -891,7 +904,19 @@ export function AdminSettingsClient() {
|
||||
};
|
||||
setRealtimeCanvasConfig({
|
||||
enabled: config.enabled ?? true,
|
||||
falApiKey: config.falApiKey || "",
|
||||
falApiKey: config.runwareApiKey || config.falApiKey || "",
|
||||
runwareModel: config.runwareModel || DEFAULT_RUNWARE_REALTIME_MODEL,
|
||||
runwareStylizeModel: config.runwareStylizeModel || DEFAULT_RUNWARE_STYLIZE_MODEL,
|
||||
runwareControlModel: config.runwareControlModel || "runware:25@1",
|
||||
runwareControlPreprocessor:
|
||||
config.runwareControlPreprocessor === "lineart" ||
|
||||
config.runwareControlPreprocessor === "depth"
|
||||
? config.runwareControlPreprocessor
|
||||
: "canny",
|
||||
renderMode:
|
||||
config.renderMode === "strict-structure" || config.renderMode === "stylize"
|
||||
? config.renderMode
|
||||
: "balanced",
|
||||
throttleInterval: Number(config.throttleInterval || 100),
|
||||
maxBuffering: Number(config.maxBuffering || 2),
|
||||
maxInputResolution: Number(config.maxInputResolution || 1024),
|
||||
@ -1756,11 +1781,39 @@ export function AdminSettingsClient() {
|
||||
// --- 实时画布API配置逻辑 ---
|
||||
const handleSaveRealtimeCanvasConfig = async () => {
|
||||
try {
|
||||
if (realtimeCanvasConfig.enabled && !realtimeCanvasConfig.falApiKey.trim()) {
|
||||
toast.error("启用实时画布功能需要配置 fal.ai API Key");
|
||||
const runwareApiKey = realtimeCanvasConfig.falApiKey.trim();
|
||||
const runwareModel =
|
||||
realtimeCanvasConfig.runwareModel.trim() || DEFAULT_RUNWARE_REALTIME_MODEL;
|
||||
const runwareStylizeModel =
|
||||
realtimeCanvasConfig.runwareStylizeModel.trim() || DEFAULT_RUNWARE_STYLIZE_MODEL;
|
||||
const runwareControlModel =
|
||||
realtimeCanvasConfig.runwareControlModel.trim() || "runware:25@1";
|
||||
|
||||
if (realtimeCanvasConfig.enabled && !runwareApiKey) {
|
||||
toast.error("启用实时画布功能需要配置 Runware API Key");
|
||||
return;
|
||||
}
|
||||
|
||||
if (realtimeCanvasConfig.enabled) {
|
||||
for (const modelId of [runwareModel, runwareStylizeModel].filter(Boolean)) {
|
||||
const probeResponse = await fetch("/api/admin/realtime/model-capability", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
apiKey: runwareApiKey,
|
||||
modelId,
|
||||
}),
|
||||
});
|
||||
const probeData = await probeResponse.json().catch(() => null);
|
||||
if (!probeResponse.ok || probeData?.supported === false) {
|
||||
toast.error(
|
||||
probeData?.message || "模型能力探针失败:当前模型不支持 seedImage,已阻止保存"
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const response = await fetch("/api/admin/settings", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
@ -1768,7 +1821,13 @@ export function AdminSettingsClient() {
|
||||
key: "realtime_canvas_config",
|
||||
value: {
|
||||
enabled: realtimeCanvasConfig.enabled,
|
||||
falApiKey: realtimeCanvasConfig.falApiKey.trim(),
|
||||
runwareApiKey,
|
||||
falApiKey: runwareApiKey,
|
||||
runwareModel,
|
||||
runwareStylizeModel,
|
||||
runwareControlModel,
|
||||
runwareControlPreprocessor: realtimeCanvasConfig.runwareControlPreprocessor,
|
||||
renderMode: realtimeCanvasConfig.renderMode,
|
||||
throttleInterval: Math.min(260, Math.max(60, Number(realtimeCanvasConfig.throttleInterval) || 100)),
|
||||
maxBuffering: Math.min(6, Math.max(1, Number(realtimeCanvasConfig.maxBuffering) || 2)),
|
||||
maxInputResolution: Math.min(
|
||||
@ -1786,10 +1845,13 @@ export function AdminSettingsClient() {
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) throw new Error("保存失败");
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => null);
|
||||
throw new Error(errorData?.message || "保存失败");
|
||||
}
|
||||
toast.success("实时画布配置保存成功");
|
||||
} catch (_error) {
|
||||
toast.error("保存实时画布配置失败");
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : "保存实时画布配置失败");
|
||||
}
|
||||
};
|
||||
|
||||
@ -7598,7 +7660,7 @@ export function AdminSettingsClient() {
|
||||
实时画布配置
|
||||
</CardTitle>
|
||||
<CardDescription>
|
||||
配置实时画布(左画右预览)功能使用的 fal realtime 参数与 API Key。该 Key 存储在数据库,前端不暴露。
|
||||
配置实时画布功能的 Runware 参数、模型与 API Key。保存时会自动探针校验模型是否支持 seedImage。
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-6 pt-6">
|
||||
@ -7624,7 +7686,7 @@ export function AdminSettingsClient() {
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="realtimeFalApiKey">
|
||||
fal.ai API Key <span className="text-destructive">*</span>
|
||||
Runware API Key <span className="text-destructive">*</span>
|
||||
</Label>
|
||||
<Input
|
||||
id="realtimeFalApiKey"
|
||||
@ -7633,11 +7695,103 @@ export function AdminSettingsClient() {
|
||||
onChange={(e) =>
|
||||
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, falApiKey: e.target.value })
|
||||
}
|
||||
placeholder="fal_xxxxxxxxxxxxx"
|
||||
placeholder="rw_xxxxxxxxxxxxx"
|
||||
disabled={!realtimeCanvasConfig.enabled}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="runwareModel">
|
||||
Runware 模型 ID <span className="text-destructive">*</span>
|
||||
</Label>
|
||||
<Input
|
||||
id="runwareModel"
|
||||
value={realtimeCanvasConfig.runwareModel}
|
||||
onChange={(e) =>
|
||||
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, runwareModel: e.target.value })
|
||||
}
|
||||
placeholder={DEFAULT_RUNWARE_REALTIME_MODEL}
|
||||
disabled={!realtimeCanvasConfig.enabled}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="runwareStylizeModel">写实增强模型 ID</Label>
|
||||
<Input
|
||||
id="runwareStylizeModel"
|
||||
value={realtimeCanvasConfig.runwareStylizeModel}
|
||||
onChange={(e) =>
|
||||
setRealtimeCanvasConfig({
|
||||
...realtimeCanvasConfig,
|
||||
runwareStylizeModel: e.target.value,
|
||||
})
|
||||
}
|
||||
placeholder={DEFAULT_RUNWARE_STYLIZE_MODEL}
|
||||
disabled={!realtimeCanvasConfig.enabled}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="runwareControlModel">ControlNet 模型 ID</Label>
|
||||
<Input
|
||||
id="runwareControlModel"
|
||||
value={realtimeCanvasConfig.runwareControlModel}
|
||||
onChange={(e) =>
|
||||
setRealtimeCanvasConfig({
|
||||
...realtimeCanvasConfig,
|
||||
runwareControlModel: e.target.value,
|
||||
})
|
||||
}
|
||||
placeholder="runware:25@1"
|
||||
disabled={!realtimeCanvasConfig.enabled}
|
||||
/>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="space-y-2">
|
||||
<Label>实时渲染模式</Label>
|
||||
<Select
|
||||
value={realtimeCanvasConfig.renderMode}
|
||||
onValueChange={(
|
||||
value: "strict-structure" | "balanced" | "stylize"
|
||||
) =>
|
||||
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, renderMode: value })
|
||||
}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="strict-structure">结构优先</SelectItem>
|
||||
<SelectItem value="balanced">平衡模式</SelectItem>
|
||||
<SelectItem value="stylize">风格优先</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label>ControlNet 预处理</Label>
|
||||
<Select
|
||||
value={realtimeCanvasConfig.runwareControlPreprocessor}
|
||||
onValueChange={(value: "canny" | "lineart" | "depth") =>
|
||||
setRealtimeCanvasConfig({
|
||||
...realtimeCanvasConfig,
|
||||
runwareControlPreprocessor: value,
|
||||
})
|
||||
}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="canny">canny</SelectItem>
|
||||
<SelectItem value="lineart">lineart</SelectItem>
|
||||
<SelectItem value="depth">depth</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
推荐:主模型 {DEFAULT_RUNWARE_REALTIME_MODEL} + ControlNet {`runware:25@1`}。写实模型 {DEFAULT_RUNWARE_STYLIZE_MODEL} 作为回退链路。
|
||||
</p>
|
||||
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="space-y-2">
|
||||
<Label>发送节流(ms)</Label>
|
||||
|
||||
41
app/api/admin/realtime/model-capability/route.ts
Normal file
41
app/api/admin/realtime/model-capability/route.ts
Normal file
@ -0,0 +1,41 @@
|
||||
import { z } from "zod";
|
||||
import { requireAdmin } from "@/lib/middleware/admin";
|
||||
import { ChatSDKError } from "@/lib/errors";
|
||||
import { probeRunwareSeedImageSupport } from "@/lib/realtime/runware-capability";
|
||||
|
||||
const schema = z.object({
|
||||
apiKey: z.string().min(1),
|
||||
modelId: z.string().min(1),
|
||||
});
|
||||
|
||||
function mapStatus(code: string) {
|
||||
if (code === "UPSTREAM_ERROR") return 502;
|
||||
return 400;
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
await requireAdmin();
|
||||
|
||||
const body = await request.json();
|
||||
const data = schema.parse(body);
|
||||
const result = await probeRunwareSeedImageSupport({
|
||||
apiKey: data.apiKey,
|
||||
modelId: data.modelId,
|
||||
});
|
||||
|
||||
if (!result.supported) {
|
||||
return Response.json(result, { status: mapStatus(result.code) });
|
||||
}
|
||||
|
||||
return Response.json(result);
|
||||
} catch (error) {
|
||||
if (error instanceof ChatSDKError) {
|
||||
return error.toResponse();
|
||||
}
|
||||
if (error instanceof z.ZodError) {
|
||||
return new ChatSDKError("bad_request:api", error.message).toResponse();
|
||||
}
|
||||
return new ChatSDKError("offline:admin").toResponse();
|
||||
}
|
||||
}
|
||||
@ -6,12 +6,26 @@ import {
|
||||
upsertSystemSetting,
|
||||
} from "@/lib/db/queries";
|
||||
import { ChatSDKError } from "@/lib/errors";
|
||||
import { probeRunwareSeedImageSupport } from "@/lib/realtime/runware-capability";
|
||||
|
||||
const settingSchema = z.object({
|
||||
key: z.string(),
|
||||
value: z.any(),
|
||||
});
|
||||
|
||||
const DEFAULT_RUNWARE_MODEL = "runware:101@1";
|
||||
const DEFAULT_RUNWARE_STYLIZE_MODEL = "runware:106@1";
|
||||
const DEFAULT_RUNWARE_CONTROL_MODEL = "runware:25@1";
|
||||
|
||||
function isObjectLike(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function mapProbeStatus(code: string) {
|
||||
if (code === "UPSTREAM_ERROR") return 502;
|
||||
return 400;
|
||||
}
|
||||
|
||||
export async function GET(request: Request) {
|
||||
try {
|
||||
await requireAdmin();
|
||||
@ -41,9 +55,69 @@ export async function POST(request: Request) {
|
||||
const body = await request.json();
|
||||
const { key, value } = settingSchema.parse(body);
|
||||
|
||||
let nextValue = value;
|
||||
|
||||
if (key === "realtime_canvas_config") {
|
||||
if (!isObjectLike(value)) {
|
||||
return new ChatSDKError("bad_request:api", "realtime_canvas_config value 必须为对象").toResponse();
|
||||
}
|
||||
|
||||
const enabled = value.enabled !== false;
|
||||
const runwareApiKey = String(value.runwareApiKey || value.falApiKey || value.apiKey || "").trim();
|
||||
const runwareModel = String(value.runwareModel || DEFAULT_RUNWARE_MODEL).trim() || DEFAULT_RUNWARE_MODEL;
|
||||
const runwareStylizeModel =
|
||||
String(value.runwareStylizeModel || DEFAULT_RUNWARE_STYLIZE_MODEL).trim() ||
|
||||
DEFAULT_RUNWARE_STYLIZE_MODEL;
|
||||
const runwareControlModel =
|
||||
String(value.runwareControlModel || DEFAULT_RUNWARE_CONTROL_MODEL).trim() ||
|
||||
DEFAULT_RUNWARE_CONTROL_MODEL;
|
||||
const runwareControlPreprocessor =
|
||||
value.runwareControlPreprocessor === "lineart" || value.runwareControlPreprocessor === "depth"
|
||||
? value.runwareControlPreprocessor
|
||||
: "canny";
|
||||
const renderMode =
|
||||
value.renderMode === "strict-structure" ||
|
||||
value.renderMode === "stylize" ||
|
||||
value.renderMode === "balanced"
|
||||
? value.renderMode
|
||||
: "balanced";
|
||||
|
||||
if (enabled) {
|
||||
if (!runwareApiKey) {
|
||||
return new ChatSDKError("bad_request:api", "启用实时画布必须配置 Runware API Key").toResponse();
|
||||
}
|
||||
for (const modelId of [runwareModel, runwareStylizeModel].filter(Boolean)) {
|
||||
const probeResult = await probeRunwareSeedImageSupport({
|
||||
apiKey: runwareApiKey,
|
||||
modelId,
|
||||
});
|
||||
if (!probeResult.supported) {
|
||||
return Response.json(
|
||||
{
|
||||
code: probeResult.code,
|
||||
message: probeResult.message,
|
||||
},
|
||||
{ status: mapProbeStatus(probeResult.code) }
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nextValue = {
|
||||
...value,
|
||||
runwareApiKey,
|
||||
falApiKey: runwareApiKey,
|
||||
runwareModel,
|
||||
runwareStylizeModel,
|
||||
runwareControlModel,
|
||||
runwareControlPreprocessor,
|
||||
renderMode,
|
||||
};
|
||||
}
|
||||
|
||||
const setting = await upsertSystemSetting({
|
||||
key,
|
||||
value,
|
||||
value: nextValue,
|
||||
updatedBy: session.user.id,
|
||||
});
|
||||
|
||||
@ -58,4 +132,3 @@ export async function POST(request: Request) {
|
||||
return new ChatSDKError("offline:admin").toResponse();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -7,14 +7,44 @@ export const runtime = "nodejs";
|
||||
export const maxDuration = 60;
|
||||
|
||||
type CreateTaskRequest = {
|
||||
taskType: "text_to_model" | "image_to_model" | "convert_model";
|
||||
taskType:
|
||||
| "text_to_model"
|
||||
| "image_to_model"
|
||||
| "multiview_to_model"
|
||||
| "convert_model"
|
||||
| "refine_model"
|
||||
| "stylize_model"
|
||||
| "texture_model"
|
||||
| "animate_prerigcheck"
|
||||
| "animate_rig"
|
||||
| "animate_retarget"
|
||||
| "mesh_segmentation"
|
||||
| "mesh_completion"
|
||||
| "highpoly_to_lowpoly";
|
||||
prompt?: string;
|
||||
imageToken?: string;
|
||||
imageUrl?: string;
|
||||
imageType?: string;
|
||||
file?: Record<string, unknown>;
|
||||
files?: Array<Record<string, unknown>>;
|
||||
imageTokens?: string[];
|
||||
imageUrls?: string[];
|
||||
originalModelTaskId?: string;
|
||||
draftModelTaskId?: string;
|
||||
format?: "GLTF" | "USDZ" | "FBX" | "OBJ" | "STL" | "3MF";
|
||||
style?: "lego" | "voxel" | "voronoi" | "minecraft";
|
||||
blockSize?: number;
|
||||
outFormat?: "glb" | "fbx";
|
||||
rigType?: "biped" | "quadruped" | "hexapod" | "octopod" | "avian" | "serpentine" | "aquatic" | "others";
|
||||
spec?: "mixamo" | "tripo";
|
||||
animation?: string;
|
||||
animations?: string[];
|
||||
texturePromptText?: string;
|
||||
texturePromptImageToken?: string;
|
||||
texturePromptStyleImageToken?: string;
|
||||
partNames?: string[];
|
||||
bakeAnimation?: boolean;
|
||||
exportWithGeometry?: boolean;
|
||||
modelVersion?: string;
|
||||
negativePrompt?: string;
|
||||
faceLimit?: number;
|
||||
@ -116,6 +146,48 @@ export async function POST(request: Request) {
|
||||
payload.file = inputFile;
|
||||
}
|
||||
|
||||
if (taskType === "multiview_to_model") {
|
||||
const inputFiles: Array<Record<string, unknown>> = [];
|
||||
|
||||
if (Array.isArray(body.files)) {
|
||||
for (const item of body.files) {
|
||||
if (item && typeof item === "object") {
|
||||
inputFiles.push(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (inputFiles.length === 0 && Array.isArray(body.imageTokens)) {
|
||||
for (const token of body.imageTokens) {
|
||||
const normalized = String(token || "").trim();
|
||||
if (!normalized) continue;
|
||||
inputFiles.push({
|
||||
type: String(body.imageType || "jpg"),
|
||||
file_token: normalized,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (inputFiles.length === 0 && Array.isArray(body.imageUrls)) {
|
||||
for (const imageUrl of body.imageUrls) {
|
||||
const normalized = String(imageUrl || "").trim();
|
||||
if (!normalized) continue;
|
||||
inputFiles.push({
|
||||
type: String(body.imageType || "jpg"),
|
||||
url: normalized,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (inputFiles.length < 2) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "多视图生成至少需要 2 张参考图" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
payload.files = inputFiles;
|
||||
}
|
||||
|
||||
if (taskType === "convert_model") {
|
||||
const originalModelTaskId = String(body.originalModelTaskId || "").trim();
|
||||
if (!originalModelTaskId) {
|
||||
@ -131,6 +203,85 @@ export async function POST(request: Request) {
|
||||
payload.format = body.format;
|
||||
}
|
||||
|
||||
if (
|
||||
taskType === "stylize_model" ||
|
||||
taskType === "texture_model" ||
|
||||
taskType === "animate_prerigcheck" ||
|
||||
taskType === "animate_rig" ||
|
||||
taskType === "animate_retarget" ||
|
||||
taskType === "mesh_segmentation" ||
|
||||
taskType === "mesh_completion" ||
|
||||
taskType === "highpoly_to_lowpoly"
|
||||
) {
|
||||
const originalModelTaskId = String(body.originalModelTaskId || "").trim();
|
||||
if (!originalModelTaskId) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "缺少 originalModelTaskId" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
payload.original_model_task_id = originalModelTaskId;
|
||||
}
|
||||
|
||||
if (taskType === "stylize_model") {
|
||||
if (!body.style) {
|
||||
return NextResponse.json({ success: false, error: "缺少 style" }, { status: 400 });
|
||||
}
|
||||
payload.style = body.style;
|
||||
}
|
||||
|
||||
if (taskType === "animate_retarget") {
|
||||
const singleAnimation = String(body.animation || "").trim();
|
||||
const animationList = Array.isArray(body.animations)
|
||||
? body.animations.map((item) => String(item || "").trim()).filter(Boolean)
|
||||
: [];
|
||||
if (!singleAnimation && animationList.length === 0) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "缺少 animation / animations" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
if (singleAnimation) {
|
||||
payload.animation = singleAnimation;
|
||||
} else {
|
||||
payload.animations = animationList;
|
||||
}
|
||||
}
|
||||
|
||||
if (taskType === "refine_model") {
|
||||
const draftModelTaskId = String(body.draftModelTaskId || "").trim();
|
||||
if (!draftModelTaskId) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "缺少 draftModelTaskId" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
payload.draft_model_task_id = draftModelTaskId;
|
||||
}
|
||||
|
||||
if (taskType === "texture_model") {
|
||||
const texturePromptText = String(body.texturePromptText || "").trim();
|
||||
const texturePromptImageToken = String(body.texturePromptImageToken || "").trim();
|
||||
const texturePromptStyleImageToken = String(body.texturePromptStyleImageToken || "").trim();
|
||||
if (texturePromptText || texturePromptImageToken || texturePromptStyleImageToken) {
|
||||
const texturePrompt: Record<string, unknown> = {};
|
||||
if (texturePromptText) texturePrompt.text = texturePromptText;
|
||||
if (texturePromptImageToken) {
|
||||
texturePrompt.image = {
|
||||
type: String(body.imageType || "jpg"),
|
||||
file_token: texturePromptImageToken,
|
||||
};
|
||||
}
|
||||
if (texturePromptStyleImageToken) {
|
||||
texturePrompt.style_image = {
|
||||
type: String(body.imageType || "jpg"),
|
||||
file_token: texturePromptStyleImageToken,
|
||||
};
|
||||
}
|
||||
payload.texture_prompt = texturePrompt;
|
||||
}
|
||||
}
|
||||
|
||||
assignMapped(payload, body as Record<string, unknown>, {
|
||||
modelVersion: "model_version",
|
||||
negativePrompt: "negative_prompt",
|
||||
@ -154,6 +305,13 @@ export async function POST(request: Request) {
|
||||
textureSize: "texture_size",
|
||||
textureFormat: "texture_format",
|
||||
scaleFactor: "scale_factor",
|
||||
blockSize: "block_size",
|
||||
outFormat: "out_format",
|
||||
rigType: "rig_type",
|
||||
spec: "spec",
|
||||
partNames: "part_names",
|
||||
bakeAnimation: "bake_animation",
|
||||
exportWithGeometry: "export_with_geometry",
|
||||
pivotToCenterBottom: "pivot_to_center_bottom",
|
||||
withAnimation: "with_animation",
|
||||
packUv: "pack_uv",
|
||||
@ -220,4 +378,3 @@ export async function POST(request: Request) {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -19,6 +19,74 @@ type EnhanceApiConfig = {
|
||||
backupModel?: string;
|
||||
};
|
||||
|
||||
const AOS_MAX_RESOLUTION = 4096;
|
||||
const ENHANCE_CREATE_TIMEOUT_MS = 120_000;
|
||||
const ENHANCE_CREATE_RETRY_COUNT = 1;
|
||||
|
||||
function getAosFriendlyErrorMessage(detail: string): string | null {
|
||||
const resolutionMatch = detail.match(/resolution excess,\s*width=(\d+),\s*height=(\d+)/i);
|
||||
if (resolutionMatch) {
|
||||
const [, width, height] = resolutionMatch;
|
||||
return `图片分辨率超限(${width}x${height}),AOS要求宽高都不超过 ${AOS_MAX_RESOLUTION}px。请缩小后重试。`;
|
||||
}
|
||||
|
||||
if (/invalid file/i.test(detail)) {
|
||||
return "图片文件无法识别,请尝试转换为 JPG/PNG 后重试。";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function isRetryableNetworkError(error: unknown): boolean {
|
||||
if (!(error instanceof Error)) return false;
|
||||
if (error.name === "AbortError") return true;
|
||||
|
||||
const message = error.message || "";
|
||||
const cause = (error as Error & { cause?: { message?: string; code?: string } }).cause;
|
||||
const causeMessage = cause?.message || "";
|
||||
const causeCode = cause?.code || "";
|
||||
const combined = `${message} ${causeMessage} ${causeCode}`;
|
||||
|
||||
return /EHOSTUNREACH|ENOTFOUND|ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|No route to host/i.test(combined);
|
||||
}
|
||||
|
||||
async function ensureAosResolutionLimit(
|
||||
buffer: Buffer,
|
||||
logPrefix: string
|
||||
): Promise<Buffer> {
|
||||
try {
|
||||
const sharp = await import("sharp");
|
||||
const image = sharp.default(buffer);
|
||||
const metadata = await image.metadata();
|
||||
const width = metadata.width;
|
||||
const height = metadata.height;
|
||||
|
||||
if (!width || !height) {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
if (width <= AOS_MAX_RESOLUTION && height <= AOS_MAX_RESOLUTION) {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
const resizedBuffer = await image
|
||||
.resize(AOS_MAX_RESOLUTION, AOS_MAX_RESOLUTION, {
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.toBuffer();
|
||||
const resizedMeta = await sharp.default(resizedBuffer).metadata();
|
||||
|
||||
console.warn(
|
||||
`[${logPrefix}] 输入图片分辨率超限,已自动缩放: ${width}x${height} -> ${resizedMeta.width}x${resizedMeta.height}`
|
||||
);
|
||||
return resizedBuffer;
|
||||
} catch (error) {
|
||||
console.error(`[${logPrefix}] 自动缩放失败,继续使用原图:`, error);
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 压缩图片生成缩略图(使用 sharp)
|
||||
*/
|
||||
@ -277,7 +345,10 @@ async function pollTaskResult(
|
||||
|
||||
// state < 0 表示失败
|
||||
if (state < 0) {
|
||||
throw result;
|
||||
const errMessage = String(
|
||||
result.data?.err_message || result.data?.err_info || result.data?.state_detail || `state=${state}`
|
||||
);
|
||||
throw new Error(`AOS任务失败: ${errMessage}`);
|
||||
}
|
||||
|
||||
// progress >= 100 表示完成
|
||||
@ -294,11 +365,11 @@ async function pollTaskResult(
|
||||
const result = await getTaskResult();
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (error && typeof error === 'object' && 'data' in error) {
|
||||
if (error && typeof error === "object" && "data" in error) {
|
||||
throw new Error(JSON.stringify(error, null, 2));
|
||||
}
|
||||
if (timeout <= 0) {
|
||||
throw new Error('轮询超时');
|
||||
throw new Error("轮询超时");
|
||||
}
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(() => {
|
||||
@ -533,7 +604,8 @@ export async function POST(request: NextRequest) {
|
||||
|
||||
// 将Base64转换为Buffer
|
||||
console.log("[Enhance API] 转换Base64为Buffer,长度:", imageBase64.length);
|
||||
const imageBuffer = Buffer.from(imageBase64, "base64");
|
||||
const rawImageBuffer = Buffer.from(imageBase64, "base64");
|
||||
const imageBuffer = await ensureAosResolutionLimit(rawImageBuffer, "Enhance API");
|
||||
console.log("[Enhance API] Buffer创建完成,大小:", imageBuffer.length, "bytes");
|
||||
|
||||
// 创建FormData(使用form-data包,Node.js环境)
|
||||
@ -564,10 +636,6 @@ export async function POST(request: NextRequest) {
|
||||
|
||||
try {
|
||||
console.log("[Enhance API] 开始创建任务...");
|
||||
// 添加超时控制(60秒)
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 60000);
|
||||
|
||||
// 将FormData转换为Buffer(form-data包是stream)
|
||||
console.log("[Enhance API] 转换FormData为Buffer...");
|
||||
const formDataBuffer = await new Promise<Buffer>((resolve, reject) => {
|
||||
@ -603,20 +671,44 @@ export async function POST(request: NextRequest) {
|
||||
});
|
||||
|
||||
console.log("[Enhance API] FormData Buffer转换完成,大小:", formDataBuffer.length, "bytes");
|
||||
|
||||
// 将Buffer转换为Uint8Array用于fetch(更兼容)
|
||||
const uint8Array = new Uint8Array(formDataBuffer);
|
||||
|
||||
console.log("[Enhance API] 发送创建任务请求...");
|
||||
const createResponse = await fetch(enhanceUrl, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: uint8Array,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
console.log("[Enhance API] 创建任务响应状态:", createResponse.status, createResponse.statusText);
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
let createResponse: Response | null = null;
|
||||
let lastFetchError: unknown;
|
||||
|
||||
for (let attempt = 1; attempt <= ENHANCE_CREATE_RETRY_COUNT + 1; attempt++) {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), ENHANCE_CREATE_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
console.log(`[Enhance API] 发送创建任务请求... attempt=${attempt}`);
|
||||
createResponse = await fetch(enhanceUrl, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: uint8Array,
|
||||
signal: controller.signal,
|
||||
});
|
||||
console.log("[Enhance API] 创建任务响应状态:", createResponse.status, createResponse.statusText);
|
||||
clearTimeout(timeoutId);
|
||||
break;
|
||||
} catch (fetchError) {
|
||||
clearTimeout(timeoutId);
|
||||
lastFetchError = fetchError;
|
||||
const canRetry = attempt <= ENHANCE_CREATE_RETRY_COUNT && isRetryableNetworkError(fetchError);
|
||||
if (canRetry) {
|
||||
console.warn(
|
||||
`[Enhance API] 创建任务失败,准备重试: attempt=${attempt}, error=${fetchError instanceof Error ? fetchError.message : String(fetchError)}`
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
continue;
|
||||
}
|
||||
throw fetchError;
|
||||
}
|
||||
}
|
||||
|
||||
if (!createResponse) {
|
||||
throw lastFetchError instanceof Error ? lastFetchError : new Error("创建任务请求失败");
|
||||
}
|
||||
|
||||
if (!createResponse.ok) {
|
||||
const errorText = await createResponse.text();
|
||||
@ -641,34 +733,34 @@ export async function POST(request: NextRequest) {
|
||||
|
||||
// 检查是否是网络连接错误
|
||||
if (fetchError instanceof Error) {
|
||||
if (fetchError.name === 'AbortError') {
|
||||
if (fetchError.name === "AbortError") {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "变清晰API请求超时",
|
||||
detail: "创建任务请求超过60秒未响应,请检查API服务器状态或网络连接",
|
||||
detail: "创建任务请求超过120秒未响应,请检查API服务器状态或网络连接",
|
||||
},
|
||||
{ status: 504 }
|
||||
);
|
||||
}
|
||||
|
||||
// 检查错误消息和cause中的网络错误
|
||||
const errorMessage = fetchError.message || '';
|
||||
const errorMessage = fetchError.message || "";
|
||||
const errorCause = (fetchError as any).cause;
|
||||
const causeMessage = errorCause?.message || '';
|
||||
const causeCode = errorCause?.code || '';
|
||||
const causeMessage = errorCause?.message || "";
|
||||
const causeCode = errorCause?.code || "";
|
||||
|
||||
// 检查各种网络连接错误
|
||||
if (errorMessage.includes('EHOSTUNREACH') ||
|
||||
errorMessage.includes('ENOTFOUND') ||
|
||||
errorMessage.includes('ECONNREFUSED') ||
|
||||
errorMessage.includes('No route to host') ||
|
||||
causeMessage.includes('EHOSTUNREACH') ||
|
||||
causeMessage.includes('ENOTFOUND') ||
|
||||
causeMessage.includes('ECONNREFUSED') ||
|
||||
causeMessage.includes('No route to host') ||
|
||||
causeCode === 'EHOSTUNREACH' ||
|
||||
causeCode === 'ENOTFOUND' ||
|
||||
causeCode === 'ECONNREFUSED') {
|
||||
if (errorMessage.includes("EHOSTUNREACH") ||
|
||||
errorMessage.includes("ENOTFOUND") ||
|
||||
errorMessage.includes("ECONNREFUSED") ||
|
||||
errorMessage.includes("No route to host") ||
|
||||
causeMessage.includes("EHOSTUNREACH") ||
|
||||
causeMessage.includes("ENOTFOUND") ||
|
||||
causeMessage.includes("ECONNREFUSED") ||
|
||||
causeMessage.includes("No route to host") ||
|
||||
causeCode === "EHOSTUNREACH" ||
|
||||
causeCode === "ENOTFOUND" ||
|
||||
causeCode === "ECONNREFUSED") {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "无法连接到变清晰API服务器",
|
||||
@ -690,7 +782,17 @@ export async function POST(request: NextRequest) {
|
||||
|
||||
// 4. 轮询获取任务结果(根据官方示例)
|
||||
console.log(`[Enhance] 开始轮询任务结果,task_id: ${taskId}`);
|
||||
const taskResult = await pollTaskResult(apiUrl, config.apiKey!, taskId);
|
||||
let taskResult: any;
|
||||
try {
|
||||
taskResult = await pollTaskResult(apiUrl, config.apiKey!, taskId);
|
||||
} catch (pollError) {
|
||||
const detail = pollError instanceof Error ? pollError.message : "轮询失败";
|
||||
const friendlyMessage = getAosFriendlyErrorMessage(detail);
|
||||
return NextResponse.json(
|
||||
{ error: "变清晰处理失败", detail: friendlyMessage || detail },
|
||||
{ status: friendlyMessage ? 400 : 500 }
|
||||
);
|
||||
}
|
||||
|
||||
const finalProcessingTime = (Date.now() - startTime) / 1000;
|
||||
|
||||
@ -847,4 +949,3 @@ export async function POST(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -5,6 +5,9 @@ type RealtimeCanvasConfig = {
|
||||
enabled?: boolean;
|
||||
runwareApiKey?: string;
|
||||
runwareModel?: string;
|
||||
runwareControlModel?: string;
|
||||
runwareControlPreprocessor?: "canny" | "lineart" | "depth";
|
||||
renderMode?: "strict-structure" | "balanced" | "stylize";
|
||||
falApiKey?: string;
|
||||
apiKey?: string;
|
||||
strictSketch?: boolean;
|
||||
@ -163,6 +166,18 @@ export async function GET() {
|
||||
},
|
||||
behavior: {
|
||||
strictSketch: config.strictSketch !== false,
|
||||
renderMode:
|
||||
config.renderMode === "strict-structure" || config.renderMode === "stylize"
|
||||
? config.renderMode
|
||||
: "balanced",
|
||||
},
|
||||
control: {
|
||||
model: String(config.runwareControlModel || "runware:25@1").trim() || "runware:25@1",
|
||||
preprocessor:
|
||||
config.runwareControlPreprocessor === "lineart" ||
|
||||
config.runwareControlPreprocessor === "depth"
|
||||
? config.runwareControlPreprocessor
|
||||
: "canny",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@ -1,10 +1,18 @@
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { getSystemSetting } from "@/lib/db/queries";
|
||||
import { getRunwareWebSocketClient } from "@/lib/realtime/runware-ws";
|
||||
|
||||
type RealtimeCanvasConfig = {
|
||||
enabled?: boolean;
|
||||
runwareApiKey?: string;
|
||||
runwareModel?: string;
|
||||
runwareStylizeModel?: string;
|
||||
runwareControlModel?: string;
|
||||
runwareControlPreprocessor?: "canny" | "lineart" | "depth";
|
||||
renderMode?: "strict-structure" | "balanced" | "stylize";
|
||||
runwareUseWebSocket?: boolean;
|
||||
falRealtimeEnabled?: boolean;
|
||||
falRealtimeModel?: string;
|
||||
falApiKey?: string;
|
||||
apiKey?: string;
|
||||
negativePrompt?: string;
|
||||
@ -17,9 +25,23 @@ type RealtimeCanvasConfig = {
|
||||
};
|
||||
};
|
||||
|
||||
type AngleEditConfig = {
|
||||
falApiKey?: string;
|
||||
};
|
||||
|
||||
type GeneratePayload = {
|
||||
requestId?: string;
|
||||
imageUrl?: string;
|
||||
maskImageUrl?: string;
|
||||
qualityProfile?: "preview" | "refine";
|
||||
roi?:
|
||||
| {
|
||||
x?: number;
|
||||
y?: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
}
|
||||
| null;
|
||||
prompt?: string;
|
||||
seed?: number;
|
||||
strictSketch?: boolean;
|
||||
@ -29,6 +51,12 @@ type GeneratePayload = {
|
||||
strength?: number;
|
||||
};
|
||||
|
||||
const FALLBACK_REALTIME_PROMPT =
|
||||
"严格按照输入草图生成:保持主体轮廓、位置、比例和数量一致,只做材质与光影细化;禁止改变构图,禁止新增无关背景纹理。";
|
||||
const FALLBACK_FAL_REALTIME_MODEL = "fal-ai/flux-2/klein/realtime";
|
||||
const FALLBACK_RUNWARE_STYLIZE_MODEL = "runware:106@1";
|
||||
const FALLBACK_RUNWARE_CONTROL_MODEL = "runware:25@1";
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
@ -63,6 +91,11 @@ function parseRunwareError(payload: any) {
|
||||
}
|
||||
if (typeof payload?.error === "string") return payload.error;
|
||||
if (typeof payload?.message === "string") return payload.message;
|
||||
if (Array.isArray(payload?.errors) && payload.errors.length > 0) {
|
||||
const firstError = payload.errors[0];
|
||||
if (typeof firstError?.message === "string") return firstError.message;
|
||||
if (typeof firstError?.code === "string") return firstError.code;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
@ -74,6 +107,128 @@ function resolveSeed(seed: unknown) {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function shouldRetryWithoutMask(errorMessage: string) {
|
||||
const normalized = errorMessage.toLowerCase();
|
||||
return (
|
||||
normalized.includes("unsupported use of 'maskimage'") ||
|
||||
normalized.includes("unsupported use of \"maskimage\"") ||
|
||||
normalized.includes("unsupported use of maskimage") ||
|
||||
normalized.includes("maskimage is not supported")
|
||||
);
|
||||
}
|
||||
|
||||
function shouldRetryWithoutNegativePrompt(errorMessage: string) {
|
||||
const normalized = errorMessage.toLowerCase();
|
||||
return (
|
||||
normalized.includes("unsupportedarchitecturenegativeprompt") ||
|
||||
normalized.includes("negative prompt is not supported") ||
|
||||
normalized.includes("unsupported use of 'negativeprompt'") ||
|
||||
normalized.includes("unsupported use of \"negativeprompt\"")
|
||||
);
|
||||
}
|
||||
|
||||
function shouldRetryWithoutControlNet(errorMessage: string) {
|
||||
const normalized = errorMessage.toLowerCase();
|
||||
return (
|
||||
normalized.includes("unsupportedcontrolnetmodel") ||
|
||||
normalized.includes("unsupported use of 'controlnet'") ||
|
||||
normalized.includes("unsupported use of \"controlnet\"") ||
|
||||
normalized.includes("controlnet is not supported")
|
||||
);
|
||||
}
|
||||
|
||||
function resolveRenderModeSettings(
|
||||
mode: "strict-structure" | "balanced" | "stylize"
|
||||
) {
|
||||
if (mode === "strict-structure") {
|
||||
return {
|
||||
controlWeight: 1,
|
||||
seedStrengthMin: 0.14,
|
||||
seedStrengthMax: 0.58,
|
||||
};
|
||||
}
|
||||
if (mode === "stylize") {
|
||||
return {
|
||||
controlWeight: 0.72,
|
||||
seedStrengthMin: 0.28,
|
||||
seedStrengthMax: 0.8,
|
||||
};
|
||||
}
|
||||
return {
|
||||
controlWeight: 0.86,
|
||||
seedStrengthMin: 0.2,
|
||||
seedStrengthMax: 0.7,
|
||||
};
|
||||
}
|
||||
|
||||
function extractFalImageUrl(payload: any) {
|
||||
const direct =
|
||||
payload?.images?.[0]?.url ||
|
||||
payload?.image?.url ||
|
||||
payload?.output?.[0]?.url ||
|
||||
payload?.data?.images?.[0]?.url;
|
||||
if (typeof direct === "string" && direct.trim()) {
|
||||
return direct.trim();
|
||||
}
|
||||
|
||||
const queue: any[] = [payload];
|
||||
const visited = new Set<any>();
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
if (!current || typeof current !== "object" || visited.has(current)) continue;
|
||||
visited.add(current);
|
||||
for (const value of Object.values(current)) {
|
||||
if (typeof value === "string" && /^https?:\/\//i.test(value)) {
|
||||
return value;
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
queue.push(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function extractRunwareOutputImage(payload: any, taskUUID: string) {
|
||||
const resultList = Array.isArray(payload)
|
||||
? payload
|
||||
: Array.isArray(payload?.data)
|
||||
? payload.data
|
||||
: [];
|
||||
const imageResult =
|
||||
resultList.find(
|
||||
(item: any) =>
|
||||
item?.taskUUID === taskUUID && (item?.imageDataURI || item?.imageURL)
|
||||
) ||
|
||||
resultList.find((item: any) => item?.imageDataURI || item?.imageURL);
|
||||
const imageDataURI = String(imageResult?.imageDataURI || "").trim();
|
||||
const imageURL = String(imageResult?.imageURL || "").trim();
|
||||
const outputImage = imageDataURI || imageURL;
|
||||
return {
|
||||
imageResult,
|
||||
imageDataURI,
|
||||
imageURL,
|
||||
outputImage,
|
||||
};
|
||||
}
|
||||
|
||||
function extractControlGuideImage(payload: any, taskUUID: string) {
|
||||
const resultList = Array.isArray(payload)
|
||||
? payload
|
||||
: Array.isArray(payload?.data)
|
||||
? payload.data
|
||||
: [];
|
||||
const guideResult =
|
||||
resultList.find(
|
||||
(item: any) =>
|
||||
item?.taskUUID === taskUUID && (item?.guideImageURL || item?.guideImageDataURI)
|
||||
) ||
|
||||
resultList.find((item: any) => item?.guideImageURL || item?.guideImageDataURI);
|
||||
const guideImageDataURI = String(guideResult?.guideImageDataURI || "").trim();
|
||||
const guideImageURL = String(guideResult?.guideImageURL || "").trim();
|
||||
return guideImageDataURI || guideImageURL;
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) {
|
||||
@ -91,9 +246,14 @@ export async function POST(request: Request) {
|
||||
if (!imageUrl) {
|
||||
return Response.json({ code: "BAD_REQUEST", message: "缺少 imageUrl" }, { status: 400 });
|
||||
}
|
||||
const maskImageUrl = String(body.maskImageUrl || "").trim();
|
||||
|
||||
const setting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null);
|
||||
const [setting, angleEditSetting] = await Promise.all([
|
||||
getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null),
|
||||
getSystemSetting({ key: "angle_edit_config" }).catch(() => null),
|
||||
]);
|
||||
const config = (setting?.value || {}) as RealtimeCanvasConfig;
|
||||
const angleEditConfig = (angleEditSetting?.value || {}) as AngleEditConfig;
|
||||
|
||||
if (config.enabled === false) {
|
||||
return Response.json(
|
||||
@ -110,6 +270,13 @@ export async function POST(request: Request) {
|
||||
process.env.RUNWARE_KEY ||
|
||||
""
|
||||
).trim();
|
||||
const falApiKey = String(
|
||||
config.falApiKey ||
|
||||
angleEditConfig.falApiKey ||
|
||||
process.env.FAL_KEY ||
|
||||
process.env.FAL_API_KEY ||
|
||||
""
|
||||
).trim();
|
||||
if (!runwareApiKey) {
|
||||
return Response.json(
|
||||
{
|
||||
@ -121,11 +288,43 @@ export async function POST(request: Request) {
|
||||
}
|
||||
|
||||
const fallbackSeedImageModel = "runware:101@1";
|
||||
const model = String(
|
||||
const configuredModel = String(
|
||||
config.runwareModel || process.env.RUNWARE_MODEL || fallbackSeedImageModel
|
||||
).trim();
|
||||
const model = configuredModel || fallbackSeedImageModel;
|
||||
const configuredStylizeModel = String(
|
||||
config.runwareStylizeModel ||
|
||||
process.env.RUNWARE_REALTIME_STYLIZE_MODEL ||
|
||||
FALLBACK_RUNWARE_STYLIZE_MODEL
|
||||
).trim();
|
||||
const stylizeModel = configuredStylizeModel || FALLBACK_RUNWARE_STYLIZE_MODEL;
|
||||
const configuredControlModel = String(
|
||||
config.runwareControlModel ||
|
||||
process.env.RUNWARE_REALTIME_CONTROL_MODEL ||
|
||||
FALLBACK_RUNWARE_CONTROL_MODEL
|
||||
).trim();
|
||||
const controlModel = configuredControlModel || FALLBACK_RUNWARE_CONTROL_MODEL;
|
||||
const controlPreprocessor =
|
||||
config.runwareControlPreprocessor === "lineart" ||
|
||||
config.runwareControlPreprocessor === "depth"
|
||||
? config.runwareControlPreprocessor
|
||||
: "canny";
|
||||
const renderMode =
|
||||
config.renderMode === "strict-structure" ||
|
||||
config.renderMode === "stylize" ||
|
||||
config.renderMode === "balanced"
|
||||
? config.renderMode
|
||||
: "balanced";
|
||||
const renderModeSettings = resolveRenderModeSettings(renderMode);
|
||||
const falRealtimeModel = String(
|
||||
config.falRealtimeModel || process.env.FAL_REALTIME_MODEL || FALLBACK_FAL_REALTIME_MODEL
|
||||
).trim();
|
||||
const preferWebSocket =
|
||||
config.runwareUseWebSocket !== false &&
|
||||
process.env.RUNWARE_REALTIME_USE_WS !== "0";
|
||||
const maxInputResolution = clamp(Number(config.maxInputResolution || 1024), 512, 1536);
|
||||
const size = resolveSquareSize(body.imageSize, maxInputResolution);
|
||||
const qualityProfile = body.qualityProfile === "preview" ? "preview" : "refine";
|
||||
const steps = clamp(
|
||||
Number(body.steps || config.defaults?.steps || 8),
|
||||
1,
|
||||
@ -144,64 +343,447 @@ export async function POST(request: Request) {
|
||||
const strictSketch = Boolean(body.strictSketch ?? config.strictSketch ?? true);
|
||||
const seed = resolveSeed(body.seed);
|
||||
|
||||
const positivePrompt =
|
||||
String(body.prompt || "").trim() ||
|
||||
"严格按照输入草图生成:保持主体轮廓、位置、比例和数量一致,只做材质与光影细化;禁止改变构图,禁止新增无关背景纹理。";
|
||||
const rawPrompt = String(body.prompt || "").trim();
|
||||
const hasUserPrompt =
|
||||
rawPrompt.length > 0 && rawPrompt !== FALLBACK_REALTIME_PROMPT;
|
||||
const positivePrompt = hasUserPrompt
|
||||
? [
|
||||
`User prompt: ${rawPrompt}`,
|
||||
"Keep the object layout and perspective from the sketch.",
|
||||
"Keep the exact silhouette and part topology from the sketch (same count of legs/supports and same frame structure).",
|
||||
"The main object category must exactly match the user prompt.",
|
||||
"Do not swap object class (for example, chair must remain chair, never animal).",
|
||||
"Do not add bulky cushions, thick armrests, or boxy masses unless they clearly exist in the sketch.",
|
||||
"Render as a full-color photorealistic product image with realistic material, lighting, and texture.",
|
||||
"No sketch style, no line art outlines, no monochrome drawing.",
|
||||
].join(" ")
|
||||
: FALLBACK_REALTIME_PROMPT;
|
||||
const negativePrompt =
|
||||
String(config.negativePrompt || "").trim() ||
|
||||
"改变构图, 改变主体位置, 多余人物, 多余肢体, 重复物体, 镜像翻转, 错误透视, 模糊轮廓, 文字水印, 大面积背景噪点";
|
||||
"改变构图, 改变主体位置, 多余人物, 多余肢体, 重复物体, 镜像翻转, 错误透视, 模糊轮廓, 文字水印, 大面积背景噪点, 线稿, 线描, 草图, 黑白素描, 轮廓线, line art, sketch, outline drawing, monochrome drawing, wrong object category, unrelated subject, animal when furniture requested, furniture when animal requested";
|
||||
let fallbackWarning = "";
|
||||
|
||||
const requestInference = async (modelId: string) => {
|
||||
const useFalRealtime =
|
||||
config.falRealtimeEnabled === true || process.env.FAL_REALTIME_ENABLED === "1";
|
||||
|
||||
if (hasUserPrompt && qualityProfile === "refine" && falApiKey && useFalRealtime) {
|
||||
try {
|
||||
const fal = await import("@fal-ai/serverless-client");
|
||||
fal.config({
|
||||
credentials: falApiKey,
|
||||
});
|
||||
|
||||
const falInput: Record<string, unknown> = {
|
||||
prompt: rawPrompt,
|
||||
negative_prompt: negativePrompt,
|
||||
image_url: imageUrl,
|
||||
image_size: body.imageSize === "square_hd" ? "square_hd" : "square",
|
||||
num_inference_steps: 6,
|
||||
schedule_mu: 2.3,
|
||||
output_feedback_strength: 0.95,
|
||||
sync_mode: true,
|
||||
};
|
||||
|
||||
const falResult: any = await fal.subscribe(falRealtimeModel, {
|
||||
input: falInput,
|
||||
logs: false,
|
||||
timeout: 90_000,
|
||||
});
|
||||
|
||||
const falImageUrl = extractFalImageUrl(falResult);
|
||||
if (falImageUrl) {
|
||||
console.log("[RealtimeFal] frame success", {
|
||||
userId: session.user.id,
|
||||
requestId: body.requestId || null,
|
||||
model: falRealtimeModel,
|
||||
promptLength: rawPrompt.length,
|
||||
qualityProfile,
|
||||
});
|
||||
return Response.json({
|
||||
ok: true,
|
||||
provider: "fal",
|
||||
model: falRealtimeModel,
|
||||
transport: "fal-http",
|
||||
imageUrl: falImageUrl,
|
||||
cost: 0,
|
||||
});
|
||||
}
|
||||
console.warn("[RealtimeFal] empty output, fallback to Runware", {
|
||||
userId: session.user.id,
|
||||
requestId: body.requestId || null,
|
||||
model: falRealtimeModel,
|
||||
});
|
||||
} catch (error) {
|
||||
fallbackWarning =
|
||||
error instanceof Error && /unauthorized|forbidden/i.test(error.message)
|
||||
? "FAL 实时模型鉴权失败,当前已回退为 Runware 草图模式;请在系统设置更新有效的 FAL API Key。"
|
||||
: "FAL 实时模型调用失败,当前已回退为 Runware 草图模式。";
|
||||
console.warn("[RealtimeFal] request failed, fallback to Runware", {
|
||||
userId: session.user.id,
|
||||
requestId: body.requestId || null,
|
||||
model: falRealtimeModel,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
type InferenceRequestResult = {
|
||||
taskUUID: string;
|
||||
modelId: string;
|
||||
runtimeStrength: number;
|
||||
runtimeCfgScale: number;
|
||||
runtimeSteps: number;
|
||||
effectiveStrictSketch: boolean;
|
||||
runtimeSeed?: number;
|
||||
usedMaskImage: boolean;
|
||||
controlGuideImage: string;
|
||||
controlNetModel: string;
|
||||
path: "seed-image" | "controlnet";
|
||||
transport: "ws" | "http";
|
||||
upstreamStatus: number;
|
||||
upstreamOk: boolean;
|
||||
upstreamJson: any;
|
||||
upstreamError: string;
|
||||
wsFallbackError: string;
|
||||
};
|
||||
|
||||
type RunwareTaskPayload = {
|
||||
taskType: string;
|
||||
taskUUID?: string;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
const sendRunwareTask = async (runwareTask: RunwareTaskPayload, timeoutMs = 90_000) => {
|
||||
let transport: "ws" | "http" = "http";
|
||||
let upstreamStatus = 0;
|
||||
let upstreamOk = false;
|
||||
let upstreamJson: any = null;
|
||||
let upstreamError = "";
|
||||
let wsFallbackError = "";
|
||||
|
||||
if (preferWebSocket) {
|
||||
try {
|
||||
const wsClient = getRunwareWebSocketClient(runwareApiKey);
|
||||
upstreamJson = await wsClient.sendTask(runwareTask, timeoutMs);
|
||||
upstreamError = parseRunwareError(upstreamJson);
|
||||
upstreamStatus = upstreamError ? 502 : 200;
|
||||
upstreamOk = !upstreamError;
|
||||
transport = "ws";
|
||||
} catch (error) {
|
||||
wsFallbackError = error instanceof Error ? error.message : "Runware WebSocket 调用失败";
|
||||
}
|
||||
}
|
||||
|
||||
if (!upstreamJson) {
|
||||
const upstreamResponse = await fetch("https://api.runware.ai/v1", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${runwareApiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json",
|
||||
},
|
||||
body: JSON.stringify([runwareTask]),
|
||||
cache: "no-store",
|
||||
});
|
||||
upstreamJson = await upstreamResponse.json().catch(() => null);
|
||||
upstreamError = parseRunwareError(upstreamJson);
|
||||
upstreamStatus = upstreamResponse.status;
|
||||
upstreamOk = upstreamResponse.ok && !upstreamError;
|
||||
transport = "http";
|
||||
}
|
||||
|
||||
return {
|
||||
transport,
|
||||
upstreamStatus,
|
||||
upstreamOk,
|
||||
upstreamJson,
|
||||
upstreamError,
|
||||
wsFallbackError,
|
||||
};
|
||||
};
|
||||
|
||||
const requestInference = async (
|
||||
modelId: string,
|
||||
useMaskImage: boolean,
|
||||
sourceImageUrl: string
|
||||
): Promise<InferenceRequestResult> => {
|
||||
const taskUUID = crypto.randomUUID();
|
||||
const runtimeStrength = strictSketch ? clamp(strength, 0.12, 0.55) : strength;
|
||||
const runtimeCfgScale = strictSketch ? clamp(cfgScale, 6, 14) : cfgScale;
|
||||
const runwareRequest = [
|
||||
{
|
||||
taskType: "imageInference",
|
||||
taskUUID,
|
||||
model: modelId,
|
||||
positivePrompt,
|
||||
negativePrompt,
|
||||
seedImage: imageUrl,
|
||||
width: size,
|
||||
height: size,
|
||||
strength: runtimeStrength,
|
||||
CFGScale: runtimeCfgScale,
|
||||
steps,
|
||||
includeCost: true,
|
||||
numberResults: 1,
|
||||
outputType: "dataURI",
|
||||
outputFormat: "JPG",
|
||||
...(typeof seed === "number" ? { seed } : {}),
|
||||
},
|
||||
];
|
||||
const usePromptStylize = hasUserPrompt;
|
||||
const previewMode = qualityProfile === "preview";
|
||||
const isStylizePass =
|
||||
usePromptStylize &&
|
||||
qualityProfile === "refine" &&
|
||||
Boolean(stylizeModel) &&
|
||||
modelId === stylizeModel;
|
||||
const relaxStructureForPrompt = usePromptStylize && !isStylizePass;
|
||||
const effectiveStrictSketch = strictSketch && !relaxStructureForPrompt;
|
||||
const profileStrength = usePromptStylize
|
||||
? previewMode
|
||||
? clamp(Math.max(strength, 0.58), 0.05, 1)
|
||||
: isStylizePass
|
||||
? clamp(Math.max(strength, 0.72), 0.05, 1)
|
||||
: clamp(Math.max(strength, 0.9), 0.05, 1)
|
||||
: previewMode
|
||||
? clamp(strength * 0.85, 0.05, 1)
|
||||
: strength;
|
||||
const profileCfgScale = usePromptStylize
|
||||
? previewMode
|
||||
? clamp(Math.max(5.5, Math.min(cfgScale, 8.5)), 1, 14)
|
||||
: isStylizePass
|
||||
? clamp(Math.max(6.5, Math.min(cfgScale, 8.5)), 1, 14)
|
||||
: clamp(Math.max(6, Math.min(cfgScale, 8.5)), 1, 14)
|
||||
: previewMode
|
||||
? clamp(cfgScale * 0.9, 1, 14)
|
||||
: cfgScale;
|
||||
const profileSteps = usePromptStylize
|
||||
? previewMode
|
||||
? clamp(Math.max(steps, 6), 1, 12)
|
||||
: isStylizePass
|
||||
? clamp(Math.max(steps, 8), 1, 10)
|
||||
: clamp(Math.max(steps, 12), 1, 12)
|
||||
: previewMode
|
||||
? clamp(Math.min(steps, 6), 1, 12)
|
||||
: clamp(Math.max(steps, 6), 1, 12);
|
||||
const runtimeStrength = effectiveStrictSketch
|
||||
? (() => {
|
||||
const minStrength = usePromptStylize && !previewMode ? 0.2 : 0.12;
|
||||
const maxStrength =
|
||||
isStylizePass ? 0.62 : usePromptStylize && !previewMode ? 0.72 : 0.55;
|
||||
return clamp(profileStrength, minStrength, maxStrength);
|
||||
})()
|
||||
: profileStrength;
|
||||
const runtimeCfgScale = effectiveStrictSketch
|
||||
? clamp(
|
||||
profileCfgScale,
|
||||
isStylizePass ? 6.5 : usePromptStylize && !previewMode ? 5 : 6,
|
||||
14
|
||||
)
|
||||
: profileCfgScale;
|
||||
const runtimeSeed = relaxStructureForPrompt ? undefined : seed;
|
||||
const isRunware106 = modelId.toLowerCase().startsWith("runware:106@");
|
||||
const inferenceSize = isRunware106 ? 1024 : size;
|
||||
const runtimeStrengthLimited = clamp(
|
||||
runtimeStrength,
|
||||
renderModeSettings.seedStrengthMin,
|
||||
renderModeSettings.seedStrengthMax
|
||||
);
|
||||
const runwareTask = {
|
||||
taskType: "imageInference",
|
||||
taskUUID,
|
||||
model: modelId,
|
||||
positivePrompt,
|
||||
negativePrompt,
|
||||
seedImage: sourceImageUrl,
|
||||
width: inferenceSize,
|
||||
height: inferenceSize,
|
||||
strength: runtimeStrengthLimited,
|
||||
CFGScale: runtimeCfgScale,
|
||||
steps: profileSteps,
|
||||
includeCost: true,
|
||||
numberResults: 1,
|
||||
outputType: "dataURI",
|
||||
outputFormat: "JPG",
|
||||
...(useMaskImage && maskImageUrl ? { maskImage: maskImageUrl } : {}),
|
||||
...(typeof runtimeSeed === "number" ? { seed: runtimeSeed } : {}),
|
||||
};
|
||||
|
||||
const upstreamResponse = await fetch("https://api.runware.ai/v1", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${runwareApiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json",
|
||||
},
|
||||
body: JSON.stringify(runwareRequest),
|
||||
cache: "no-store",
|
||||
});
|
||||
|
||||
const upstreamJson = await upstreamResponse.json().catch(() => null);
|
||||
const upstreamError = parseRunwareError(upstreamJson);
|
||||
const upstream = await sendRunwareTask(runwareTask);
|
||||
|
||||
return {
|
||||
taskUUID,
|
||||
modelId,
|
||||
runtimeStrength,
|
||||
runtimeStrength: runtimeStrengthLimited,
|
||||
runtimeCfgScale,
|
||||
upstreamResponse,
|
||||
upstreamJson,
|
||||
upstreamError,
|
||||
runtimeSteps: profileSteps,
|
||||
effectiveStrictSketch,
|
||||
runtimeSeed,
|
||||
usedMaskImage: useMaskImage && Boolean(maskImageUrl),
|
||||
controlGuideImage: "",
|
||||
controlNetModel: "",
|
||||
path: "seed-image" as const,
|
||||
...upstream,
|
||||
};
|
||||
};
|
||||
|
||||
let requestResult = await requestInference(model);
|
||||
const requestWithMaskFallback = async (
|
||||
modelId: string,
|
||||
useMaskImage: boolean,
|
||||
sourceImageUrl: string
|
||||
) => {
|
||||
let requestResult = await requestInference(modelId, useMaskImage, sourceImageUrl);
|
||||
let maskEnabled = useMaskImage;
|
||||
|
||||
if (
|
||||
maskEnabled &&
|
||||
shouldRetryWithoutMask(requestResult.upstreamError || "")
|
||||
) {
|
||||
requestResult = await requestInference(modelId, false, sourceImageUrl);
|
||||
maskEnabled = false;
|
||||
}
|
||||
|
||||
return { requestResult, maskEnabled };
|
||||
};
|
||||
|
||||
const requestControlNetGuide = async (sourceImageUrl: string) => {
|
||||
const taskUUID = crypto.randomUUID();
|
||||
const controlTask = {
|
||||
taskType: "imageControlNetPreProcess",
|
||||
taskUUID,
|
||||
inputImage: sourceImageUrl,
|
||||
preProcessorType: controlPreprocessor,
|
||||
};
|
||||
const upstream = await sendRunwareTask(controlTask, 45_000);
|
||||
return {
|
||||
taskUUID,
|
||||
guideImage: extractControlGuideImage(upstream.upstreamJson, taskUUID),
|
||||
...upstream,
|
||||
};
|
||||
};
|
||||
|
||||
const requestControlNetInference = async (
|
||||
modelId: string,
|
||||
guideImage: string,
|
||||
includeNegativePrompt: boolean
|
||||
): Promise<InferenceRequestResult> => {
|
||||
const taskUUID = crypto.randomUUID();
|
||||
const previewMode = qualityProfile === "preview";
|
||||
const profileSteps = previewMode
|
||||
? clamp(Math.max(steps, 6), 1, 12)
|
||||
: clamp(Math.max(steps, 10), 1, 12);
|
||||
const profileCfgScale = previewMode
|
||||
? clamp(Math.max(5, Math.min(cfgScale, 8)), 1, 14)
|
||||
: clamp(Math.max(6, Math.min(cfgScale, 9)), 1, 14);
|
||||
const controlWeight = previewMode
|
||||
? clamp(renderModeSettings.controlWeight - 0.06, 0.55, 1)
|
||||
: renderModeSettings.controlWeight;
|
||||
|
||||
const runwareTask: RunwareTaskPayload = {
|
||||
taskType: "imageInference",
|
||||
taskUUID,
|
||||
model: modelId,
|
||||
positivePrompt,
|
||||
width: size,
|
||||
height: size,
|
||||
CFGScale: profileCfgScale,
|
||||
steps: profileSteps,
|
||||
includeCost: true,
|
||||
numberResults: 1,
|
||||
outputType: "dataURI",
|
||||
outputFormat: "JPG",
|
||||
...(typeof seed === "number" ? { seed } : {}),
|
||||
...(includeNegativePrompt ? { negativePrompt } : {}),
|
||||
controlNet: [
|
||||
{
|
||||
model: controlModel,
|
||||
guideImage,
|
||||
weight: controlWeight,
|
||||
startStep: 0,
|
||||
endStep: profileSteps,
|
||||
controlMode: "balanced",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const upstream = await sendRunwareTask(runwareTask, 120_000);
|
||||
return {
|
||||
taskUUID,
|
||||
modelId,
|
||||
runtimeStrength: 0,
|
||||
runtimeCfgScale: profileCfgScale,
|
||||
runtimeSteps: profileSteps,
|
||||
effectiveStrictSketch: strictSketch,
|
||||
runtimeSeed: typeof seed === "number" ? seed : undefined,
|
||||
usedMaskImage: false,
|
||||
controlGuideImage: guideImage,
|
||||
controlNetModel: controlModel,
|
||||
path: "controlnet" as const,
|
||||
...upstream,
|
||||
};
|
||||
};
|
||||
|
||||
const runTwoPassStylize = false;
|
||||
let maskEnabled = Boolean(maskImageUrl);
|
||||
let requestResult: InferenceRequestResult;
|
||||
let pathLabel: "seed-image" | "controlnet" = "seed-image";
|
||||
let controlGuideUsed = "";
|
||||
let controlPreprocessUsed = false;
|
||||
let controlGuideFallback = false;
|
||||
|
||||
const shouldUseControlNetPath =
|
||||
hasUserPrompt && qualityProfile === "refine" && strictSketch;
|
||||
|
||||
if (shouldUseControlNetPath) {
|
||||
const guideResult = await requestControlNetGuide(imageUrl);
|
||||
const preprocessOk = guideResult.upstreamOk && !guideResult.upstreamError;
|
||||
const guideImage = preprocessOk && guideResult.guideImage ? guideResult.guideImage : imageUrl;
|
||||
controlGuideUsed = guideImage;
|
||||
controlPreprocessUsed = preprocessOk;
|
||||
controlGuideFallback = !preprocessOk || !guideResult.guideImage;
|
||||
|
||||
if (!preprocessOk && (guideResult.upstreamError || "").trim()) {
|
||||
fallbackWarning = "ControlNet 预处理失败,已回退使用原始线稿引导。";
|
||||
}
|
||||
|
||||
let controlResult = await requestControlNetInference(model, guideImage, true);
|
||||
if (shouldRetryWithoutNegativePrompt(controlResult.upstreamError || "")) {
|
||||
controlResult = await requestControlNetInference(model, guideImage, false);
|
||||
}
|
||||
|
||||
if (controlResult.upstreamOk && !controlResult.upstreamError) {
|
||||
requestResult = controlResult;
|
||||
maskEnabled = false;
|
||||
pathLabel = "controlnet";
|
||||
} else if (shouldRetryWithoutControlNet(controlResult.upstreamError || "")) {
|
||||
fallbackWarning = "当前模型不支持 ControlNet,已回退到 seedImage 模式。";
|
||||
const fallbackPass = await requestWithMaskFallback(stylizeModel || model, maskEnabled, imageUrl);
|
||||
requestResult = fallbackPass.requestResult;
|
||||
maskEnabled = fallbackPass.maskEnabled;
|
||||
pathLabel = "seed-image";
|
||||
} else {
|
||||
requestResult = controlResult;
|
||||
maskEnabled = false;
|
||||
pathLabel = "controlnet";
|
||||
}
|
||||
} else if (runTwoPassStylize) {
|
||||
const basePass = await requestWithMaskFallback(model, maskEnabled, imageUrl);
|
||||
requestResult = basePass.requestResult;
|
||||
maskEnabled = basePass.maskEnabled;
|
||||
|
||||
if (requestResult.upstreamOk && !requestResult.upstreamError) {
|
||||
const baseImage = extractRunwareOutputImage(
|
||||
requestResult.upstreamJson,
|
||||
requestResult.taskUUID
|
||||
).outputImage;
|
||||
if (baseImage) {
|
||||
console.log("[RealtimeRunware] two-pass base success", {
|
||||
userId: session.user.id,
|
||||
requestId: body.requestId || null,
|
||||
baseModel: model,
|
||||
stylizeModel,
|
||||
baseImageType: baseImage.startsWith("data:") ? "dataURI" : "url",
|
||||
});
|
||||
const stylizePass = await requestWithMaskFallback(stylizeModel, false, baseImage);
|
||||
requestResult = stylizePass.requestResult;
|
||||
maskEnabled = stylizePass.maskEnabled;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const initialModel =
|
||||
hasUserPrompt && qualityProfile === "refine" ? stylizeModel || model : model;
|
||||
const initialPass = await requestWithMaskFallback(initialModel, maskEnabled, imageUrl);
|
||||
requestResult = initialPass.requestResult;
|
||||
maskEnabled = initialPass.maskEnabled;
|
||||
pathLabel = "seed-image";
|
||||
}
|
||||
|
||||
if (pathLabel === "controlnet" && (!requestResult.upstreamOk || requestResult.upstreamError)) {
|
||||
fallbackWarning = "ControlNet 推理失败,已自动回退为 seedImage 模式。";
|
||||
const fallbackPass = await requestWithMaskFallback(stylizeModel || model, maskEnabled, imageUrl);
|
||||
requestResult = fallbackPass.requestResult;
|
||||
maskEnabled = fallbackPass.maskEnabled;
|
||||
pathLabel = "seed-image";
|
||||
}
|
||||
|
||||
const shouldRetryWithSeedImageModel =
|
||||
(requestResult.upstreamError || "")
|
||||
.toLowerCase()
|
||||
@ -209,36 +791,34 @@ export async function POST(request: Request) {
|
||||
model !== fallbackSeedImageModel;
|
||||
|
||||
if (shouldRetryWithSeedImageModel) {
|
||||
requestResult = await requestInference(fallbackSeedImageModel);
|
||||
fallbackWarning =
|
||||
"当前模型不支持草图引导,已自动切换到兼容的实时草图模型。";
|
||||
const fallbackResult = await requestWithMaskFallback(
|
||||
fallbackSeedImageModel,
|
||||
maskEnabled,
|
||||
imageUrl
|
||||
);
|
||||
requestResult = fallbackResult.requestResult;
|
||||
maskEnabled = fallbackResult.maskEnabled;
|
||||
}
|
||||
|
||||
if (!requestResult.upstreamResponse.ok || requestResult.upstreamError) {
|
||||
if (!requestResult.upstreamOk || requestResult.upstreamError) {
|
||||
return Response.json(
|
||||
{
|
||||
code: "RUNWARE_REQUEST_FAILED",
|
||||
message:
|
||||
requestResult.upstreamError ||
|
||||
`Runware 请求失败(${requestResult.upstreamResponse.status})`,
|
||||
requestResult.wsFallbackError ||
|
||||
`Runware 请求失败(${requestResult.upstreamStatus})`,
|
||||
},
|
||||
{ status: 502 }
|
||||
);
|
||||
}
|
||||
|
||||
const resultList = Array.isArray(requestResult.upstreamJson)
|
||||
? requestResult.upstreamJson
|
||||
: Array.isArray(requestResult.upstreamJson?.data)
|
||||
? requestResult.upstreamJson.data
|
||||
: [];
|
||||
const imageResult =
|
||||
resultList.find(
|
||||
(item: any) =>
|
||||
item?.taskUUID === requestResult.taskUUID && (item?.imageDataURI || item?.imageURL)
|
||||
) ||
|
||||
resultList.find((item: any) => item?.imageDataURI || item?.imageURL);
|
||||
|
||||
const imageDataURI = String(imageResult?.imageDataURI || "").trim();
|
||||
const imageURL = String(imageResult?.imageURL || "").trim();
|
||||
const outputImage = imageDataURI || imageURL;
|
||||
const { imageResult, imageDataURI, imageURL, outputImage } = extractRunwareOutputImage(
|
||||
requestResult.upstreamJson,
|
||||
requestResult.taskUUID
|
||||
);
|
||||
|
||||
if (!outputImage) {
|
||||
return Response.json(
|
||||
@ -253,20 +833,44 @@ export async function POST(request: Request) {
|
||||
requestId: body.requestId || null,
|
||||
taskUUID: requestResult.taskUUID,
|
||||
model: requestResult.modelId,
|
||||
seed: typeof seed === "number" ? seed : null,
|
||||
steps,
|
||||
transport: requestResult.transport,
|
||||
seed: typeof requestResult.runtimeSeed === "number" ? requestResult.runtimeSeed : null,
|
||||
hasUserPrompt,
|
||||
promptLength: rawPrompt.length,
|
||||
promptPreview: hasUserPrompt ? rawPrompt.slice(0, 32) : null,
|
||||
qualityProfile,
|
||||
steps: requestResult.runtimeSteps,
|
||||
cfgScale: requestResult.runtimeCfgScale,
|
||||
strength: requestResult.runtimeStrength,
|
||||
strictSketchApplied: requestResult.effectiveStrictSketch,
|
||||
maskEnabled,
|
||||
roi: body.roi || null,
|
||||
cost,
|
||||
renderMode,
|
||||
path: pathLabel,
|
||||
controlNetModel: pathLabel === "controlnet" ? requestResult.controlNetModel : null,
|
||||
controlGuideType:
|
||||
pathLabel === "controlnet" && controlGuideUsed
|
||||
? controlGuideUsed.startsWith("data:")
|
||||
? "dataURI"
|
||||
: "url"
|
||||
: null,
|
||||
controlGuideFallback,
|
||||
controlPreprocessUsed,
|
||||
twoPassStylize: runTwoPassStylize,
|
||||
fallbackWarning: fallbackWarning || null,
|
||||
});
|
||||
|
||||
return Response.json({
|
||||
ok: true,
|
||||
provider: "runware",
|
||||
model: requestResult.modelId,
|
||||
transport: requestResult.transport,
|
||||
taskUUID: requestResult.taskUUID,
|
||||
path: pathLabel,
|
||||
imageDataURI: imageDataURI || undefined,
|
||||
imageUrl: imageURL || undefined,
|
||||
cost,
|
||||
warning: fallbackWarning || undefined,
|
||||
});
|
||||
}
|
||||
|
||||
@ -89,6 +89,7 @@ async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/pn
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const AOS_MAX_RESOLUTION = 4096;
|
||||
const AOS_CREATE_TIMEOUT_MS = 120_000;
|
||||
const AOS_POLL_TIMEOUT_MS = 60_000;
|
||||
const AOS_DOWNLOAD_TIMEOUT_MS = 60_000;
|
||||
@ -106,6 +107,71 @@ const withTimeout = async <T,>(
|
||||
}
|
||||
};
|
||||
|
||||
function getAosFriendlyErrorMessage(detail: string): string | null {
|
||||
const resolutionMatch = detail.match(/resolution excess,\s*width=(\d+),\s*height=(\d+)/i);
|
||||
if (resolutionMatch) {
|
||||
const [, width, height] = resolutionMatch;
|
||||
return `图片分辨率超限(${width}x${height}),AOS要求宽高都不超过 ${AOS_MAX_RESOLUTION}px。请缩小后重试。`;
|
||||
}
|
||||
|
||||
if (/invalid file/i.test(detail)) {
|
||||
return "图片文件无法识别,请尝试转换为 JPG/PNG 后重试。";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function prepareImageForAos(
|
||||
imageBuffer: Buffer
|
||||
): Promise<{ buffer: Buffer; mimeType: string; filename: string }> {
|
||||
try {
|
||||
const sharp = await import("sharp");
|
||||
const image = sharp.default(imageBuffer);
|
||||
const metadata = await image.metadata();
|
||||
const width = metadata.width;
|
||||
const height = metadata.height;
|
||||
const format = (metadata.format || "png").toLowerCase();
|
||||
|
||||
const ext = format === "jpeg" ? "jpg" : format;
|
||||
const filenameExt =
|
||||
ext === "jpg" || ext === "png" || ext === "webp" || ext === "tiff" ? ext : "png";
|
||||
const mimeType =
|
||||
ext === "jpg" ? "image/jpeg" :
|
||||
ext === "webp" ? "image/webp" :
|
||||
ext === "tiff" ? "image/tiff" :
|
||||
"image/png";
|
||||
|
||||
if (!width || !height || (width <= AOS_MAX_RESOLUTION && height <= AOS_MAX_RESOLUTION)) {
|
||||
return {
|
||||
buffer: imageBuffer,
|
||||
mimeType,
|
||||
filename: `image.${filenameExt}`,
|
||||
};
|
||||
}
|
||||
|
||||
const resizedBuffer = await image
|
||||
.resize(AOS_MAX_RESOLUTION, AOS_MAX_RESOLUTION, {
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.toBuffer();
|
||||
const resizedMeta = await sharp.default(resizedBuffer).metadata();
|
||||
|
||||
console.warn(
|
||||
`[AOS RMBG] 输入图片分辨率超限,已自动缩放: ${width}x${height} -> ${resizedMeta.width}x${resizedMeta.height}`
|
||||
);
|
||||
|
||||
return {
|
||||
buffer: resizedBuffer,
|
||||
mimeType,
|
||||
filename: `image.${filenameExt}`,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("[AOS RMBG] 图片预处理失败,继续使用原图:", error);
|
||||
return { buffer: imageBuffer, mimeType: "image/png", filename: "image.png" };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用 RMBG-2.0 API 进行抠图
|
||||
*/
|
||||
@ -254,9 +320,10 @@ async function processWithAosSegmentation(
|
||||
const headers = { "X-API-KEY": config.apiKey };
|
||||
|
||||
try {
|
||||
const normalizedImage = await prepareImageForAos(imageBuffer);
|
||||
const createForm = new FormData();
|
||||
const imageBlob = new Blob([new Uint8Array(imageBuffer)], { type: "image/png" });
|
||||
createForm.append("image_file", imageBlob, "image.png");
|
||||
const imageBlob = new Blob([new Uint8Array(normalizedImage.buffer)], { type: normalizedImage.mimeType });
|
||||
createForm.append("image_file", imageBlob, normalizedImage.filename);
|
||||
createForm.append("sync", "0");
|
||||
createForm.append("return_type", "1");
|
||||
createForm.append("output_type", "2");
|
||||
@ -527,9 +594,11 @@ export async function POST(request: NextRequest) {
|
||||
});
|
||||
|
||||
if (!result.success || !result.imageBuffer) {
|
||||
const detail = result.error || "AOS处理失败";
|
||||
const friendlyMessage = getAosFriendlyErrorMessage(detail);
|
||||
return NextResponse.json(
|
||||
{ error: "抠图处理失败", detail: result.error },
|
||||
{ status: 500 }
|
||||
{ error: "抠图处理失败", detail: friendlyMessage || detail },
|
||||
{ status: friendlyMessage ? 400 : 500 }
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -60,7 +60,6 @@ import { CanvasWorkspace } from "@/components/canvas";
|
||||
import { CanvasChatPanel } from "@/components/canvas/CanvasChatPanel";
|
||||
import { CanvasFileListPanel } from "@/components/canvas/CanvasFileListPanel";
|
||||
import { ObjectPickerOverlay } from "@/components/canvas/ObjectPickerOverlay";
|
||||
import { RealtimeCanvasDialog } from "@/components/canvas/RealtimeCanvasDialog";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { cn, fetcher } from "@/lib/utils";
|
||||
import { MessageSquare, FolderOpen, Grid2X2 } from "lucide-react";
|
||||
@ -124,7 +123,6 @@ export default function CanvasEditorPage({ params }: PageProps) {
|
||||
const [isPanelOpen, setIsPanelOpen] = useState(true);
|
||||
// 面板模式: 'chat' 聊天面板, 'fileList' 文件列表
|
||||
const [panelMode, setPanelMode] = useState<'chat' | 'fileList'>('chat');
|
||||
const [isRealtimeCanvasOpen, setIsRealtimeCanvasOpen] = useState(false);
|
||||
|
||||
const { data: siteSettingsData } = useSWR<{
|
||||
siteName?: string;
|
||||
@ -359,38 +357,6 @@ export default function CanvasEditorPage({ params }: PageProps) {
|
||||
});
|
||||
};
|
||||
|
||||
const handleApplyRealtimeResult = useCallback(
|
||||
({ imageUrl, prompt: imagePrompt }: { imageUrl: string; prompt: string }) => {
|
||||
const viewportX = viewport.x || 0;
|
||||
const viewportY = viewport.y || 0;
|
||||
const viewportZoom = viewport.zoom || 1;
|
||||
const stageWidth = typeof window !== "undefined" ? window.innerWidth : 1280;
|
||||
const stageHeight = typeof window !== "undefined" ? window.innerHeight : 720;
|
||||
const targetWidth = 360;
|
||||
const targetHeight = 360;
|
||||
|
||||
const x = (stageWidth * 0.45 - viewportX) / viewportZoom - targetWidth / 2;
|
||||
const y = (stageHeight * 0.45 - viewportY) / viewportZoom - targetHeight / 2;
|
||||
|
||||
addNode({
|
||||
type: "image",
|
||||
position: { x, y },
|
||||
size: { width: targetWidth, height: targetHeight },
|
||||
data: {
|
||||
src: imageUrl,
|
||||
status: "completed",
|
||||
title: "实时画布结果",
|
||||
prompt: imagePrompt,
|
||||
model: "fal-ai/flux-2/klein/realtime",
|
||||
},
|
||||
connections: [],
|
||||
});
|
||||
|
||||
toast.success("实时画布结果已添加");
|
||||
},
|
||||
[addNode, viewport.x, viewport.y, viewport.zoom]
|
||||
);
|
||||
|
||||
// 处理图片操作(通过事件分发到选中图片节点执行)
|
||||
const handleAction = (action: string) => {
|
||||
if (!selectedImageNode) {
|
||||
@ -762,7 +728,7 @@ export default function CanvasEditorPage({ params }: PageProps) {
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={() => setIsRealtimeCanvasOpen(true)}
|
||||
onClick={() => router.push(`/canvas/${id}/realtime`)}
|
||||
className="rounded-xl h-9 w-9"
|
||||
>
|
||||
<PenTool className="h-4 w-4" />
|
||||
@ -825,13 +791,6 @@ export default function CanvasEditorPage({ params }: PageProps) {
|
||||
|
||||
{/* 6. 物品拾取模式覆盖层 */}
|
||||
<ObjectPickerOverlay />
|
||||
|
||||
<RealtimeCanvasDialog
|
||||
open={isRealtimeCanvasOpen}
|
||||
onOpenChange={setIsRealtimeCanvasOpen}
|
||||
initialImageUrl={selectedImageNode ? ((selectedImageNode.data as any)?.src as string) : null}
|
||||
onApplyResult={handleApplyRealtimeResult}
|
||||
/>
|
||||
</div>
|
||||
</TooltipProvider>
|
||||
);
|
||||
|
||||
82
app/canvas/[id]/realtime/page.tsx
Normal file
82
app/canvas/[id]/realtime/page.tsx
Normal file
@ -0,0 +1,82 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback } from "react";
|
||||
import { useParams, useRouter } from "next/navigation";
|
||||
import { toast } from "sonner";
|
||||
import { RealtimeCanvasDialog } from "@/components/canvas/RealtimeCanvasDialog";
|
||||
import { canvasService } from "@/services/canvas";
|
||||
import { useCanvasStore, canvasSelectors } from "@/store/canvas";
|
||||
|
||||
export default function CanvasRealtimePage() {
|
||||
const router = useRouter();
|
||||
const routeParams = useParams();
|
||||
const id = String(routeParams?.id || "");
|
||||
|
||||
const viewport = useCanvasStore(canvasSelectors.viewport);
|
||||
const selectedNodeIds = useCanvasStore(canvasSelectors.selectedNodeIds);
|
||||
const nodes = useCanvasStore(canvasSelectors.nodes);
|
||||
const addNode = useCanvasStore((state) => state.addNode);
|
||||
const getCanvasState = useCanvasStore((state) => state.getCanvasState);
|
||||
|
||||
const selectedImageNode =
|
||||
selectedNodeIds.length === 1
|
||||
? nodes.find((node) => node.id === selectedNodeIds[0] && node.type === "image")
|
||||
: null;
|
||||
|
||||
const handleOpenChange = useCallback(
|
||||
(nextOpen: boolean) => {
|
||||
if (!nextOpen) {
|
||||
router.push(`/canvas/${id}`);
|
||||
}
|
||||
},
|
||||
[id, router]
|
||||
);
|
||||
|
||||
const handleApplyResult = useCallback(
|
||||
async ({ imageUrl, prompt }: { imageUrl: string; prompt: string }) => {
|
||||
const viewportX = viewport.x || 0;
|
||||
const viewportY = viewport.y || 0;
|
||||
const viewportZoom = viewport.zoom || 1;
|
||||
const stageWidth = typeof window !== "undefined" ? window.innerWidth : 1280;
|
||||
const stageHeight = typeof window !== "undefined" ? window.innerHeight : 720;
|
||||
const targetWidth = 360;
|
||||
const targetHeight = 360;
|
||||
|
||||
const x = (stageWidth * 0.45 - viewportX) / viewportZoom - targetWidth / 2;
|
||||
const y = (stageHeight * 0.45 - viewportY) / viewportZoom - targetHeight / 2;
|
||||
|
||||
addNode({
|
||||
type: "image",
|
||||
position: { x, y },
|
||||
size: { width: targetWidth, height: targetHeight },
|
||||
data: {
|
||||
src: imageUrl,
|
||||
status: "completed",
|
||||
title: "实时画布结果",
|
||||
prompt,
|
||||
model: "fal-ai/flux-2/klein/realtime",
|
||||
},
|
||||
connections: [],
|
||||
});
|
||||
|
||||
try {
|
||||
await canvasService.saveCanvasState(id, getCanvasState());
|
||||
} catch (error) {
|
||||
console.error("[RealtimeCanvasPage] 保存结果失败:", error);
|
||||
}
|
||||
|
||||
toast.success("实时画布结果已添加");
|
||||
},
|
||||
[addNode, getCanvasState, id, viewport.x, viewport.y, viewport.zoom]
|
||||
);
|
||||
|
||||
return (
|
||||
<RealtimeCanvasDialog
|
||||
standalone
|
||||
open
|
||||
onOpenChange={handleOpenChange}
|
||||
initialImageUrl={selectedImageNode ? ((selectedImageNode.data as any)?.src as string) : null}
|
||||
onApplyResult={handleApplyResult}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@ -35,6 +35,7 @@ import {
|
||||
Undo2,
|
||||
Redo2,
|
||||
Send,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
|
||||
@ -68,6 +69,7 @@ type RealtimeCanvasDialogProps = {
|
||||
onOpenChange: (open: boolean) => void;
|
||||
initialImageUrl?: string | null;
|
||||
onApplyResult: (payload: { imageUrl: string; prompt: string }) => void;
|
||||
standalone?: boolean;
|
||||
};
|
||||
|
||||
const FALLBACK_PROMPT =
|
||||
@ -122,7 +124,7 @@ async function canvasToJpegDataUri(
|
||||
options?: {
|
||||
autoCrop?: boolean;
|
||||
}
|
||||
) {
|
||||
): Promise<{ dataUri: string; width: number; height: number }> {
|
||||
type CropBox = {
|
||||
x: number;
|
||||
y: number;
|
||||
@ -455,12 +457,114 @@ async function canvasToJpegDataUri(
|
||||
throw new Error("导出图片失败");
|
||||
}
|
||||
|
||||
return await new Promise<string>((resolve, reject) => {
|
||||
const dataUri = await new Promise<string>((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => resolve(String(reader.result || ""));
|
||||
reader.onerror = () => reject(new Error("读取导出图片失败"));
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
|
||||
return {
|
||||
dataUri,
|
||||
width: exportWidth,
|
||||
height: exportHeight,
|
||||
};
|
||||
}
|
||||
|
||||
type FlushMode = "preview" | "refine";
|
||||
|
||||
type NormalizedRegion = {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
|
||||
function clamp01(value: number) {
|
||||
return Math.min(1, Math.max(0, value));
|
||||
}
|
||||
|
||||
function mergeRegion(
|
||||
current: NormalizedRegion | null,
|
||||
next: NormalizedRegion
|
||||
): NormalizedRegion {
|
||||
if (!current) return next;
|
||||
const x1 = Math.min(current.x, next.x);
|
||||
const y1 = Math.min(current.y, next.y);
|
||||
const x2 = Math.max(current.x + current.width, next.x + next.width);
|
||||
const y2 = Math.max(current.y + current.height, next.y + next.height);
|
||||
return {
|
||||
x: clamp01(x1),
|
||||
y: clamp01(y1),
|
||||
width: clamp01(x2 - x1),
|
||||
height: clamp01(y2 - y1),
|
||||
};
|
||||
}
|
||||
|
||||
function toNormalizedRegion(
|
||||
canvas: HTMLCanvasElement,
|
||||
rect: {
|
||||
minX: number;
|
||||
minY: number;
|
||||
maxX: number;
|
||||
maxY: number;
|
||||
}
|
||||
): NormalizedRegion | null {
|
||||
const bounds = canvas.getBoundingClientRect();
|
||||
const width = Math.max(1, bounds.width);
|
||||
const height = Math.max(1, bounds.height);
|
||||
const x = clamp01(rect.minX / width);
|
||||
const y = clamp01(rect.minY / height);
|
||||
const right = clamp01(rect.maxX / width);
|
||||
const bottom = clamp01(rect.maxY / height);
|
||||
const regionWidth = right - x;
|
||||
const regionHeight = bottom - y;
|
||||
if (regionWidth <= 0 || regionHeight <= 0) return null;
|
||||
return { x, y, width: regionWidth, height: regionHeight };
|
||||
}
|
||||
|
||||
async function createMaskDataUri(
|
||||
width: number,
|
||||
height: number,
|
||||
region: NormalizedRegion
|
||||
) {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, width);
|
||||
canvas.height = Math.max(1, height);
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) {
|
||||
throw new Error("无法初始化遮罩画布");
|
||||
}
|
||||
|
||||
const basePad = Math.max(12, Math.floor(Math.max(width, height) * 0.02));
|
||||
let x = Math.floor(region.x * width) - basePad;
|
||||
let y = Math.floor(region.y * height) - basePad;
|
||||
let regionWidth = Math.ceil(region.width * width) + basePad * 2;
|
||||
let regionHeight = Math.ceil(region.height * height) + basePad * 2;
|
||||
|
||||
const minPatch = Math.max(40, Math.floor(Math.max(width, height) * 0.1));
|
||||
if (regionWidth < minPatch) {
|
||||
const delta = Math.ceil((minPatch - regionWidth) / 2);
|
||||
x -= delta;
|
||||
regionWidth += delta * 2;
|
||||
}
|
||||
if (regionHeight < minPatch) {
|
||||
const delta = Math.ceil((minPatch - regionHeight) / 2);
|
||||
y -= delta;
|
||||
regionHeight += delta * 2;
|
||||
}
|
||||
|
||||
x = Math.max(0, x);
|
||||
y = Math.max(0, y);
|
||||
const right = Math.min(width, x + regionWidth);
|
||||
const bottom = Math.min(height, y + regionHeight);
|
||||
|
||||
ctx.fillStyle = "#000000";
|
||||
ctx.fillRect(0, 0, width, height);
|
||||
ctx.fillStyle = "#ffffff";
|
||||
ctx.fillRect(x, y, Math.max(1, right - x), Math.max(1, bottom - y));
|
||||
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export function RealtimeCanvasDialog({
|
||||
@ -468,6 +572,7 @@ export function RealtimeCanvasDialog({
|
||||
onOpenChange,
|
||||
initialImageUrl,
|
||||
onApplyResult,
|
||||
standalone = false,
|
||||
}: RealtimeCanvasDialogProps) {
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const previewCanvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
@ -479,8 +584,15 @@ export function RealtimeCanvasDialog({
|
||||
const pendingFlushTimerRef = useRef<number | null>(null);
|
||||
const flushInFlightRef = useRef(false);
|
||||
const dirtyRef = useRef(false);
|
||||
const dirtyRegionRef = useRef<NormalizedRegion | null>(null);
|
||||
const draftVersionRef = useRef(0);
|
||||
const pendingFlushModeRef = useRef<FlushMode | null>(null);
|
||||
const pendingFlushForceRef = useRef(false);
|
||||
const latencyEmaRef = useRef<number | null>(null);
|
||||
const baseThrottleMsRef = useRef<number>(defaultRealtimeConfig.transport.throttleInterval);
|
||||
const pumpFlushRef = useRef<(() => void) | null>(null);
|
||||
const warningMessageRef = useRef("");
|
||||
const seqRef = useRef(0);
|
||||
const requestSeqMapRef = useRef<Map<string, number>>(new Map());
|
||||
const requestStartMapRef = useRef<Map<string, number>>(new Map());
|
||||
const lastRenderedSeqRef = useRef(0);
|
||||
const metricsQueueRef = useRef<any[]>([]);
|
||||
@ -574,8 +686,42 @@ export function RealtimeCanvasDialog({
|
||||
setCanRedo(false);
|
||||
}, []);
|
||||
|
||||
const markCanvasDirty = useCallback((region?: NormalizedRegion | null) => {
|
||||
dirtyRef.current = true;
|
||||
draftVersionRef.current += 1;
|
||||
if (!region) {
|
||||
dirtyRegionRef.current = { x: 0, y: 0, width: 1, height: 1 };
|
||||
return;
|
||||
}
|
||||
dirtyRegionRef.current = mergeRegion(dirtyRegionRef.current, region);
|
||||
}, []);
|
||||
|
||||
const markCanvasDirtyRect = useCallback(
|
||||
(rect: { minX: number; minY: number; maxX: number; maxY: number }) => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) return;
|
||||
const region = toNormalizedRegion(canvas, rect);
|
||||
if (!region) return;
|
||||
markCanvasDirty(region);
|
||||
},
|
||||
[markCanvasDirty]
|
||||
);
|
||||
|
||||
const markCanvasDirtyStroke = useCallback(
|
||||
(from: { x: number; y: number }, to: { x: number; y: number }, lineWidth: number) => {
|
||||
const pad = Math.max(6, Math.ceil(lineWidth * 1.2));
|
||||
markCanvasDirtyRect({
|
||||
minX: Math.min(from.x, to.x) - pad,
|
||||
minY: Math.min(from.y, to.y) - pad,
|
||||
maxX: Math.max(from.x, to.x) + pad,
|
||||
maxY: Math.max(from.y, to.y) + pad,
|
||||
});
|
||||
},
|
||||
[markCanvasDirtyRect]
|
||||
);
|
||||
|
||||
const flushFrame = useCallback(
|
||||
async (force = false) => {
|
||||
async (mode: FlushMode, force = false) => {
|
||||
if (!open) return;
|
||||
if (!connectionRef.current) return;
|
||||
if (flushInFlightRef.current) return;
|
||||
@ -588,20 +734,37 @@ export function RealtimeCanvasDialog({
|
||||
const seq = seqRef.current + 1;
|
||||
seqRef.current = seq;
|
||||
const requestId = `${sessionIdRef.current}-${seq}`;
|
||||
requestSeqMapRef.current.set(requestId, seq);
|
||||
requestStartMapRef.current.set(requestId, Date.now());
|
||||
const draftVersionAtSend = draftVersionRef.current;
|
||||
const dirtyRegionAtSend = dirtyRegionRef.current
|
||||
? { ...dirtyRegionRef.current }
|
||||
: null;
|
||||
|
||||
try {
|
||||
const imageDataUri = await canvasToJpegDataUri(
|
||||
const exportedImage = await canvasToJpegDataUri(
|
||||
canvas,
|
||||
config.image.maxInputResolution,
|
||||
config.image.jpegQuality,
|
||||
canvasBackgroundRef.current,
|
||||
{ autoCrop: true }
|
||||
{ autoCrop: false }
|
||||
);
|
||||
const imageDataUri = exportedImage.dataUri;
|
||||
const shouldUseMask =
|
||||
Boolean(dirtyRegionAtSend) &&
|
||||
mode !== "refine" &&
|
||||
(dirtyRegionAtSend?.width || 0) * (dirtyRegionAtSend?.height || 0) < 0.92;
|
||||
const maskImageUrl =
|
||||
shouldUseMask && dirtyRegionAtSend
|
||||
? await createMaskDataUri(
|
||||
exportedImage.width,
|
||||
exportedImage.height,
|
||||
dirtyRegionAtSend
|
||||
)
|
||||
: undefined;
|
||||
|
||||
const activeConnection = connectionRef.current;
|
||||
if (!activeConnection) {
|
||||
requestStartMapRef.current.delete(requestId);
|
||||
setStatus("connecting");
|
||||
setStatusMessage("连接重建中,正在重试...");
|
||||
return;
|
||||
@ -623,6 +786,9 @@ export function RealtimeCanvasDialog({
|
||||
steps: config.model.defaults.steps,
|
||||
cfgScale: config.model.defaults.cfgScale,
|
||||
strength: config.model.defaults.strength,
|
||||
qualityProfile: mode,
|
||||
maskImageUrl,
|
||||
roi: dirtyRegionAtSend || undefined,
|
||||
}),
|
||||
});
|
||||
|
||||
@ -630,6 +796,8 @@ export function RealtimeCanvasDialog({
|
||||
imageUrl?: string;
|
||||
imageDataURI?: string;
|
||||
cost?: number;
|
||||
transport?: string;
|
||||
warning?: string;
|
||||
message?: string;
|
||||
error?: string;
|
||||
};
|
||||
@ -646,7 +814,6 @@ export function RealtimeCanvasDialog({
|
||||
const startedAt = requestStartMapRef.current.get(requestId) || Date.now();
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
requestStartMapRef.current.delete(requestId);
|
||||
requestSeqMapRef.current.delete(requestId);
|
||||
|
||||
if (seq > 0 && seq < lastRenderedSeqRef.current) {
|
||||
return;
|
||||
@ -655,8 +822,29 @@ export function RealtimeCanvasDialog({
|
||||
lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq);
|
||||
setPreviewUrl(nextPreviewUrl);
|
||||
setStatus("live");
|
||||
setStatusMessage("实时渲染中");
|
||||
dirtyRef.current = false;
|
||||
setStatusMessage(
|
||||
payload.transport === "ws" ? "实时渲染中(WS)" : "实时渲染中(HTTP)"
|
||||
);
|
||||
const upstreamWarning = String(payload.warning || "").trim();
|
||||
if (upstreamWarning && upstreamWarning !== warningMessageRef.current) {
|
||||
warningMessageRef.current = upstreamWarning;
|
||||
toast.warning(upstreamWarning);
|
||||
}
|
||||
if (draftVersionAtSend === draftVersionRef.current) {
|
||||
dirtyRef.current = false;
|
||||
dirtyRegionRef.current = null;
|
||||
}
|
||||
|
||||
const previousLatency = latencyEmaRef.current;
|
||||
const nextLatency =
|
||||
previousLatency === null ? latencyMs : previousLatency * 0.72 + latencyMs * 0.28;
|
||||
latencyEmaRef.current = nextLatency;
|
||||
const computedThrottle = Math.round(
|
||||
Math.max(baseThrottleMsRef.current, Math.min(520, nextLatency * 0.45))
|
||||
);
|
||||
setAdaptiveThrottleMs((prev) =>
|
||||
Math.abs(prev - computedThrottle) >= 8 ? computedThrottle : prev
|
||||
);
|
||||
|
||||
pushMetric({
|
||||
name: "frame_send",
|
||||
@ -669,41 +857,81 @@ export function RealtimeCanvasDialog({
|
||||
seq,
|
||||
latencyMs,
|
||||
queueDepth: Number(payload.cost || 0),
|
||||
qualityProfile: mode,
|
||||
hasMask: Boolean(maskImageUrl),
|
||||
transport: payload.transport || "unknown",
|
||||
});
|
||||
} catch (error) {
|
||||
requestStartMapRef.current.delete(requestId);
|
||||
requestSeqMapRef.current.delete(requestId);
|
||||
console.error("[RealtimeCanvas] flush frame failed:", error);
|
||||
setStatus("error");
|
||||
setStatusMessage(error instanceof Error ? error.message : "发送失败");
|
||||
} finally {
|
||||
flushInFlightRef.current = false;
|
||||
if (pendingFlushModeRef.current) {
|
||||
if (pendingFlushTimerRef.current) {
|
||||
window.clearTimeout(pendingFlushTimerRef.current);
|
||||
pendingFlushTimerRef.current = null;
|
||||
}
|
||||
const shouldDelay = pendingFlushModeRef.current === "preview";
|
||||
if (shouldDelay) {
|
||||
pendingFlushTimerRef.current = window.setTimeout(() => {
|
||||
pendingFlushTimerRef.current = null;
|
||||
pumpFlushRef.current?.();
|
||||
}, adaptiveThrottleMs);
|
||||
} else {
|
||||
pumpFlushRef.current?.();
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
[config, open, prompt, pushMetric]
|
||||
[adaptiveThrottleMs, config, open, prompt, pushMetric]
|
||||
);
|
||||
|
||||
const pumpFlush = useCallback(() => {
|
||||
if (flushInFlightRef.current) return;
|
||||
const mode = pendingFlushModeRef.current;
|
||||
const force = pendingFlushForceRef.current;
|
||||
if (!mode) return;
|
||||
pendingFlushModeRef.current = null;
|
||||
pendingFlushForceRef.current = false;
|
||||
void flushFrame(mode, force);
|
||||
}, [flushFrame]);
|
||||
|
||||
const scheduleFlush = useCallback(
|
||||
(force = false, markDirty = true) => {
|
||||
(force = false, markDirty = true, mode?: FlushMode) => {
|
||||
if (markDirty) {
|
||||
dirtyRef.current = true;
|
||||
markCanvasDirty(null);
|
||||
}
|
||||
if (!dirtyRef.current) return;
|
||||
if (force) {
|
||||
const nextMode: FlushMode = mode || (force ? "refine" : "preview");
|
||||
if (pendingFlushModeRef.current === "refine" || nextMode === "refine") {
|
||||
pendingFlushModeRef.current = "refine";
|
||||
} else {
|
||||
pendingFlushModeRef.current = "preview";
|
||||
}
|
||||
pendingFlushForceRef.current = pendingFlushForceRef.current || force;
|
||||
|
||||
if (!dirtyRef.current && !pendingFlushForceRef.current) {
|
||||
pendingFlushModeRef.current = null;
|
||||
pendingFlushForceRef.current = false;
|
||||
return;
|
||||
}
|
||||
|
||||
if (force || pendingFlushModeRef.current === "refine") {
|
||||
if (pendingFlushTimerRef.current) {
|
||||
window.clearTimeout(pendingFlushTimerRef.current);
|
||||
pendingFlushTimerRef.current = null;
|
||||
}
|
||||
void flushFrame(true);
|
||||
pumpFlush();
|
||||
return;
|
||||
}
|
||||
if (pendingFlushTimerRef.current) return;
|
||||
pendingFlushTimerRef.current = window.setTimeout(() => {
|
||||
pendingFlushTimerRef.current = null;
|
||||
void flushFrame();
|
||||
pumpFlush();
|
||||
}, adaptiveThrottleMs);
|
||||
},
|
||||
[adaptiveThrottleMs, flushFrame]
|
||||
[adaptiveThrottleMs, markCanvasDirty, pumpFlush]
|
||||
);
|
||||
|
||||
const scheduleFlushRef = useRef(scheduleFlush);
|
||||
@ -713,6 +941,10 @@ export function RealtimeCanvasDialog({
|
||||
scheduleFlushRef.current = scheduleFlush;
|
||||
}, [scheduleFlush]);
|
||||
|
||||
useEffect(() => {
|
||||
pumpFlushRef.current = pumpFlush;
|
||||
}, [pumpFlush]);
|
||||
|
||||
useEffect(() => {
|
||||
flushMetricsRef.current = flushMetrics;
|
||||
}, [flushMetrics]);
|
||||
@ -797,10 +1029,10 @@ export function RealtimeCanvasDialog({
|
||||
|
||||
ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight);
|
||||
hasLoadedInitialImageRef.current = true;
|
||||
dirtyRef.current = true;
|
||||
markCanvasDirty(null);
|
||||
saveHistory(); // Save initial image state
|
||||
return true;
|
||||
}, [initialImageUrl, open, saveHistory]);
|
||||
}, [initialImageUrl, markCanvasDirty, open, saveHistory]);
|
||||
|
||||
const initializeCanvas = useCallback(() => {
|
||||
const canvas = canvasRef.current;
|
||||
@ -923,11 +1155,21 @@ export function RealtimeCanvasDialog({
|
||||
clearPreviewCanvas();
|
||||
} else {
|
||||
drawStroke(point, point);
|
||||
scheduleFlush();
|
||||
const lineWidth = brushMode === "eraser" ? brushSize * 1.45 : brushSize;
|
||||
markCanvasDirtyStroke(point, point, lineWidth);
|
||||
scheduleFlush(false, false, "preview");
|
||||
}
|
||||
event.currentTarget.setPointerCapture(event.pointerId);
|
||||
},
|
||||
[brushMode, clearPreviewCanvas, drawStroke, getCanvasPoint, scheduleFlush]
|
||||
[
|
||||
brushMode,
|
||||
brushSize,
|
||||
clearPreviewCanvas,
|
||||
drawStroke,
|
||||
getCanvasPoint,
|
||||
markCanvasDirtyStroke,
|
||||
scheduleFlush,
|
||||
]
|
||||
);
|
||||
|
||||
const handlePointerLeave = useCallback(() => {
|
||||
@ -940,7 +1182,7 @@ export function RealtimeCanvasDialog({
|
||||
shapeStartPointRef.current = null;
|
||||
clearPreviewCanvas();
|
||||
if (wasDrawing) {
|
||||
scheduleFlush(true, false);
|
||||
scheduleFlush(true, false, "refine");
|
||||
}
|
||||
}, [clearPreviewCanvas, scheduleFlush]);
|
||||
|
||||
@ -978,11 +1220,21 @@ export function RealtimeCanvasDialog({
|
||||
}
|
||||
} else {
|
||||
drawStroke(lastPointRef.current, point);
|
||||
const lineWidth = brushMode === "eraser" ? brushSize * 1.45 : brushSize;
|
||||
markCanvasDirtyStroke(lastPointRef.current, point, lineWidth);
|
||||
}
|
||||
lastPointRef.current = point;
|
||||
scheduleFlush();
|
||||
scheduleFlush(false, false, "preview");
|
||||
},
|
||||
[drawShape, drawStroke, getCanvasPoint, scheduleFlush, brushMode, brushSize]
|
||||
[
|
||||
drawShape,
|
||||
drawStroke,
|
||||
getCanvasPoint,
|
||||
scheduleFlush,
|
||||
brushMode,
|
||||
brushSize,
|
||||
markCanvasDirtyStroke,
|
||||
]
|
||||
);
|
||||
|
||||
const handlePointerUp = useCallback(
|
||||
@ -993,6 +1245,14 @@ export function RealtimeCanvasDialog({
|
||||
const shapeStart = shapeStartPointRef.current;
|
||||
if (isShapeMode(brushMode) && endPoint && shapeStart) {
|
||||
drawShape(shapeStart, endPoint);
|
||||
const lineWidth = Math.max(1, brushSize * 0.9);
|
||||
const pad = Math.max(6, Math.ceil(lineWidth * 1.2));
|
||||
markCanvasDirtyRect({
|
||||
minX: Math.min(shapeStart.x, endPoint.x) - pad,
|
||||
minY: Math.min(shapeStart.y, endPoint.y) - pad,
|
||||
maxX: Math.max(shapeStart.x, endPoint.x) + pad,
|
||||
maxY: Math.max(shapeStart.y, endPoint.y) + pad,
|
||||
});
|
||||
clearPreviewCanvas();
|
||||
}
|
||||
lastPointRef.current = null;
|
||||
@ -1003,9 +1263,18 @@ export function RealtimeCanvasDialog({
|
||||
// ignore
|
||||
}
|
||||
saveHistory(); // Save state after stroke
|
||||
scheduleFlush(true, false);
|
||||
scheduleFlush(true, false, "refine");
|
||||
},
|
||||
[brushMode, clearPreviewCanvas, drawShape, getCanvasPoint, saveHistory, scheduleFlush]
|
||||
[
|
||||
brushMode,
|
||||
brushSize,
|
||||
clearPreviewCanvas,
|
||||
drawShape,
|
||||
getCanvasPoint,
|
||||
markCanvasDirtyRect,
|
||||
saveHistory,
|
||||
scheduleFlush,
|
||||
]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
@ -1043,8 +1312,14 @@ export function RealtimeCanvasDialog({
|
||||
sessionIdRef.current = `rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
seqRef.current = 0;
|
||||
lastRenderedSeqRef.current = 0;
|
||||
requestSeqMapRef.current.clear();
|
||||
requestStartMapRef.current.clear();
|
||||
draftVersionRef.current = 0;
|
||||
dirtyRef.current = false;
|
||||
dirtyRegionRef.current = null;
|
||||
pendingFlushModeRef.current = null;
|
||||
pendingFlushForceRef.current = false;
|
||||
latencyEmaRef.current = null;
|
||||
warningMessageRef.current = "";
|
||||
metricsQueueRef.current = [];
|
||||
}, [open]);
|
||||
|
||||
@ -1132,6 +1407,7 @@ export function RealtimeCanvasDialog({
|
||||
};
|
||||
|
||||
setConfig(resolvedConfig);
|
||||
baseThrottleMsRef.current = resolvedConfig.transport.throttleInterval;
|
||||
setAdaptiveThrottleMs(resolvedConfig.transport.throttleInterval);
|
||||
|
||||
if (cancelled) return;
|
||||
@ -1169,6 +1445,8 @@ export function RealtimeCanvasDialog({
|
||||
setStatusMessage("等待连接");
|
||||
pointerDownRef.current = false;
|
||||
lastPointRef.current = null;
|
||||
pendingFlushModeRef.current = null;
|
||||
pendingFlushForceRef.current = false;
|
||||
};
|
||||
}, [open, pushMetric, refreshToken]);
|
||||
|
||||
@ -1182,8 +1460,7 @@ export function RealtimeCanvasDialog({
|
||||
const clearSketch = useCallback(() => {
|
||||
resetCanvas();
|
||||
clearPreviewCanvas();
|
||||
dirtyRef.current = true;
|
||||
scheduleFlush(true, false);
|
||||
scheduleFlush(true, true, "refine");
|
||||
}, [clearPreviewCanvas, resetCanvas, scheduleFlush]);
|
||||
|
||||
const handleSketchUpload = useCallback(
|
||||
@ -1268,9 +1545,14 @@ export function RealtimeCanvasDialog({
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) return;
|
||||
try {
|
||||
const dataUrl = await canvasToJpegDataUri(canvas, 2048, 0.92, canvasBackgroundRef.current);
|
||||
const exportedImage = await canvasToJpegDataUri(
|
||||
canvas,
|
||||
2048,
|
||||
0.92,
|
||||
canvasBackgroundRef.current
|
||||
);
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = dataUrl;
|
||||
anchor.href = exportedImage.dataUri;
|
||||
anchor.download = `realtime-canvas-${Date.now()}.jpg`;
|
||||
anchor.click();
|
||||
toast.success("已保存当前画布");
|
||||
@ -1292,13 +1574,31 @@ export function RealtimeCanvasDialog({
|
||||
onOpenChange(false);
|
||||
}, [onApplyResult, onOpenChange, previewUrl, prompt]);
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="h-[95vh] w-[98vw] max-w-[1800px] overflow-hidden rounded-[20px] border-0 bg-background p-0 shadow-2xl sm:rounded-[20px] md:rounded-[24px]">
|
||||
<DialogHeader className="sr-only">
|
||||
<DialogTitle>实时画布</DialogTitle>
|
||||
<DialogDescription>左侧绘制草图,右侧实时预览。</DialogDescription>
|
||||
</DialogHeader>
|
||||
const content = (
|
||||
<>
|
||||
{standalone ? (
|
||||
<div className="sr-only">
|
||||
<h2>实时画布</h2>
|
||||
<p>左侧绘制草图,右侧实时预览。</p>
|
||||
</div>
|
||||
) : (
|
||||
<DialogHeader className="sr-only">
|
||||
<DialogTitle>实时画布</DialogTitle>
|
||||
<DialogDescription>左侧绘制草图,右侧实时预览。</DialogDescription>
|
||||
</DialogHeader>
|
||||
)}
|
||||
{standalone ? (
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="absolute right-4 top-4 z-[60] h-9 w-9 rounded-full bg-background/85 shadow-sm"
|
||||
onClick={() => onOpenChange(false)}
|
||||
title="返回画布"
|
||||
>
|
||||
<X className="h-5 w-5" />
|
||||
</Button>
|
||||
) : null}
|
||||
{/* Top Header - Minimal */}
|
||||
<div className="absolute left-0 right-0 top-0 z-50 flex h-14 items-center justify-start px-6 bg-gradient-to-b from-black/20 to-transparent pointer-events-none">
|
||||
<div className="pointer-events-auto flex items-center gap-3 rounded-full bg-background/95 backdrop-blur-sm px-4 py-1.5 border border-border/50 shadow-sm">
|
||||
@ -1622,7 +1922,21 @@ export function RealtimeCanvasDialog({
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
|
||||
if (standalone) {
|
||||
return (
|
||||
<div className="h-dvh w-full overflow-hidden bg-background">
|
||||
<div className="relative h-full w-full overflow-hidden bg-background">{content}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="h-[95vh] w-[98vw] max-w-[1800px] overflow-hidden rounded-[20px] border-0 bg-background p-0 shadow-2xl sm:rounded-[20px] md:rounded-[24px]">
|
||||
{content}
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
|
||||
@ -1,14 +1,50 @@
|
||||
"use client";
|
||||
|
||||
import React, { Suspense, useMemo } from "react";
|
||||
import React, { Suspense, useEffect, useMemo } from "react";
|
||||
import { Canvas } from "@react-three/fiber";
|
||||
import { Environment, OrbitControls, useGLTF } from "@react-three/drei";
|
||||
import { Environment, Grid, OrbitControls, useGLTF } from "@react-three/drei";
|
||||
import { AlertCircle, Box, ExternalLink, Loader2 } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import type { Mesh, Object3D } from "three";
|
||||
|
||||
export interface Model3DLayer {
|
||||
id: string;
|
||||
name: string;
|
||||
triangles: number;
|
||||
vertices: number;
|
||||
}
|
||||
|
||||
export interface Model3DStats {
|
||||
triangles: number;
|
||||
vertices: number;
|
||||
meshCount: number;
|
||||
}
|
||||
|
||||
export interface Model3DViewerSettings {
|
||||
envPreset:
|
||||
| "city"
|
||||
| "sunset"
|
||||
| "studio"
|
||||
| "dawn"
|
||||
| "night"
|
||||
| "warehouse"
|
||||
| "forest"
|
||||
| "apartment"
|
||||
| "park"
|
||||
| "lobby";
|
||||
lightIntensity: number;
|
||||
autoRotate: boolean;
|
||||
showGrid: boolean;
|
||||
wireframe: boolean;
|
||||
}
|
||||
|
||||
interface Model3DViewerProps {
|
||||
modelUrl?: string | null;
|
||||
previewImageUrl?: string | null;
|
||||
settings: Model3DViewerSettings;
|
||||
layerVisibility?: Record<string, boolean>;
|
||||
onLayersChange?: (layers: Model3DLayer[]) => void;
|
||||
onStatsChange?: (stats: Model3DStats) => void;
|
||||
}
|
||||
|
||||
class ViewerErrorBoundary extends React.Component<
|
||||
@ -38,9 +74,100 @@ class ViewerErrorBoundary extends React.Component<
|
||||
}
|
||||
}
|
||||
|
||||
function ModelScene({ url }: { url: string }) {
|
||||
function isMesh(node: Object3D): node is Mesh {
|
||||
return (node as Mesh).isMesh === true;
|
||||
}
|
||||
|
||||
function collectSceneMeta(scene: Object3D): {
|
||||
layers: Model3DLayer[];
|
||||
stats: Model3DStats;
|
||||
meshRefs: Array<{ layerId: string; mesh: Mesh }>;
|
||||
} {
|
||||
const layers: Model3DLayer[] = [];
|
||||
const meshRefs: Array<{ layerId: string; mesh: Mesh }> = [];
|
||||
let triangles = 0;
|
||||
let vertices = 0;
|
||||
let meshCount = 0;
|
||||
let meshIndex = 0;
|
||||
|
||||
scene.traverse((node) => {
|
||||
if (!isMesh(node)) return;
|
||||
const geometry = node.geometry;
|
||||
if (!geometry) return;
|
||||
|
||||
const id = `${node.name || "Mesh"}__${meshIndex}`;
|
||||
meshIndex += 1;
|
||||
const indexCount = geometry.index?.count || 0;
|
||||
const positionCount = geometry.attributes.position?.count || 0;
|
||||
const meshTriangles = indexCount > 0 ? Math.floor(indexCount / 3) : Math.floor(positionCount / 3);
|
||||
const meshVertices = positionCount;
|
||||
|
||||
triangles += meshTriangles;
|
||||
vertices += meshVertices;
|
||||
meshCount += 1;
|
||||
|
||||
layers.push({
|
||||
id,
|
||||
name: node.name?.trim() || `mesh_${meshCount}`,
|
||||
triangles: meshTriangles,
|
||||
vertices: meshVertices,
|
||||
});
|
||||
meshRefs.push({ layerId: id, mesh: node });
|
||||
});
|
||||
|
||||
return {
|
||||
layers,
|
||||
stats: {
|
||||
triangles,
|
||||
vertices,
|
||||
meshCount,
|
||||
},
|
||||
meshRefs,
|
||||
};
|
||||
}
|
||||
|
||||
function ModelScene({
|
||||
url,
|
||||
settings,
|
||||
layerVisibility,
|
||||
onLayersChange,
|
||||
onStatsChange,
|
||||
}: {
|
||||
url: string;
|
||||
settings: Model3DViewerSettings;
|
||||
layerVisibility?: Record<string, boolean>;
|
||||
onLayersChange?: (layers: Model3DLayer[]) => void;
|
||||
onStatsChange?: (stats: Model3DStats) => void;
|
||||
}) {
|
||||
const model = useGLTF(url);
|
||||
const scene = useMemo(() => model.scene.clone(), [model.scene]);
|
||||
const meta = useMemo(() => collectSceneMeta(scene), [scene]);
|
||||
|
||||
useEffect(() => {
|
||||
onLayersChange?.(meta.layers);
|
||||
onStatsChange?.(meta.stats);
|
||||
}, [meta.layers, meta.stats, onLayersChange, onStatsChange]);
|
||||
|
||||
useEffect(() => {
|
||||
for (const { layerId, mesh } of meta.meshRefs) {
|
||||
const nextVisible = layerVisibility?.[layerId];
|
||||
mesh.visible = nextVisible !== false;
|
||||
}
|
||||
}, [layerVisibility, meta.meshRefs]);
|
||||
|
||||
useEffect(() => {
|
||||
for (const { mesh } of meta.meshRefs) {
|
||||
const materials = Array.isArray(mesh.material) ? mesh.material : [mesh.material];
|
||||
for (const material of materials) {
|
||||
if (!material) continue;
|
||||
if ("wireframe" in material) {
|
||||
(material as any).wireframe = settings.wireframe;
|
||||
material.needsUpdate = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}, [meta.meshRefs, settings.wireframe]);
|
||||
|
||||
return <primitive object={scene} scale={1} />;
|
||||
}
|
||||
|
||||
@ -66,7 +193,14 @@ function EmptyState({ previewImageUrl }: { previewImageUrl?: string | null }) {
|
||||
);
|
||||
}
|
||||
|
||||
export function Model3DViewer({ modelUrl, previewImageUrl }: Model3DViewerProps) {
|
||||
export function Model3DViewer({
|
||||
modelUrl,
|
||||
previewImageUrl,
|
||||
settings,
|
||||
layerVisibility,
|
||||
onLayersChange,
|
||||
onStatsChange,
|
||||
}: Model3DViewerProps) {
|
||||
if (!modelUrl) {
|
||||
return <EmptyState previewImageUrl={previewImageUrl} />;
|
||||
}
|
||||
@ -104,8 +238,20 @@ export function Model3DViewer({ modelUrl, previewImageUrl }: Model3DViewerProps)
|
||||
}
|
||||
>
|
||||
<Canvas camera={{ position: [2, 1.5, 2], fov: 45 }}>
|
||||
<ambientLight intensity={0.8} />
|
||||
<directionalLight position={[5, 8, 5]} intensity={1.2} />
|
||||
<ambientLight intensity={0.5 * settings.lightIntensity} />
|
||||
<directionalLight position={[5, 8, 5]} intensity={1.2 * settings.lightIntensity} />
|
||||
{settings.showGrid ? (
|
||||
<Grid
|
||||
position={[0, -0.01, 0]}
|
||||
args={[12, 12]}
|
||||
cellColor="#8c8c8c"
|
||||
sectionColor="#525252"
|
||||
cellSize={0.5}
|
||||
sectionSize={2}
|
||||
fadeDistance={40}
|
||||
fadeStrength={1.5}
|
||||
/>
|
||||
) : null}
|
||||
<Suspense
|
||||
fallback={
|
||||
<mesh>
|
||||
@ -114,10 +260,16 @@ export function Model3DViewer({ modelUrl, previewImageUrl }: Model3DViewerProps)
|
||||
</mesh>
|
||||
}
|
||||
>
|
||||
<ModelScene url={modelUrl} />
|
||||
<Environment preset="city" />
|
||||
<ModelScene
|
||||
url={modelUrl}
|
||||
settings={settings}
|
||||
layerVisibility={layerVisibility}
|
||||
onLayersChange={onLayersChange}
|
||||
onStatsChange={onStatsChange}
|
||||
/>
|
||||
<Environment preset={settings.envPreset} />
|
||||
</Suspense>
|
||||
<OrbitControls enablePan enableZoom enableRotate />
|
||||
<OrbitControls enablePan enableZoom enableRotate autoRotate={settings.autoRotate} />
|
||||
</Canvas>
|
||||
<div className="pointer-events-none absolute left-3 top-3 rounded-md bg-background/80 px-2 py-1 text-xs text-muted-foreground">
|
||||
<span className="inline-flex items-center gap-1">
|
||||
|
||||
302
lib/realtime/runware-capability.ts
Normal file
302
lib/realtime/runware-capability.ts
Normal file
@ -0,0 +1,302 @@
|
||||
const RUNWARE_ENDPOINT = "https://api.runware.ai/v1";
|
||||
const DEFAULT_TIMEOUT_MS = 12_000;
|
||||
const PROBE_IMAGE_URL =
|
||||
"https://upload.wikimedia.org/wikipedia/commons/thumb/a/a9/Example.jpg/512px-Example.jpg";
|
||||
const DEFAULT_MODEL_ID = "runware:101@1";
|
||||
|
||||
export type RunwareCapabilityCode =
|
||||
| "SUPPORTED"
|
||||
| "UNSUPPORTED_SEED_IMAGE"
|
||||
| "MODEL_REQUIRES_MASK"
|
||||
| "AUTH_FAILED"
|
||||
| "MODEL_INVALID"
|
||||
| "UPSTREAM_ERROR";
|
||||
|
||||
export type RunwareSeedImageProbeResult = {
|
||||
supported: boolean;
|
||||
code: RunwareCapabilityCode;
|
||||
message: string;
|
||||
status?: number;
|
||||
};
|
||||
|
||||
type RunwareErrorInfo = {
|
||||
code: string;
|
||||
message: string;
|
||||
parameter: string;
|
||||
};
|
||||
|
||||
function toText(value: unknown) {
|
||||
return typeof value === "string" ? value.trim() : "";
|
||||
}
|
||||
|
||||
function collectErrorInfos(payload: unknown): RunwareErrorInfo[] {
|
||||
const infos: RunwareErrorInfo[] = [];
|
||||
const visited = new Set<unknown>();
|
||||
const queue: unknown[] = [payload];
|
||||
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
if (!current || typeof current !== "object" || visited.has(current)) {
|
||||
continue;
|
||||
}
|
||||
visited.add(current);
|
||||
|
||||
if (Array.isArray(current)) {
|
||||
for (const item of current) {
|
||||
queue.push(item);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const objectValue = current as Record<string, unknown>;
|
||||
const message = toText(objectValue.message || objectValue.error);
|
||||
const code = toText(objectValue.code);
|
||||
const parameter = toText(objectValue.parameter);
|
||||
const lowerCode = code.toLowerCase();
|
||||
const looksLikeErrorCode = containsAny(lowerCode, [
|
||||
"error",
|
||||
"invalid",
|
||||
"unsupported",
|
||||
"unauthorized",
|
||||
"forbidden",
|
||||
"notfound",
|
||||
"not_found",
|
||||
"failed",
|
||||
]);
|
||||
if (message || parameter || looksLikeErrorCode) {
|
||||
infos.push({ code, message, parameter });
|
||||
}
|
||||
|
||||
const nestedErrors = objectValue.errors;
|
||||
const nestedData = objectValue.data;
|
||||
if (nestedErrors) {
|
||||
queue.push(nestedErrors);
|
||||
}
|
||||
if (nestedData) {
|
||||
queue.push(nestedData);
|
||||
}
|
||||
}
|
||||
|
||||
return infos;
|
||||
}
|
||||
|
||||
function containsAny(text: string, patterns: string[]) {
|
||||
return patterns.some((pattern) => text.includes(pattern));
|
||||
}
|
||||
|
||||
function normalizeErrorText(errorInfo: RunwareErrorInfo) {
|
||||
return `${errorInfo.code} ${errorInfo.parameter} ${errorInfo.message}`.toLowerCase();
|
||||
}
|
||||
|
||||
function isUnsupportedSeedImage(errorInfo: RunwareErrorInfo) {
|
||||
const text = normalizeErrorText(errorInfo);
|
||||
return (
|
||||
containsAny(text, [
|
||||
"unsupported use of 'seedimage'",
|
||||
'unsupported use of "seedimage"',
|
||||
"unsupported use of seedimage",
|
||||
"seedimage is not supported",
|
||||
"unsupportedparameter seedimage",
|
||||
"unsupported parameter seedimage",
|
||||
]) ||
|
||||
(text.includes("seedimage") && text.includes("unsupported"))
|
||||
);
|
||||
}
|
||||
|
||||
function isAuthFailure(status: number, errorInfo: RunwareErrorInfo) {
|
||||
const text = normalizeErrorText(errorInfo);
|
||||
if (status === 401 || status === 403) return true;
|
||||
return containsAny(text, [
|
||||
"unauthorized",
|
||||
"forbidden",
|
||||
"invalid api key",
|
||||
"invalidapikey",
|
||||
"authentication failed",
|
||||
"auth failed",
|
||||
"invalid credentials",
|
||||
]);
|
||||
}
|
||||
|
||||
function isModelInvalid(errorInfo: RunwareErrorInfo) {
|
||||
const text = normalizeErrorText(errorInfo);
|
||||
return containsAny(text, [
|
||||
"invalid model",
|
||||
"model not found",
|
||||
"unknown model",
|
||||
"model does not exist",
|
||||
"invalidmodel",
|
||||
]);
|
||||
}
|
||||
|
||||
function modelRequiresMask(errorInfo: RunwareErrorInfo) {
|
||||
const text = normalizeErrorText(errorInfo);
|
||||
return containsAny(text, [
|
||||
"missingmaskimage",
|
||||
"missing maskimage",
|
||||
"requires a mask image",
|
||||
"requires mask image",
|
||||
"requires a mask",
|
||||
]);
|
||||
}
|
||||
|
||||
function responseHasImage(payload: unknown, taskUUID: string) {
|
||||
if (!payload) return false;
|
||||
const list = Array.isArray(payload)
|
||||
? payload
|
||||
: Array.isArray((payload as Record<string, unknown>)?.data)
|
||||
? ((payload as Record<string, unknown>).data as unknown[])
|
||||
: [];
|
||||
|
||||
return list.some((item) => {
|
||||
if (!item || typeof item !== "object") return false;
|
||||
const row = item as Record<string, unknown>;
|
||||
const image = toText(row.imageDataURI || row.imageURL || row.imageUrl);
|
||||
if (!image) return false;
|
||||
const rowTaskUUID = toText(row.taskUUID);
|
||||
return !rowTaskUUID || rowTaskUUID === taskUUID;
|
||||
});
|
||||
}
|
||||
|
||||
function parseProbeFailure(
|
||||
status: number,
|
||||
modelId: string,
|
||||
infos: RunwareErrorInfo[]
|
||||
): RunwareSeedImageProbeResult {
|
||||
const firstMessage = infos.find((info) => info.message)?.message;
|
||||
|
||||
if (status === 401 || status === 403) {
|
||||
return {
|
||||
supported: false,
|
||||
code: "AUTH_FAILED",
|
||||
message: "Runware API Key 无效或无权限,请检查后重试。",
|
||||
status,
|
||||
};
|
||||
}
|
||||
|
||||
if (infos.some((info) => isUnsupportedSeedImage(info))) {
|
||||
return {
|
||||
supported: false,
|
||||
code: "UNSUPPORTED_SEED_IMAGE",
|
||||
message: `模型 ${modelId} 不支持 seedImage,已阻止保存。`,
|
||||
status,
|
||||
};
|
||||
}
|
||||
|
||||
if (infos.some((info) => modelRequiresMask(info))) {
|
||||
return {
|
||||
supported: false,
|
||||
code: "MODEL_REQUIRES_MASK",
|
||||
message: `模型 ${modelId} 依赖 maskImage,当前实时画布链路不兼容。`,
|
||||
status,
|
||||
};
|
||||
}
|
||||
|
||||
if (infos.some((info) => isAuthFailure(status, info))) {
|
||||
return {
|
||||
supported: false,
|
||||
code: "AUTH_FAILED",
|
||||
message: "Runware API Key 无效或无权限,请检查后重试。",
|
||||
status,
|
||||
};
|
||||
}
|
||||
|
||||
if (infos.some((info) => isModelInvalid(info))) {
|
||||
return {
|
||||
supported: false,
|
||||
code: "MODEL_INVALID",
|
||||
message: `模型 ${modelId} 不存在或不可用,请更换模型后重试。`,
|
||||
status,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
supported: false,
|
||||
code: "UPSTREAM_ERROR",
|
||||
message: firstMessage || `Runware 探针请求失败(${status || 0})`,
|
||||
status,
|
||||
};
|
||||
}
|
||||
|
||||
export async function probeRunwareSeedImageSupport(params: {
|
||||
apiKey: string;
|
||||
modelId?: string;
|
||||
timeoutMs?: number;
|
||||
}): Promise<RunwareSeedImageProbeResult> {
|
||||
const apiKey = params.apiKey.trim();
|
||||
const modelId = (params.modelId || DEFAULT_MODEL_ID).trim() || DEFAULT_MODEL_ID;
|
||||
const timeoutMs = Number.isFinite(params.timeoutMs) ? Number(params.timeoutMs) : DEFAULT_TIMEOUT_MS;
|
||||
|
||||
if (!apiKey) {
|
||||
return {
|
||||
supported: false,
|
||||
code: "AUTH_FAILED",
|
||||
message: "缺少 Runware API Key。",
|
||||
};
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), Math.max(2_000, timeoutMs));
|
||||
|
||||
let status = 0;
|
||||
let payload: unknown = null;
|
||||
|
||||
try {
|
||||
const taskUUID = crypto.randomUUID();
|
||||
const response = await fetch(RUNWARE_ENDPOINT, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json",
|
||||
},
|
||||
body: JSON.stringify([
|
||||
{
|
||||
taskType: "imageInference",
|
||||
taskUUID,
|
||||
model: modelId,
|
||||
positivePrompt: "probe",
|
||||
seedImage: PROBE_IMAGE_URL,
|
||||
width: 512,
|
||||
height: 512,
|
||||
strength: 0.2,
|
||||
CFGScale: 7,
|
||||
steps: 1,
|
||||
numberResults: 1,
|
||||
outputType: "dataURI",
|
||||
},
|
||||
]),
|
||||
cache: "no-store",
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
status = response.status;
|
||||
payload = await response.json().catch(() => null);
|
||||
const infos = collectErrorInfos(payload);
|
||||
|
||||
if (response.ok && infos.length === 0 && responseHasImage(payload, taskUUID)) {
|
||||
return {
|
||||
supported: true,
|
||||
code: "SUPPORTED",
|
||||
message: "模型支持 seedImage。",
|
||||
status,
|
||||
};
|
||||
}
|
||||
|
||||
return parseProbeFailure(status, modelId, infos);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") {
|
||||
return {
|
||||
supported: false,
|
||||
code: "UPSTREAM_ERROR",
|
||||
message: "Runware 探针请求超时,请稍后重试。",
|
||||
};
|
||||
}
|
||||
return {
|
||||
supported: false,
|
||||
code: "UPSTREAM_ERROR",
|
||||
message: error instanceof Error ? error.message : "Runware 探针请求失败。",
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
279
lib/realtime/runware-ws.ts
Normal file
279
lib/realtime/runware-ws.ts
Normal file
@ -0,0 +1,279 @@
|
||||
type RunwareTask = {
|
||||
taskType: string;
|
||||
taskUUID?: string;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
type RunwarePayload = {
|
||||
data?: Array<Record<string, unknown>>;
|
||||
errors?: Array<Record<string, unknown>>;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
type PendingTask = {
|
||||
timeout: ReturnType<typeof setTimeout>;
|
||||
resolve: (value: RunwarePayload) => void;
|
||||
reject: (reason?: unknown) => void;
|
||||
};
|
||||
|
||||
class RunwareWebSocketClient {
|
||||
private readonly endpoint = "wss://ws-api.runware.ai/v1";
|
||||
private socket: WebSocket | null = null;
|
||||
private connectionSessionUUID = "";
|
||||
private authenticated = false;
|
||||
private connectPromise: Promise<void> | null = null;
|
||||
private pingTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private authWaiter:
|
||||
| {
|
||||
timeout: ReturnType<typeof setTimeout>;
|
||||
resolve: () => void;
|
||||
reject: (reason?: unknown) => void;
|
||||
}
|
||||
| null = null;
|
||||
private pending = new Map<string, PendingTask>();
|
||||
|
||||
constructor(private readonly apiKey: string) {}
|
||||
|
||||
private closeSocket() {
|
||||
if (!this.socket) return;
|
||||
try {
|
||||
this.socket.close();
|
||||
} catch {}
|
||||
this.socket = null;
|
||||
}
|
||||
|
||||
private clearPingTimer() {
|
||||
if (!this.pingTimer) return;
|
||||
clearInterval(this.pingTimer);
|
||||
this.pingTimer = null;
|
||||
}
|
||||
|
||||
private rejectPending(reason: unknown) {
|
||||
for (const [, task] of this.pending) {
|
||||
clearTimeout(task.timeout);
|
||||
task.reject(reason);
|
||||
}
|
||||
this.pending.clear();
|
||||
}
|
||||
|
||||
private rejectAuthWaiter(reason: unknown) {
|
||||
if (!this.authWaiter) return;
|
||||
clearTimeout(this.authWaiter.timeout);
|
||||
this.authWaiter.reject(reason);
|
||||
this.authWaiter = null;
|
||||
}
|
||||
|
||||
private startPing() {
|
||||
this.clearPingTimer();
|
||||
this.pingTimer = setInterval(() => {
|
||||
const socket = this.socket;
|
||||
if (!socket || socket.readyState !== WebSocket.OPEN) return;
|
||||
try {
|
||||
socket.send(
|
||||
JSON.stringify([
|
||||
{
|
||||
taskType: "ping",
|
||||
ping: true,
|
||||
},
|
||||
])
|
||||
);
|
||||
} catch {}
|
||||
}, 55_000);
|
||||
}
|
||||
|
||||
private sendAuth() {
|
||||
const socket = this.socket;
|
||||
if (!socket || socket.readyState !== WebSocket.OPEN) {
|
||||
throw new Error("Runware WebSocket 未连接");
|
||||
}
|
||||
const authTask: Record<string, unknown> = {
|
||||
taskType: "authentication",
|
||||
apiKey: this.apiKey,
|
||||
};
|
||||
if (this.connectionSessionUUID) {
|
||||
authTask.connectionSessionUUID = this.connectionSessionUUID;
|
||||
}
|
||||
socket.send(JSON.stringify([authTask]));
|
||||
}
|
||||
|
||||
private parseMessage(raw: unknown) {
|
||||
if (typeof raw !== "string") return null;
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as RunwarePayload;
|
||||
return parsed;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private handleMessage(raw: unknown) {
|
||||
const payload = this.parseMessage(raw);
|
||||
if (!payload) return;
|
||||
const dataList = Array.isArray(payload.data) ? payload.data : [];
|
||||
const errorList = Array.isArray(payload.errors) ? payload.errors : [];
|
||||
|
||||
if (this.authWaiter) {
|
||||
const authError = errorList.find((item) => item?.taskType === "authentication");
|
||||
const authData = dataList.find((item) => item?.taskType === "authentication");
|
||||
if (authError) {
|
||||
const message = String(authError.message || "Runware WebSocket 鉴权失败");
|
||||
this.rejectAuthWaiter(new Error(message));
|
||||
} else if (authData) {
|
||||
const sessionId = String(authData.connectionSessionUUID || "").trim();
|
||||
if (sessionId) {
|
||||
this.connectionSessionUUID = sessionId;
|
||||
}
|
||||
this.authenticated = true;
|
||||
this.startPing();
|
||||
this.authWaiter.resolve();
|
||||
clearTimeout(this.authWaiter.timeout);
|
||||
this.authWaiter = null;
|
||||
}
|
||||
}
|
||||
|
||||
for (const item of dataList) {
|
||||
const taskUUID = String(item?.taskUUID || "").trim();
|
||||
if (!taskUUID) continue;
|
||||
const pendingTask = this.pending.get(taskUUID);
|
||||
if (!pendingTask) continue;
|
||||
clearTimeout(pendingTask.timeout);
|
||||
this.pending.delete(taskUUID);
|
||||
pendingTask.resolve(payload);
|
||||
}
|
||||
|
||||
for (const item of errorList) {
|
||||
const taskUUID = String(item?.taskUUID || "").trim();
|
||||
if (!taskUUID) continue;
|
||||
const pendingTask = this.pending.get(taskUUID);
|
||||
if (!pendingTask) continue;
|
||||
clearTimeout(pendingTask.timeout);
|
||||
this.pending.delete(taskUUID);
|
||||
const message = String(item?.message || item?.code || "Runware WebSocket 任务失败");
|
||||
pendingTask.reject(new Error(message));
|
||||
}
|
||||
}
|
||||
|
||||
private handleSocketClose() {
|
||||
this.authenticated = false;
|
||||
this.clearPingTimer();
|
||||
this.rejectAuthWaiter(new Error("Runware WebSocket 连接已断开"));
|
||||
this.rejectPending(new Error("Runware WebSocket 连接已断开"));
|
||||
this.socket = null;
|
||||
}
|
||||
|
||||
async ensureConnected(timeoutMs = 10_000) {
|
||||
if (this.socket?.readyState === WebSocket.OPEN && this.authenticated) {
|
||||
return;
|
||||
}
|
||||
if (this.connectPromise) {
|
||||
return this.connectPromise;
|
||||
}
|
||||
|
||||
this.connectPromise = new Promise<void>((resolve, reject) => {
|
||||
this.closeSocket();
|
||||
this.authenticated = false;
|
||||
this.clearPingTimer();
|
||||
|
||||
const socket = new WebSocket(this.endpoint);
|
||||
this.socket = socket;
|
||||
|
||||
const cleanupAndReject = (error: unknown) => {
|
||||
this.handleSocketClose();
|
||||
reject(error instanceof Error ? error : new Error("Runware WebSocket 连接失败"));
|
||||
};
|
||||
|
||||
socket.onmessage = (event) => {
|
||||
this.handleMessage(event.data);
|
||||
};
|
||||
|
||||
socket.onerror = () => {
|
||||
if (!this.authenticated) {
|
||||
cleanupAndReject(new Error("Runware WebSocket 连接错误"));
|
||||
}
|
||||
};
|
||||
|
||||
socket.onclose = () => {
|
||||
this.handleSocketClose();
|
||||
};
|
||||
|
||||
socket.onopen = () => {
|
||||
this.authWaiter = {
|
||||
timeout: setTimeout(() => {
|
||||
this.rejectAuthWaiter(new Error("Runware WebSocket 鉴权超时"));
|
||||
this.closeSocket();
|
||||
reject(new Error("Runware WebSocket 鉴权超时"));
|
||||
}, timeoutMs),
|
||||
resolve,
|
||||
reject: (reason?: unknown) => {
|
||||
cleanupAndReject(reason);
|
||||
},
|
||||
};
|
||||
try {
|
||||
this.sendAuth();
|
||||
} catch (error) {
|
||||
cleanupAndReject(error);
|
||||
}
|
||||
};
|
||||
}).finally(() => {
|
||||
this.connectPromise = null;
|
||||
});
|
||||
|
||||
return this.connectPromise;
|
||||
}
|
||||
|
||||
async sendTask(task: RunwareTask, timeoutMs = 90_000) {
|
||||
const taskUUID = String(task.taskUUID || "").trim();
|
||||
if (!taskUUID) {
|
||||
throw new Error("Runware 任务缺少 taskUUID");
|
||||
}
|
||||
|
||||
await this.ensureConnected();
|
||||
const socket = this.socket;
|
||||
if (!socket || socket.readyState !== WebSocket.OPEN || !this.authenticated) {
|
||||
throw new Error("Runware WebSocket 未就绪");
|
||||
}
|
||||
|
||||
return await new Promise<RunwarePayload>((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
this.pending.delete(taskUUID);
|
||||
reject(new Error("Runware WebSocket 推理超时"));
|
||||
}, timeoutMs);
|
||||
|
||||
this.pending.set(taskUUID, {
|
||||
timeout,
|
||||
resolve,
|
||||
reject,
|
||||
});
|
||||
|
||||
try {
|
||||
socket.send(JSON.stringify([task]));
|
||||
} catch (error) {
|
||||
clearTimeout(timeout);
|
||||
this.pending.delete(taskUUID);
|
||||
reject(error instanceof Error ? error : new Error("Runware WebSocket 发送失败"));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
declare global {
|
||||
// eslint-disable-next-line no-var
|
||||
var __runwareWebSocketPool: Map<string, RunwareWebSocketClient> | undefined;
|
||||
}
|
||||
|
||||
function getRunwarePool() {
|
||||
if (!globalThis.__runwareWebSocketPool) {
|
||||
globalThis.__runwareWebSocketPool = new Map<string, RunwareWebSocketClient>();
|
||||
}
|
||||
return globalThis.__runwareWebSocketPool;
|
||||
}
|
||||
|
||||
export function getRunwareWebSocketClient(apiKey: string) {
|
||||
const pool = getRunwarePool();
|
||||
let client = pool.get(apiKey);
|
||||
if (!client) {
|
||||
client = new RunwareWebSocketClient(apiKey);
|
||||
pool.set(apiKey, client);
|
||||
}
|
||||
return client;
|
||||
}
|
||||
@ -126,21 +126,48 @@ export async function tripoRequest<T = unknown>(
|
||||
|
||||
export function pickTaskModelUrl(output?: Record<string, unknown> | null): string | null {
|
||||
if (!output) return null;
|
||||
const candidates = [output.pbr_model, output.model, output.base_model];
|
||||
const candidates = [
|
||||
output.pbr_model,
|
||||
output.model,
|
||||
output.base_model,
|
||||
output.rigged_model,
|
||||
output.animated_model,
|
||||
output.low_poly_model,
|
||||
output.result_model,
|
||||
output.glb,
|
||||
output.fbx,
|
||||
output.usdz,
|
||||
output.obj,
|
||||
output.stl,
|
||||
];
|
||||
for (const item of candidates) {
|
||||
const value = typeof item === "string" ? item.trim() : "";
|
||||
if (value) return value;
|
||||
if (typeof item === "string") {
|
||||
const value = item.trim();
|
||||
if (value) return value;
|
||||
continue;
|
||||
}
|
||||
if (item && typeof item === "object") {
|
||||
const nested =
|
||||
typeof (item as Record<string, unknown>).url === "string"
|
||||
? String((item as Record<string, unknown>).url).trim()
|
||||
: "";
|
||||
if (nested) return nested;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function pickTaskPreviewImageUrl(output?: Record<string, unknown> | null): string | null {
|
||||
if (!output) return null;
|
||||
const candidates = [output.generated_image, output.rendered_image];
|
||||
const candidates = [
|
||||
output.generated_image,
|
||||
output.rendered_image,
|
||||
output.preview_image,
|
||||
output.thumbnail,
|
||||
];
|
||||
for (const item of candidates) {
|
||||
const value = typeof item === "string" ? item.trim() : "";
|
||||
if (value) return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@ -1,4 +1,17 @@
|
||||
export type TripoTaskType = "text_to_model" | "image_to_model" | "convert_model";
|
||||
export type TripoTaskType =
|
||||
| "text_to_model"
|
||||
| "image_to_model"
|
||||
| "multiview_to_model"
|
||||
| "convert_model"
|
||||
| "refine_model"
|
||||
| "stylize_model"
|
||||
| "texture_model"
|
||||
| "animate_prerigcheck"
|
||||
| "animate_rig"
|
||||
| "animate_retarget"
|
||||
| "mesh_segmentation"
|
||||
| "mesh_completion"
|
||||
| "highpoly_to_lowpoly";
|
||||
|
||||
export type TripoTaskStatus =
|
||||
| "queued"
|
||||
@ -43,4 +56,3 @@ export interface TripoApiEnvelope<T> {
|
||||
message?: string;
|
||||
suggestion?: string;
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user