lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(canvas): add layout replace and vip98k provider

Browse Source
This commit is contained in:
134 2026-07-31 18:17:29 +08:00
parent 0a81f15f11
commit d5737b70c0
54 changed files with 4538 additions and 460 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/(admin)/settings/_lib/types.ts
View File

@ -92,6 +92,7 @@ export type AdminAiProvider = {
| "bananaimage"
| "visionary"
| "atlascloud"
| "vip98k"
| "stepfun-image"
| "midjourney"
| "fal-video"

82
app/(admin)/settings/admin-settings-client.tsx
View File

@ -36,6 +36,7 @@ import {
ImagePlus,
Key,
Layers,
LayoutTemplate,
Loader2,
type LucideIcon,
Maximize2,
@ -625,6 +626,7 @@ type AiProvider = {
| "bananaimage"
| "visionary"
| "atlascloud"
| "vip98k"
| "stepfun-image"
| "midjourney"
| "fal-video"
@ -654,6 +656,9 @@ type ImageGenConfig = {
supportsSeed?: boolean; // 是否支持随机种子
supportedAspectRatios?: string[]; // 支持的图片比例
supportedImageSizes?: string[]; // 支持的分辨率/K 数
maxReferenceImages?: number; // 最大参考图数量
maxVideoReferences?: number; // 最大参考视频数量
supportsQuality?: boolean; // 是否支持质量档位
creditCostPerImage?: number; // 每张图片积分消耗
creditCostMultiplierBySize?: Record<string, number>; // 各图片尺寸积分倍率
creditDiscountRate?: number; // 积分折扣，按“折”配置，如 5 = 5折
@ -663,6 +668,7 @@ type ImageGenConfig = {
creditCostPerVideoByResolution?: Record<string, number>; // 各分辨率积分
enableLoadingPlaceholder?: boolean; // 是否启用加载占位图
loadingColor?: string; // 占位图主题色，如 "purple", "blue", "green"
vip98k?: Record<string, unknown>; // VIP98K 目录元数据
};
function createDefaultImageGenConfig(): ImageGenConfig {
@ -1037,6 +1043,7 @@ const CANVAS_TOOLBAR_FALLBACK_ICONS: Record<CanvasToolbarItemKey, LucideIcon> =
imageRemoveBackground: Scissors,
imageBoxCutout: Scan,
imageLocalEdit: Wand2,
imageLayoutReplace: LayoutTemplate,
imageRemoveWatermark: Eraser,
imageResize: Maximize2,
imageTextEdit: Type,
@ -1585,6 +1592,7 @@ const PROVIDER_TYPES = [
{ value: "bananaimage", label: "BananaImage 生图" },
{ value: "visionary", label: "Visionary 生图" },
{ value: "atlascloud", label: "Atlas Cloud 图片 / 3D 模型" },
{ value: "vip98k", label: "VIP98K 混合模型服务" },
{ value: "stepfun-image", label: "阶跃星辰 Step Image Edit" },
{ value: "bltcy-video", label: "柏拉图统一视频" },
{ value: "grsai-gpt-image", label: "Grsai GPT Image" },
@ -1622,6 +1630,7 @@ const PROVIDER_DEFAULT_BASE_URL: Record<string, string> = {
bananaimage: "https://bananaimage.aigenmedia.art",
visionary: "https://api.visionary.beer",
atlascloud: "https://api.atlascloud.ai",
vip98k: "https://api.vip98k.com",
"stepfun-image": "https://api.stepfun.com",
"bltcy-video": "https://api.bltcy.ai",
"grsai-gpt-image": "https://grsai.dakka.com.cn",
@ -1629,7 +1638,7 @@ const PROVIDER_DEFAULT_BASE_URL: Record<string, string> = {
midjourney: "https://your-midjourney-api.example.com",
"fal-video": "https://fal.run",
"tripo-3d": "https://api.tripo3d.ai/v2/openapi",
"worldlabs-3d": "https://api.worldlabs.ai",
"worldlabs-3d": "https://api.vip98k.com",
"kittycad-cad": "https://api.zoo.dev",
runway: "https://api.dev.runwayml.com",
krea: "https://api.krea.ai",
@ -4055,6 +4064,7 @@ export function AdminSettingsClient() {
| "bananaimage"
| "visionary"
| "atlascloud"
| "vip98k"
| "stepfun-image"
| "midjourney"
| "fal-video"
@ -6329,6 +6339,7 @@ export function AdminSettingsClient() {
const isBananaImageProvider = providerForm.type === "bananaimage";
const isVisionaryProvider = providerForm.type === "visionary";
const isAtlasCloudProvider = providerForm.type === "atlascloud";
const isVip98kProvider = providerForm.type === "vip98k";
const isBltcyVideoProvider = providerForm.type === "bltcy-video";
const isLemonProvider =
providerForm.type === "openai" &&
@ -6504,6 +6515,17 @@ export function AdminSettingsClient() {
}
}
if (isNewProvider && isVip98kProvider && providerId) {
try {
await handleQuickAddVip98kModels(providerId, { skipConfirm: true });
} catch (modelError) {
console.error("[Auto Sync VIP98K Models] Error:", modelError);
toast.warning(
'VIP98K 模型同步失败，请手动点击"同步 VIP98K 全部模型"按钮'
);
}
}
if (isNewProvider && isBltcyVideoProvider && providerId) {
try {
await handleQuickAddBltcyVideoModels(providerId, {
@ -7591,6 +7613,46 @@ export function AdminSettingsClient() {
}
};
const handleQuickAddVip98kModels = async (
providerId: string,
options?: { skipConfirm?: boolean }
) => {
if (
!options?.skipConfirm &&
!confirm(
"将从 VIP98K 模型目录同步全部模型，并按图片、视频、3D 世界、文本、图片工具、OCR、音频工具分类。图片模型会配置比例、1K/2K/4K 和多参考图能力。\n\n继续吗？"
)
) {
return;
}
try {
toast.info("正在读取 VIP98K 模型目录...");
const response = await fetch("/api/admin/vip98k-models", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ providerId }),
});
const data = await response.json();
if (!response.ok) throw new Error(data.error || "同步失败");
const totalSynced =
Number(data.addedCount || 0) + Number(data.updatedCount || 0);
const totalCatalog = Number(data.counts?.total || totalSynced);
toast.success(
`已同步 ${totalSynced} 个 VIP98K 模型（目录共 ${totalCatalog} 个）`
);
mutateModels();
if (data.errors?.length) {
toast.warning(`${data.errors.length} 个 VIP98K 模型同步失败`);
}
} catch (error: any) {
console.error("[Quick Add VIP98K Models] Error:", error);
toast.error(error.message || "同步 VIP98K 模型失败");
throw error;
}
};
const handleUseFalImageProviderPreset = () => {
setProviderForm({
id: "",
@ -15800,6 +15862,13 @@ export function AdminSettingsClient() {
并按每个模型自己的 Schema 配置多图数量、比例及 1K/2K/4K 档位。
</p>
)}
{providerForm.type === "vip98k" && (
<p className="text-muted-foreground text-xs">
VIP98K 使用 Bearer API Key 认证，Base URL 只填写
https://api.vip98k.com。保存后会自动同步目录中的图片、视频、3D 世界、
文本和工具模型；图片模型会配置比例、1K/2K/4K 与多参考图能力。
</p>
)}
{providerForm.type === "bfl-image" &&
isBltcyBflBaseUrl(providerForm.baseUrl) && (
<p className="text-muted-foreground text-xs">
@ -15957,6 +16026,17 @@ export function AdminSettingsClient() {
同步 Atlas Cloud 图片 / 3D 模型
</Button>
)}
{provider.type === "vip98k" && (
<Button
className="mt-2"
onClick={() => handleQuickAddVip98kModels(provider.id)}
size="sm"
variant="secondary"
>
<Sparkles className="mr-1 h-4 w-4" />
同步 VIP98K 全部模型
</Button>
)}
{provider.type === "bltcy-video" && (
<Button
className="mt-2"

2
app/(chat)/api/chat/_lib/image-failover.ts
View File

@ -18,6 +18,7 @@ export const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([
"bananaimage",
"visionary",
"atlascloud",
"vip98k",
"stepfun-image",
"runway",
"krea",
@ -249,6 +250,7 @@ export function getImageFailoverConvertProvider(providerType?: string | null) {
case "bananaimage":
case "visionary":
case "atlascloud":
case "vip98k":
case "stepfun-image":
case "midjourney":
case "runway":

1
app/(chat)/api/chat/_lib/image-media.ts
View File

@ -527,6 +527,7 @@ export async function convertMessagesWithImages(
normalizedProviderType === "bananaimage" ||
normalizedProviderType === "visionary" ||
normalizedProviderType === "atlascloud" ||
normalizedProviderType === "vip98k" ||
normalizedProviderType === "stepfun-image" ||
normalizedProviderType === "midjourney" ||
normalizedProviderType === "runway" ||

1
app/(chat)/api/chat/_lib/stream.ts
View File

@ -190,6 +190,7 @@ function normalizeImageLoadingProviderType(providerType?: string | null): string
normalized === "bananaimage" ||
normalized === "visionary" ||
normalized === "atlascloud" ||
normalized === "vip98k" ||
normalized === "stepfun-image" ||
normalized === "midjourney" ||
normalized === "runway" ||

63
app/(chat)/api/chat/route.ts
View File

@ -5813,6 +5813,7 @@ export async function POST(request: Request) {
Array.from(replacementReferences.entries()),
AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT,
async ([referenceIndex, reference]) => {
const itemStartedAt = Date.now();
if (reference.passthrough === true) {
const referenceSourceImageUrls = Array.from(
new Set([
@ -5833,6 +5834,12 @@ export async function POST(request: Request) {
streamLabel: "agent-mode:image:detail-replacement:item-passthrough",
});
successCount += 1;
chatApiLogger.info("[Chat API] Direct detail replacement item passthrough completed:", {
referenceIndex: referenceIndex + 1,
totalReferences: replacementReferences.length,
referenceLabel: reference.label,
elapsedMs: Date.now() - itemStartedAt,
});
return;
}
@ -5849,6 +5856,7 @@ export async function POST(request: Request) {
);
try {
const planningStartedAt = Date.now();
const replacementExecutionPlan = await analyzeReplacementExecutionPlan({
modeType: "detail_replacement",
replacementStrategy: detailReplacementStrategy,
@ -5863,10 +5871,12 @@ export async function POST(request: Request) {
planningRuntime: replacementScenePlanningRuntime,
models: replacementVisionModels,
});
const planningElapsedMs = Date.now() - planningStartedAt;
chatApiLogger.info("[Chat API] Direct detail replacement reference decision:", {
referenceIndex: referenceIndex + 1,
totalReferences: replacementReferences.length,
referenceLabel: reference.label,
planningElapsedMs,
shouldReplace: replacementExecutionPlan.shouldReplace,
reason: replacementExecutionPlan.reason,
detectionSource: replacementExecutionPlan.detectionSource,
@ -5976,7 +5986,12 @@ export async function POST(request: Request) {
hasPromptPackBlock: replacementPrompt.includes("【必须生效的前置场景词包】"),
});
let generationStartedAt = 0;
let generationElapsedMs: number | null = null;
let normalizeStartedAt = 0;
let normalizeElapsedMs: number | null = null;
try {
generationStartedAt = Date.now();
const result = await generateCanvasImage({
prompt: replacementPrompt,
negativePrompt: effectiveImageParams?.negativePrompt,
@ -6016,6 +6031,7 @@ export async function POST(request: Request) {
}
: undefined,
});
generationElapsedMs = Date.now() - generationStartedAt;
resolvedModelDisplayName = result.model || resolvedModelDisplayName;
const firstImage = result.images[0];
if (!firstImage) {
@ -6025,6 +6041,7 @@ export async function POST(request: Request) {
let finalImage = firstImage;
let finalImageSizeLabel: string = resolvedImageSize;
try {
normalizeStartedAt = Date.now();
const normalizedImage = await normalizeReplacementResultImage({
generatedImageUrl: firstImage.originalUrl,
generatedThumbnailUrl: firstImage.thumbnailUrl,
@ -6038,7 +6055,11 @@ export async function POST(request: Request) {
thumbnailUrl: normalizedImage.thumbnailUrl || normalizedImage.originalUrl,
};
finalImageSizeLabel = normalizedImage.imageSize;
normalizeElapsedMs = Date.now() - normalizeStartedAt;
} catch (normalizeError) {
if (normalizeStartedAt > 0 && normalizeElapsedMs === null) {
normalizeElapsedMs = Date.now() - normalizeStartedAt;
}
chatApiLogger.warn("[Chat API] Failed to normalize direct detail replacement output:", {
referenceLabel: reference.label,
referenceImageUrl: reference.imageUrl,
@ -6056,6 +6077,22 @@ export async function POST(request: Request) {
}
}
chatApiLogger.info("[Chat API] Direct detail replacement image generation completed:", {
referenceIndex: referenceIndex + 1,
totalReferences: replacementReferences.length,
referenceLabel: reference.label,
modelDisplayName: resolvedModelDisplayName,
resultImageSize: finalImageSizeLabel,
sourceImageCount: sourceImageUrls.length,
generationAspectRatio,
pageKind: replacementExecutionPlan.scenePlan.pageKind,
generationRoute: replacementExecutionPlan.scenePlan.generationRoute,
promptPackPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy,
elapsedMs: Date.now() - itemStartedAt,
planningElapsedMs,
generationElapsedMs,
normalizeElapsedMs,
});
await emitReplacementResultMessage({
reference,
image: finalImage,
@ -6067,9 +6104,16 @@ export async function POST(request: Request) {
});
successCount += 1;
} catch (replacementItemError) {
if (generationStartedAt > 0 && generationElapsedMs === null) {
generationElapsedMs = Date.now() - generationStartedAt;
}
chatApiLogger.error("[Chat API] Direct detail replacement item failed:", {
referenceIndex,
referenceLabel: reference.label,
elapsedMs: Date.now() - itemStartedAt,
planningElapsedMs,
generationElapsedMs,
normalizeElapsedMs,
error: replacementItemError,
});
await emitReplacementResultMessage({
@ -6093,6 +6137,7 @@ export async function POST(request: Request) {
chatApiLogger.error("[Chat API] Direct detail replacement item failed before generation:", {
referenceIndex,
referenceLabel: reference.label,
elapsedMs: Date.now() - itemStartedAt,
error: replacementPlanError,
});
await emitReplacementResultMessage({
@ -9151,6 +9196,7 @@ export async function POST(request: Request) {
bananaimage: 7,
visionary: 7,
atlascloud: 7,
vip98k: 7,
"stepfun-image": 7,
"grsai-gpt-image": 8,
runway: 9,
@ -9545,6 +9591,7 @@ export async function POST(request: Request) {
"bananaimage",
"visionary",
"atlascloud",
"vip98k",
"stepfun-image",
"midjourney",
"runway",
@ -9773,6 +9820,8 @@ export async function POST(request: Request) {
? "visionary"
: (languageModel as any).provider === "atlascloud"
? "atlascloud"
: (languageModel as any).provider === "vip98k"
? "vip98k"
: (languageModel as any).provider === "stepfun-image"
? "stepfun-image"
: (languageModel as any).provider === "midjourney"
@ -9879,6 +9928,7 @@ export async function POST(request: Request) {
bananaimage: 7,
visionary: 7,
atlascloud: 7,
vip98k: 7,
"stepfun-image": 7,
runway: 8,
krea: 9,
@ -9928,6 +9978,8 @@ export async function POST(request: Request) {
? "visionary"
: candidate.provider.type === "atlascloud"
? "atlascloud"
: candidate.provider.type === "vip98k"
? "vip98k"
: candidate.provider.type === "stepfun-image"
? "stepfun-image"
: candidate.provider.type === "midjourney"
@ -10317,6 +10369,7 @@ export async function POST(request: Request) {
bananaimage: 6,
visionary: 6,
atlascloud: 6,
vip98k: 6,
"stepfun-image": 6,
runway: 7,
krea: 8,
@ -10367,6 +10420,8 @@ export async function POST(request: Request) {
? "visionary"
: candidate.provider.type === "atlascloud"
? "atlascloud"
: candidate.provider.type === "vip98k"
? "vip98k"
: candidate.provider.type === "stepfun-image"
? "stepfun-image"
: candidate.provider.type === "midjourney"
@ -10580,7 +10635,7 @@ export async function POST(request: Request) {
// ========== 千问/FAL/RUXA 视频生成独立处理 ==========
if (
["qianwen-video", "fal-video", "ruxa-video", "runway", "krea", "bltcy-video"].includes((languageModel as any).provider) &&
["qianwen-video", "fal-video", "ruxa-video", "runway", "krea", "bltcy-video", "vip98k"].includes((languageModel as any).provider) &&
typeof (languageModel as any).doGenerate === "function"
) {
const currentVideoProvider = (languageModel as any).provider as
@ -10589,7 +10644,8 @@ export async function POST(request: Request) {
| "ruxa-video"
| "runway"
| "krea"
| "bltcy-video";
| "bltcy-video"
| "vip98k";
chatApiLogger.info(`[Chat API] Using ${currentVideoProvider} video generation`);
const videoGenerationStream = createUIMessageStream({
@ -11199,6 +11255,7 @@ export async function POST(request: Request) {
bananaimage: 7,
visionary: 7,
atlascloud: 7,
vip98k: 7,
"stepfun-image": 7,
runway: 8,
krea: 9,
@ -11251,6 +11308,8 @@ export async function POST(request: Request) {
? "visionary"
: candidate.provider.type === "atlascloud"
? "atlascloud"
: candidate.provider.type === "vip98k"
? "vip98k"
: candidate.provider.type === "stepfun-image"
? "stepfun-image"
: candidate.provider.type === "midjourney"

6
app/api/admin/ai-models/route.ts
View File

@ -33,6 +33,11 @@ const imageGenConfigSchema = z.object({
supportsNegativePrompt: z.boolean().optional(),
supportsWatermark: z.boolean().optional(),
supportsSeed: z.boolean().optional(),
supportsQuality: z.boolean().optional(),
supportedAspectRatios: z.array(z.string().min(1).max(32)).max(32).optional(),
supportedImageSizes: z.array(z.string().min(1).max(32)).max(32).optional(),
maxReferenceImages: z.number().min(0).max(64).optional(),
maxVideoReferences: z.number().min(0).max(16).optional(),
ruxaTaskInputMode: z
.enum(["size-image", "resolution-based"])
.optional(),
@ -45,6 +50,7 @@ const imageGenConfigSchema = z.object({
creditCostPerVideoByResolution: z.record(z.string(), z.number().min(0).max(100000)).optional(),
enableLoadingPlaceholder: z.boolean().optional(),
loadingColor: z.string().optional(),
vip98k: z.record(z.string(), z.unknown()).optional(),
}).optional();
const capabilitiesSchema = z.record(z.string(), z.unknown()).nullable().optional();

13
app/api/admin/ai-providers/route.ts
View File

@ -9,6 +9,7 @@ import {
} from "@/lib/db/queries";
import { ChatSDKError } from "@/lib/errors";
import { BLTCY_VIDEO_BASE_URL, isBltcyVideoBaseUrl } from "@/lib/ai/bltcy-video-models";
import { VIP98K_BASE_URL } from "@/lib/ai/vip98k-models";
import { clearScopedSiteSettingsCache } from "@/lib/site-settings/public-cache";
const providerTypes = [
@ -35,6 +36,7 @@ const providerTypes = [
"bananaimage",
"visionary",
"atlascloud",
"vip98k",
"stepfun-image",
"midjourney",
"fal-video",
@ -79,11 +81,12 @@ const OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1";
const BANANAIMAGE_BASE_URL = "https://bananaimage.aigenmedia.art";
const VISIONARY_BASE_URL = "https://api.visionary.beer";
const ATLASCLOUD_BASE_URL = "https://api.atlascloud.ai";
const VIP98K_PROVIDER_BASE_URL = VIP98K_BASE_URL;
const STEPFUN_IMAGE_BASE_URL = "https://api.stepfun.com";
const BFL_IMAGE_BASE_URL = "https://api.bfl.ai";
const MIDJOURNEY_BASE_URL = "https://your-midjourney-api.example.com";
const TRIPO_3D_BASE_URL = "https://api.tripo3d.ai/v2/openapi";
const WORLDLABS_3D_BASE_URL = "https://api.worldlabs.ai";
const WORLDLABS_3D_BASE_URL = "https://api.vip98k.com";
const KITTYCAD_CAD_BASE_URL = "https://api.zoo.dev";
const RUNWAY_BASE_URL = "https://api.dev.runwayml.com";
const KREA_BASE_URL = "https://api.krea.ai";
@ -249,6 +252,8 @@ export async function POST(request: Request) {
? data.baseUrl || VISIONARY_BASE_URL
: data.type === "atlascloud"
? data.baseUrl || ATLASCLOUD_BASE_URL
: data.type === "vip98k"
? data.baseUrl || VIP98K_PROVIDER_BASE_URL
: data.type === "stepfun-image"
? data.baseUrl || STEPFUN_IMAGE_BASE_URL
: data.type === "midjourney"
@ -311,6 +316,12 @@ export async function PUT(request: Request) {
...data,
apiKey: normalizedApiKey,
type: normalizeProviderTypeForStorage(data.type),
baseUrl:
data.baseUrl !== undefined
? data.baseUrl
: data.type === "vip98k"
? VIP98K_PROVIDER_BASE_URL
: undefined,
updatedBy: session.user.id,
});

185
app/api/admin/vip98k-models/route.ts Normal file
View File

@ -0,0 +1,185 @@
import { and, eq } from "drizzle-orm";
import { NextResponse } from "next/server";
import {
fetchVip98kModelPresets,
type Vip98kModelPreset,
} from "@/lib/ai/vip98k-models";
import { db } from "@/lib/db";
import { getAiProviderById } from "@/lib/db/queries";
import { aiModel, type AiModel } from "@/lib/db/schema";
import { requireAdmin } from "@/lib/middleware/admin";
export const runtime = "nodejs";
export const maxDuration = 300;
function summarizeByType(presets: Vip98kModelPreset[]) {
return presets.reduce<Record<string, number>>((acc, preset) => {
const type = String(preset.type || "model");
acc[type] = (acc[type] || 0) + 1;
return acc;
}, {});
}
export async function GET(request: Request) {
try {
await requireAdmin();
const { searchParams } = new URL(request.url);
const providerId = String(searchParams.get("providerId") || "").trim();
if (!providerId) {
return NextResponse.json(
{ error: "Provider ID is required to fetch VIP98K catalog" },
{ status: 400 },
);
}
const provider = await getAiProviderById({ id: providerId });
if (!provider || provider.type !== "vip98k") {
return NextResponse.json(
{ error: "Provider must be a VIP98K provider" },
{ status: 400 },
);
}
const presets = await fetchVip98kModelPresets({
baseURL: provider.baseUrl,
apiKey: provider.apiKey,
});
return NextResponse.json({
presets,
counts: {
total: presets.length,
byType: summarizeByType(presets),
},
});
} catch (error) {
console.error("[VIP98K Models API] GET error:", error);
return NextResponse.json(
{
error:
error instanceof Error
? error.message
: "Failed to fetch VIP98K models",
},
{ status: 500 },
);
}
}
export async function POST(request: Request) {
try {
await requireAdmin();
const body = await request.json();
const providerId = String(body?.providerId || "").trim();
const requestedModelIds = Array.isArray(body?.modelIds)
? new Set(
body.modelIds.map((item: unknown) => String(item || "").trim()).filter(Boolean),
)
: null;
if (!providerId) {
return NextResponse.json({ error: "Provider ID is required" }, { status: 400 });
}
const provider = await getAiProviderById({ id: providerId });
if (!provider || provider.type !== "vip98k") {
return NextResponse.json(
{ error: "Provider must be a VIP98K provider" },
{ status: 400 },
);
}
const catalogPresets = await fetchVip98kModelPresets({
baseURL: provider.baseUrl,
apiKey: provider.apiKey,
});
const presets = requestedModelIds
? catalogPresets.filter((preset) => requestedModelIds.has(preset.modelId))
: catalogPresets;
const addedModels: AiModel[] = [];
const updatedModels: AiModel[] = [];
const errors: Array<{ modelId: string; error: string }> = [];
for (const preset of presets) {
try {
const existing = await db.query.aiModel.findFirst({
where: and(
eq(aiModel.providerId, providerId),
eq(aiModel.modelId, preset.modelId),
),
});
const values = {
displayName: preset.displayName,
avatarUrl: null,
description: preset.description,
supportsVision: preset.supportsVision,
supportsImageGen: preset.supportsImageGen,
isEnabled: true,
isHidden: preset.isHidden,
sortOrder: preset.sortOrder,
capabilities: preset.capabilities,
defaultParams: preset.defaultParams,
imageGenConfig: preset.imageGenConfig,
updatedAt: new Date(),
};
if (existing) {
const [updated] = await db
.update(aiModel)
.set(values)
.where(
and(
eq(aiModel.providerId, providerId),
eq(aiModel.modelId, preset.modelId),
),
)
.returning();
if (updated) updatedModels.push(updated);
} else {
const [created] = await db
.insert(aiModel)
.values({
providerId,
modelId: preset.modelId,
...values,
isDefault: false,
isSubModel: false,
parentModelId: null,
})
.returning();
if (created) addedModels.push(created);
}
} catch (error) {
console.error(`[VIP98K Models API] Failed to sync ${preset.modelId}:`, error);
errors.push({
modelId: preset.modelId,
error: error instanceof Error ? error.message : "Sync failed",
});
}
}
return NextResponse.json({
success: true,
addedCount: addedModels.length,
updatedCount: updatedModels.length,
addedModels,
updatedModels,
errors: errors.length > 0 ? errors : undefined,
counts: {
requested: presets.length,
total: catalogPresets.length,
byType: summarizeByType(catalogPresets),
},
});
} catch (error) {
console.error("[VIP98K Models API] POST error:", error);
return NextResponse.json(
{
error:
error instanceof Error
? error.message
: "Failed to sync VIP98K models",
},
{ status: 500 },
);
}
}

60
app/api/canvas/[id]/route.ts
View File

@ -181,26 +181,56 @@ export async function PUT(
})
: undefined;
// 创建新画布（使用指定 ID）
const nextWorkspaceType = workspaceType || "canvas";
const nextCanvasData = syncedCanvasData || {
nodes: [],
edges: [],
viewport: { x: 0, y: 0, zoom: 1 },
};
const nextTitle = normalizeCanvasProjectTitle(
title,
getDefaultWorkspaceTitle(nextWorkspaceType)
);
const insertData: any = {
id, // 使用指定的 ID
userId: session.user.id,
title: nextTitle,
workspaceType: nextWorkspaceType,
canvasData: nextCanvasData,
};
const conflictUpdateData: any = {
title: nextTitle,
workspaceType: nextWorkspaceType,
canvasData: nextCanvasData,
updatedAt: new Date(),
};
if (description !== undefined) {
insertData.description = description;
conflictUpdateData.description = description;
}
if (thumbnail !== undefined) {
insertData.thumbnail = thumbnail;
conflictUpdateData.thumbnail = thumbnail;
}
// 创建新画布（使用指定 ID），并兼容同一用户并发创建同一画布的场景。
const [newProject] = await db
.insert(canvasProject)
.values({
id, // 使用指定的 ID
userId: session.user.id,
title: normalizeCanvasProjectTitle(
title,
getDefaultWorkspaceTitle(workspaceType || "canvas")
),
description,
workspaceType: workspaceType || "canvas",
canvasData: syncedCanvasData || {
nodes: [],
edges: [],
viewport: { x: 0, y: 0, zoom: 1 },
},
.values(insertData)
.onConflictDoUpdate({
target: canvasProject.id,
set: conflictUpdateData,
where: eq(canvasProject.userId, session.user.id),
})
.returning();
if (!newProject) {
return NextResponse.json(
{ success: false, error: '画布ID已被占用' },
{ status: 409 }
);
}
return NextResponse.json({ success: true, data: newProject });
}
} catch (error) {

10
app/api/canvas/generate/image/route.ts
View File

@ -38,6 +38,8 @@ function toGenerateImagePayload(body: GenerateImageRequest): CanvasGenerateImage
: "";
return {
prompt: String(body.prompt || "").trim(),
displayPrompt:
typeof body.displayPrompt === "string" ? body.displayPrompt : undefined,
negativePrompt:
typeof body.negativePrompt === "string" ? body.negativePrompt : undefined,
model: typeof body.model === "string" ? body.model : undefined,
@ -65,6 +67,14 @@ function toGenerateImagePayload(body: GenerateImageRequest): CanvasGenerateImage
referenceImages: Array.isArray(body.referenceImages)
? body.referenceImages.map((item) => String(item || "").trim()).filter(Boolean)
: [],
debugLabel:
typeof body.debugLabel === "string" ? body.debugLabel : undefined,
debugMeta:
body.debugMeta &&
typeof body.debugMeta === "object" &&
!Array.isArray(body.debugMeta)
? body.debugMeta
: undefined,
};
}

116
app/api/canvas/layout-replace/route.ts Normal file
View File

@ -0,0 +1,116 @@
import { NextResponse } from "next/server";
import { auth } from "@/app/(auth)/auth";
import {
createCanvasImageTask,
enqueueCanvasImageTask,
} from "@/lib/canvas/image-task-queue";
import { getPointsSystemConfig } from "@/lib/db/queries";
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
import { getImageToolCreditCost } from "@/lib/points/config";
import type { CanvasLayoutReplaceTaskRequest } from "@/lib/canvas/layout-replace-task";
export const maxDuration = 1200;
function normalizeHttpUrl(value: unknown) {
const url = String(value || "").trim();
return /^https?:\/\//i.test(url) ? url : "";
}
export async function POST(request: Request) {
try {
const session = await auth();
if (!session?.user?.id) {
return NextResponse.json(
{ success: false, error: "未登录" },
{ status: 401 },
);
}
const body = await request.json().catch(() => ({}));
const sourceImageUrl = normalizeHttpUrl(body?.sourceImageUrl);
const referenceImageUrl = normalizeHttpUrl(body?.referenceImageUrl);
if (!sourceImageUrl) {
return NextResponse.json(
{ success: false, error: "缺少当前产品图片" },
{ status: 400 },
);
}
if (!referenceImageUrl) {
return NextResponse.json(
{ success: false, error: "缺少排版参考图" },
{ status: 400 },
);
}
const pointsConfig = await getPointsSystemConfig();
const pointsCharge = await prepareFeaturePointsCharge({
userId: session.user.id,
unitCost: getImageToolCreditCost(pointsConfig, "repaint"),
reason: "image_repaint",
modelId: "canvas-layout-replace",
note: "排版替换消耗积分",
pointsConfig,
});
if (!pointsCharge.ok) {
return NextResponse.json(pointsCharge.payload, {
status: pointsCharge.status,
});
}
const taskRequest: CanvasLayoutReplaceTaskRequest = {
taskKind: "canvas-layout-replace",
sourceImageUrl,
referenceImageUrl,
supplementalPrompt:
typeof body?.supplementalPrompt === "string"
? body.supplementalPrompt
: undefined,
model: typeof body?.model === "string" ? body.model : undefined,
imageSize:
typeof body?.imageSize === "string" && body.imageSize.trim()
? body.imageSize.trim()
: "2K",
sourceNodeId:
typeof body?.sourceNodeId === "string" ? body.sourceNodeId : undefined,
referenceLabel:
typeof body?.referenceLabel === "string" ? body.referenceLabel : undefined,
debugMeta:
body?.debugMeta &&
typeof body.debugMeta === "object" &&
!Array.isArray(body.debugMeta)
? body.debugMeta
: undefined,
};
const task = await createCanvasImageTask({
userId: session.user.id,
request: taskRequest as unknown as Record<string, unknown>,
});
await enqueueCanvasImageTask(task.id);
if (pointsCharge.chargeable) {
try {
await pointsCharge.consume({
note: `排版替换消耗积分（${task.id}）`,
});
} catch (pointsError) {
console.error("[Points] Failed to consume canvas layout replace points:", pointsError);
}
}
return NextResponse.json({
success: true,
taskId: task.id,
status: task.status,
});
} catch (error) {
console.error("[Canvas Layout Replace] Error:", error);
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : "排版替换提交失败",
},
{ status: 500 },
);
}
}

2
app/api/canvas/local-edit/route.ts
View File

@ -323,6 +323,7 @@ function isScaleAdjustMaskCapableModel(
return false;
}
if (entry.provider.type === "openai-image") return true;
if (entry.provider.type === "vip98k") return true;
if (entry.provider.type === "fal-image") {
return isFalGptImage2EditModelId(entry.model.modelId || "");
}
@ -371,6 +372,7 @@ function getScaleAdjustProviderPriority(providerType: string) {
case "alibaba-image":
case "bfl-image":
case "openai-image":
case "vip98k":
case "stepfun-image":
case "volcano-vision":
return 60;

15
app/api/canvas/text-edit/route.ts
View File

@ -14,6 +14,7 @@ import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
import { createBananaImage } from "@/lib/ai/providers/bananaimage";
import { createVisionary } from "@/lib/ai/providers/visionary";
import { createAtlasCloud } from "@/lib/ai/providers/atlascloud";
import { createVip98k } from "@/lib/ai/providers/vip98k";
import { createStepfunImage } from "@/lib/ai/providers/stepfun-image";
import { createMidjourney } from "@/lib/ai/providers/midjourney";
import { createRunway } from "@/lib/ai/providers/runway";
@ -57,6 +58,7 @@ const SUPPORTED_TEXT_EDIT_PROVIDER_TYPES = new Set([
"bananaimage",
"visionary",
"atlascloud",
"vip98k",
"stepfun-image",
"midjourney",
"runway",
@ -127,6 +129,8 @@ function createTextEditProvider(
return createVisionary({ baseURL, apiKey });
case "atlascloud":
return createAtlasCloud({ baseURL, apiKey });
case "vip98k":
return createVip98k({ baseURL, apiKey });
case "stepfun-image":
return createStepfunImage({ baseURL, apiKey });
case "midjourney":
@ -493,10 +497,17 @@ async function generateTextEditImageWithFailover(params: {
imageCount: 1,
},
};
if (selected.provider.type === "atlascloud") {
if (selected.provider.type === "atlascloud" || selected.provider.type === "vip98k") {
generationOptions.imageGenConfig = selected.model.imageGenConfig || null;
}
const result = await provider.chat(runtimeModelId).doGenerate(generationOptions);
const modelInstance =
selected.provider.type === "vip98k"
? (provider as any).chat(runtimeModelId, {
capabilities: selected.model.capabilities || null,
imageGenConfig: selected.model.imageGenConfig || null,
})
: provider.chat(runtimeModelId);
const result = await modelInstance.doGenerate(generationOptions);
const resultText = String(result?.text || "").trim();
const finishReason = String(result?.finishReason || "").toLowerCase();

398
app/api/canvas/world/tasks/[taskId]/route.ts
View File

@ -7,9 +7,16 @@ import { resolveThreeDModelAliasId } from "@/lib/model3d/model-alias-config";
import { getThreeDModelAliasConfigForRuntime } from "@/lib/model3d/model-alias-config-server";
import { createLogger } from "@/lib/observability/logger";
import { uploadFile } from "@/lib/storage/service";
import { WorldLabsApiError, worldLabsRequest } from "@/lib/worldlabs/client";
import {
isWorldLabsExternalTaskApi,
WORLDLABS_EXTERNAL_TASK_MODEL,
WorldLabsApiError,
worldLabsRequest,
} from "@/lib/worldlabs/client";
import { getWorldLabsProviderConfig } from "@/lib/worldlabs/server";
import type {
WorldLabsExternalTaskResponse,
WorldLabsExternalTaskResultData,
WorldLabsOperation,
WorldLabsWorld,
WorldLabsWorldAssets,
@ -419,6 +426,214 @@ function buildAssetPayload(
};
}
function unwrapExternalTaskData<T extends Record<string, unknown>>(payload: unknown) {
const root = getObjectValue(payload);
const data = getObjectValue(root?.data);
return (data || root || {}) as T;
}
function normalizeExternalTaskStatus(status?: string | null, done?: boolean) {
const normalized = String(status || "").trim().toLowerCase();
if (done === true || ["succeeded", "success", "completed", "done"].includes(normalized)) {
return "success" as const;
}
if (["failed", "failure", "error", "cancelled", "canceled"].includes(normalized)) {
return "failed" as const;
}
return "running" as const;
}
function readStringRecord(value: unknown) {
const record = getObjectValue(value);
if (!record) return null;
const entries = Object.entries(record).filter(
(entry): entry is [string, string] =>
Boolean(entry[0]) && typeof entry[1] === "string" && entry[1].trim().length > 0
);
return entries.length > 0 ? Object.fromEntries(entries) : null;
}
function findNestedString(
value: unknown,
keys: string[],
depth = 0
): string | null {
if (depth > 8) return null;
const record = getObjectValue(value);
if (!record) return null;
for (const key of keys) {
const directValue = getStringValue(record[key]);
if (directValue) return directValue;
}
for (const child of Object.values(record)) {
const found = findNestedString(child, keys, depth + 1);
if (found) return found;
}
return null;
}
function findNestedObject(
value: unknown,
keys: string[],
depth = 0
): Record<string, unknown> | null {
if (depth > 8) return null;
const record = getObjectValue(value);
if (!record) return null;
for (const key of keys) {
const directValue = getObjectValue(record[key]);
if (directValue) return directValue;
}
for (const child of Object.values(record)) {
const found = findNestedObject(child, keys, depth + 1);
if (found) return found;
}
return null;
}
function findNestedStringRecord(
value: unknown,
keys: string[],
depth = 0
): Record<string, string> | null {
if (depth > 8) return null;
const record = getObjectValue(value);
if (!record) return null;
for (const key of keys) {
const directValue = readStringRecord(record[key]);
if (directValue) return directValue;
}
for (const child of Object.values(record)) {
const found = findNestedStringRecord(child, keys, depth + 1);
if (found) return found;
}
return null;
}
function extractExternalTaskErrorMessage(
data: WorldLabsExternalTaskResultData,
payload: WorldLabsExternalTaskResponse<WorldLabsExternalTaskResultData>
) {
const dataError = getObjectValue(data.error);
const payloadError = getObjectValue(payload.error);
return (
getStringValue(
data.errorMessage,
data.message,
dataError?.message,
payload.message,
payloadError?.message,
payload.error
) || "World Labs 外部任务失败"
);
}
function extractExternalWorldAssetPayload(data: WorldLabsExternalTaskResultData) {
const root = getObjectValue(data) || {};
const resultRoot =
getObjectValue(root.result) ||
getObjectValue(root.output) ||
getObjectValue(root.world) ||
root;
const assets = findNestedObject(resultRoot, ["assets"]);
const worldAssets = assets as Partial<WorldLabsWorldAssets> | null;
const splats = getObjectValue(worldAssets?.splats);
return {
worldId: getStringValue(
findNestedString(resultRoot, ["worldId", "world_id"]),
findNestedString(root, ["worldId", "world_id"])
),
worldUrl: getStringValue(
findNestedString(resultRoot, [
"worldUrl",
"world_marble_url",
"sourceWorldUrl",
"viewerUrl",
"marbleUrl",
]),
findNestedString(root, [
"worldUrl",
"world_marble_url",
"sourceWorldUrl",
"viewerUrl",
"marbleUrl",
])
),
displayName: getStringValue(
findNestedString(resultRoot, ["displayName", "display_name"]),
findNestedString(root, ["displayName", "display_name"])
),
model:
getStringValue(
findNestedString(resultRoot, ["model", "modelId", "model_id"]),
findNestedString(root, ["model", "modelId", "model_id"])
) || WORLDLABS_EXTERNAL_TASK_MODEL,
caption: getStringValue(
worldAssets?.caption,
findNestedString(resultRoot, ["caption"]),
findNestedString(root, ["caption"])
),
thumbnailUrl: getStringValue(
worldAssets?.thumbnail_url,
findNestedString(resultRoot, [
"thumbnailUrl",
"thumbnail_url",
"previewImageUrl",
"preview_image_url",
]),
findNestedString(root, [
"thumbnailUrl",
"thumbnail_url",
"previewImageUrl",
"preview_image_url",
])
),
panoUrl: getStringValue(
worldAssets?.imagery?.pano_url,
findNestedString(resultRoot, ["panoUrl", "pano_url", "panoramaUrl", "panorama_url"]),
findNestedString(root, ["panoUrl", "pano_url", "panoramaUrl", "panorama_url"])
),
meshUrl: getStringValue(
worldAssets?.mesh?.collider_mesh_url,
findNestedString(resultRoot, [
"meshUrl",
"mesh_url",
"modelUrl",
"model_url",
"glbUrl",
"glb_url",
"colliderMeshUrl",
"collider_mesh_url",
]),
findNestedString(root, [
"meshUrl",
"mesh_url",
"modelUrl",
"model_url",
"glbUrl",
"glb_url",
"colliderMeshUrl",
"collider_mesh_url",
])
),
spzUrls:
readStringRecord(splats?.spz_urls) ||
findNestedStringRecord(resultRoot, ["spzUrls", "spz_urls"]) ||
findNestedStringRecord(root, ["spzUrls", "spz_urls"]),
semanticsMetadata:
getObjectValue(splats?.semantics_metadata) ||
findNestedObject(resultRoot, ["semanticsMetadata", "semantics_metadata"]) ||
findNestedObject(root, ["semanticsMetadata", "semantics_metadata"]),
};
}
interface RouteParams {
params: Promise<{ taskId: string }>;
}
@ -452,6 +667,187 @@ export async function GET(request: Request, { params }: RouteParams) {
});
providerForLog = config.providerId || null;
const modelAliasConfig = await getThreeDModelAliasConfigForRuntime();
const persistedAssetPayload =
getObjectValue(persistedWorldAsset?.assetPayload) || {};
const shouldUseExternalTaskResultApi =
isWorldLabsExternalTaskApi(config.baseUrl) &&
persistedAssetPayload.externalTaskApi !== false &&
persistedAssetPayload.marbleApi !== true;
if (shouldUseExternalTaskResultApi) {
const externalPayload = await worldLabsRequest<
WorldLabsExternalTaskResponse<WorldLabsExternalTaskResultData>
>(config, {
method: "GET",
path: `/api/external/task/result?taskId=${encodeURIComponent(normalizedTaskId)}`,
timeoutMs: resolveWorldLabsStatusTimeoutMs(),
});
const externalData =
unwrapExternalTaskData<WorldLabsExternalTaskResultData>(externalPayload);
const externalStatusText = getStringValue(externalData.status) || "running";
const externalStatus = normalizeExternalTaskStatus(
externalStatusText,
externalData.done === true
);
const externalAssetPayloadBase = {
...persistedAssetPayload,
providerId: config.providerId || null,
providerName: config.providerName || null,
externalTaskApi: true,
externalTaskStatus: externalStatusText,
requestId: getStringValue(externalPayload.requestId, externalPayload.request_id),
};
if (externalStatus === "running") {
return NextResponse.json({
success: true,
data: {
taskId: normalizedTaskId,
type: "world",
status: "running",
progress: null,
output: externalData,
assetPayload: externalAssetPayloadBase,
},
});
}
if (externalStatus === "failed") {
const errorMessage = extractExternalTaskErrorMessage(
externalData,
externalPayload
);
await updatePersistedWorldAsset({
id: persistedWorldAsset?.id,
status: "failed",
progress: null,
error: errorMessage,
assetPayload: externalAssetPayloadBase,
});
logger.warn("external task failed status", {
elapsedMs: Date.now() - startedAt,
taskIdLength: normalizedTaskId.length,
providerId: config.providerId || null,
error: errorMessage,
});
return NextResponse.json({
success: true,
data: {
taskId: normalizedTaskId,
type: "world",
status: "failed",
progress: null,
errorMessage,
output: externalData,
assetPayload: externalAssetPayloadBase,
},
});
}
const extractedAssetPayload = extractExternalWorldAssetPayload(externalData);
const rawMeshUrl = extractedAssetPayload.meshUrl;
const rawThumbnailUrl = extractedAssetPayload.thumbnailUrl;
const rawPanoUrl = extractedAssetPayload.panoUrl;
const stableWorldUrl: string | null = null;
const stableModelUrl = await archiveRemoteAssetIfNeeded({
cacheKey: `worldlabs:${normalizedTaskId}:mesh`,
sourceUrl: rawMeshUrl,
filenamePrefix: `worldlabs-mesh-${normalizedTaskId}`,
fallbackExtension: "glb",
validateAs: "glb",
});
const stableThumbnailUrl = await archiveRemoteAssetIfNeeded({
cacheKey: `worldlabs:${normalizedTaskId}:thumb`,
sourceUrl: rawThumbnailUrl,
filenamePrefix: `worldlabs-thumb-${normalizedTaskId}`,
fallbackExtension: "jpg",
});
const stablePanoUrl = await archiveRemoteAssetIfNeeded({
cacheKey: `worldlabs:${normalizedTaskId}:pano`,
sourceUrl: rawPanoUrl,
filenamePrefix: `worldlabs-pano-${normalizedTaskId}`,
fallbackExtension: "jpg",
});
const stableSpzUrls = await archiveRemoteAssetMapIfNeeded({
cacheKeyPrefix: `worldlabs:${normalizedTaskId}:spz`,
sourceMap: extractedAssetPayload.spzUrls,
filenamePrefix: `worldlabs-splat-${normalizedTaskId}`,
fallbackExtension: "spz",
validateAs: "spz",
});
const assetPayload = {
...externalAssetPayloadBase,
worldId: extractedAssetPayload.worldId,
worldUrl: stableWorldUrl,
displayName: extractedAssetPayload.displayName,
model: resolveThreeDModelAliasId(
extractedAssetPayload.model,
modelAliasConfig.worldModels
),
caption: extractedAssetPayload.caption,
thumbnailUrl: stableThumbnailUrl || extractedAssetPayload.thumbnailUrl,
panoUrl: stablePanoUrl || extractedAssetPayload.panoUrl,
meshUrl: stableModelUrl,
spzUrls: stableSpzUrls,
semanticsMetadata: extractedAssetPayload.semanticsMetadata,
storageArchivedAt: new Date().toISOString(),
sourceWorldUrl: extractedAssetPayload.worldUrl || null,
sourceMeshUrl:
rawMeshUrl && stableModelUrl && rawMeshUrl !== stableModelUrl
? rawMeshUrl
: null,
sourceThumbnailUrl:
rawThumbnailUrl && stableThumbnailUrl && rawThumbnailUrl !== stableThumbnailUrl
? rawThumbnailUrl
: null,
sourcePanoUrl:
rawPanoUrl && stablePanoUrl && rawPanoUrl !== stablePanoUrl
? rawPanoUrl
: null,
sourceSpzUrls:
extractedAssetPayload.spzUrls && stableSpzUrls !== extractedAssetPayload.spzUrls
? extractedAssetPayload.spzUrls
: null,
};
await updatePersistedWorldAsset({
id: persistedWorldAsset?.id,
status: "success",
progress: 100,
modelUrl: stableModelUrl,
previewImageUrl: stableThumbnailUrl || stablePanoUrl || null,
error: null,
assetPayload,
});
logger.info("external task success status", {
elapsedMs: Date.now() - startedAt,
taskIdLength: normalizedTaskId.length,
providerId: config.providerId || null,
hasMesh: Boolean(stableModelUrl),
hasPano: Boolean(stablePanoUrl),
hasThumbnail: Boolean(stableThumbnailUrl),
hasSplat: Boolean(stableSpzUrls && Object.keys(stableSpzUrls).length > 0),
});
return NextResponse.json({
success: true,
data: {
taskId: normalizedTaskId,
type: "world",
status: "success",
progress: 100,
output: externalData,
modelUrl: stableModelUrl,
previewImageUrl: stableThumbnailUrl || stablePanoUrl || null,
worldId: assetPayload.worldId,
worldUrl: stableWorldUrl,
panoUrl: stablePanoUrl,
caption: assetPayload.caption,
assetPayload,
},
});
}
const operation = await worldLabsRequest<WorldLabsOperation>(config, {
method: "GET",
path: `/marble/v1/operations/${encodeURIComponent(normalizedTaskId)}`,

250
app/api/canvas/world/tasks/route.ts
View File

@ -12,13 +12,23 @@ import {
import { getThreeDModelAliasConfigForRuntime } from "@/lib/model3d/model-alias-config-server";
import { createLogger } from "@/lib/observability/logger";
import { get3DWorldCreditCost } from "@/lib/points/config";
import { WorldLabsApiError, worldLabsRequest } from "@/lib/worldlabs/client";
import {
isWorldLabsExternalTaskApi,
WORLDLABS_EXTERNAL_TASK_MODEL,
WorldLabsApiError,
worldLabsRequest,
} from "@/lib/worldlabs/client";
import {
getWorldLabsProviderConfig,
getWorldLabsProviderConfigs,
reportWorldLabsProviderRuntimeFailure,
} from "@/lib/worldlabs/server";
import type { WorldLabsModel, WorldLabsOperation } from "@/lib/worldlabs/types";
import type {
WorldLabsExternalTaskCreateData,
WorldLabsExternalTaskResponse,
WorldLabsModel,
WorldLabsOperation,
} from "@/lib/worldlabs/types";
export const runtime = "nodejs";
export const maxDuration = 180;
@ -47,8 +57,10 @@ type CreateWorldTaskRequest = {
makePublic?: boolean;
reconstructImages?: boolean;
mediaAssetId?: string;
sourceUrl?: string;
mediaAssets?: Array<{
mediaAssetId?: string;
sourceUrl?: string;
azimuth?: number | null;
}>;
};
@ -58,10 +70,89 @@ function readOptionalTrimmedText(input: unknown) {
return value || null;
}
function readOptionalHttpUrl(input: unknown) {
const value = readOptionalTrimmedText(input);
if (!value) return null;
try {
const parsed = new URL(value);
return parsed.protocol === "http:" || parsed.protocol === "https:"
? parsed.toString()
: null;
} catch {
return null;
}
}
function getObjectValue(value: unknown) {
return value && typeof value === "object" ? (value as Record<string, unknown>) : null;
}
function unwrapExternalTaskData<T extends Record<string, unknown>>(payload: unknown) {
const root = getObjectValue(payload);
const data = getObjectValue(root?.data);
return (data || root || {}) as T;
}
function normalizeExternalTaskStatus(status?: string | null, done?: boolean) {
const normalized = String(status || "").trim().toLowerCase();
if (done === true || ["succeeded", "success", "completed", "done"].includes(normalized)) {
return "success" as const;
}
if (["failed", "failure", "error", "cancelled", "canceled"].includes(normalized)) {
return "failed" as const;
}
return "running" as const;
}
function normalizeExternalTaskCreateOperation(
payload: WorldLabsExternalTaskResponse<WorldLabsExternalTaskCreateData>
): WorldLabsOperation {
const data = unwrapExternalTaskData<WorldLabsExternalTaskCreateData>(payload);
const taskId = readOptionalTrimmedText(data.taskId || data.task_id || data.id);
const status = readOptionalTrimmedText(data.status) || "accepted";
const normalizedStatus = normalizeExternalTaskStatus(status, data.done === true);
return {
operation_id: taskId || undefined,
done: normalizedStatus === "success",
error:
normalizedStatus === "failed"
? {
message:
readOptionalTrimmedText(
(payload as Record<string, unknown>)?.message ||
(payload as Record<string, unknown>)?.error
) || "World Labs 外部任务创建失败",
}
: null,
metadata: {
externalTaskApi: true,
status,
resultPath: readOptionalTrimmedText(data.resultPath || data.result_path),
requestId: readOptionalTrimmedText(payload.requestId || payload.request_id),
},
response: data,
};
}
function shouldCharge3DPoints(mode: WorldTaskMode) {
return mode === "text" || mode === "image" || mode === "multi-image" || mode === "video";
}
function shouldUseExternalTaskCreateApi(
config: Awaited<ReturnType<typeof getWorldLabsProviderConfig>>,
mode: WorldTaskMode
) {
return isWorldLabsExternalTaskApi(config.baseUrl) && mode === "text";
}
function resolveOfficialMarbleModel(model?: string | null) {
const normalized = readOptionalTrimmedText(model);
if (!normalized || normalized === WORLDLABS_EXTERNAL_TASK_MODEL) {
return "marble-1.1";
}
return normalized;
}
function buildWorldPrompt(body: CreateWorldTaskRequest, mode: WorldTaskMode) {
const textPrompt = readOptionalTrimmedText(body.prompt);
@ -80,15 +171,21 @@ function buildWorldPrompt(body: CreateWorldTaskRequest, mode: WorldTaskMode) {
if (mode === "image") {
const mediaAssetId = readOptionalTrimmedText(body.mediaAssetId);
if (!mediaAssetId) {
throw new Error("图生世界缺少 mediaAssetId");
const sourceUrl = readOptionalHttpUrl(body.sourceUrl);
if (!mediaAssetId && !sourceUrl) {
throw new Error("图生世界缺少 mediaAssetId 或 sourceUrl");
}
return {
type: "image",
image_prompt: {
source: "media_asset",
media_asset_id: mediaAssetId,
},
image_prompt: mediaAssetId
? {
source: "media_asset",
media_asset_id: mediaAssetId,
}
: {
source: "uri",
uri: sourceUrl,
},
...(textPrompt ? { text_prompt: textPrompt } : {}),
...(body.disableRecaption !== undefined
? { disable_recaption: body.disableRecaption }
@ -102,7 +199,8 @@ function buildWorldPrompt(body: CreateWorldTaskRequest, mode: WorldTaskMode) {
const multiImagePrompt = mediaAssets
.map((item) => {
const mediaAssetId = readOptionalTrimmedText(item?.mediaAssetId);
if (!mediaAssetId) return null;
const sourceUrl = readOptionalHttpUrl(item?.sourceUrl);
if (!mediaAssetId && !sourceUrl) return null;
const azimuth =
item?.azimuth === null || item?.azimuth === undefined
? null
@ -111,10 +209,15 @@ function buildWorldPrompt(body: CreateWorldTaskRequest, mode: WorldTaskMode) {
: null;
return {
...(azimuth === null ? {} : { azimuth }),
content: {
source: "media_asset",
media_asset_id: mediaAssetId,
},
content: mediaAssetId
? {
source: "media_asset",
media_asset_id: mediaAssetId,
}
: {
source: "uri",
uri: sourceUrl,
},
};
})
.filter(Boolean);
@ -137,16 +240,22 @@ function buildWorldPrompt(body: CreateWorldTaskRequest, mode: WorldTaskMode) {
}
const mediaAssetId = readOptionalTrimmedText(body.mediaAssetId);
if (!mediaAssetId) {
throw new Error("视频生成世界缺少 mediaAssetId");
const sourceUrl = readOptionalHttpUrl(body.sourceUrl);
if (!mediaAssetId && !sourceUrl) {
throw new Error("视频生成世界缺少 mediaAssetId 或 sourceUrl");
}
return {
type: "video",
video_prompt: {
source: "media_asset",
media_asset_id: mediaAssetId,
},
video_prompt: mediaAssetId
? {
source: "media_asset",
media_asset_id: mediaAssetId,
}
: {
source: "uri",
uri: sourceUrl,
},
...(textPrompt ? { text_prompt: textPrompt } : {}),
};
}
@ -187,12 +296,29 @@ export async function POST(request: Request) {
);
modelForLog = resolvedWorldModel || "marble-1.1";
const preferredProviderId = readOptionalTrimmedText(body.providerId);
const displayName =
readOptionalTrimmedText(body.displayName) ||
readOptionalTrimmedText(body.prompt) ||
`World ${new Date().toLocaleString("zh-CN")}`;
const worldPrompt = buildWorldPrompt(body, mode);
const candidates = preferredProviderId
? [await getWorldLabsProviderConfig(preferredProviderId)]
: await getWorldLabsProviderConfigs();
if (candidates.length === 0) {
throw new Error("未找到可用的 World Labs Key");
}
const officialMarbleModel = resolveOfficialMarbleModel(resolvedWorldModel);
const predictedBillingModel = candidates.some((candidate) =>
shouldUseExternalTaskCreateApi(candidate, mode)
)
? WORLDLABS_EXTERNAL_TASK_MODEL
: officialMarbleModel;
const pointsConfig = await getPointsSystemConfig();
const shouldChargePoints = shouldCharge3DPoints(mode);
const unitCost = get3DWorldCreditCost(
pointsConfig,
resolvedWorldModel || "marble-1.1",
);
const unitCost = get3DWorldCreditCost(pointsConfig, predictedBillingModel);
if (
shouldChargePoints &&
@ -220,42 +346,51 @@ export async function POST(request: Request) {
}
}
const preferredProviderId = readOptionalTrimmedText(body.providerId);
const displayName =
readOptionalTrimmedText(body.displayName) ||
readOptionalTrimmedText(body.prompt) ||
`World ${new Date().toLocaleString("zh-CN")}`;
const worldPrompt = buildWorldPrompt(body, mode);
const candidates = preferredProviderId
? [await getWorldLabsProviderConfig(preferredProviderId)]
: await getWorldLabsProviderConfigs();
if (candidates.length === 0) {
throw new Error("未找到可用的 World Labs Key");
}
let result: WorldLabsOperation | null = null;
let usedProvider = null as Awaited<
ReturnType<typeof getWorldLabsProviderConfig>
> | null;
let usedExternalTaskApi = false;
let lastError: unknown = null;
for (const candidate of candidates) {
try {
result = await worldLabsRequest<WorldLabsOperation>(candidate, {
method: "POST",
path: "/marble/v1/worlds:generate",
timeoutMs: resolveWorldLabsCreateTimeoutMs(),
json: {
display_name: displayName,
model: resolvedWorldModel || "marble-1.1",
permission: {
public: body.makePublic === true,
const useExternalTaskApi = shouldUseExternalTaskCreateApi(candidate, mode);
if (useExternalTaskApi) {
const prompt = readOptionalTrimmedText(body.prompt);
if (!prompt) {
throw new Error("请输入世界描述");
}
const externalResult = await worldLabsRequest<
WorldLabsExternalTaskResponse<WorldLabsExternalTaskCreateData>
>(candidate, {
method: "POST",
path: "/api/external/task/generate",
timeoutMs: resolveWorldLabsCreateTimeoutMs(),
json: {
model: WORLDLABS_EXTERNAL_TASK_MODEL,
displayName,
prompt,
},
world_prompt: worldPrompt,
},
});
});
result = normalizeExternalTaskCreateOperation(externalResult);
} else {
result = await worldLabsRequest<WorldLabsOperation>(candidate, {
method: "POST",
path: "/marble/v1/worlds:generate",
timeoutMs: resolveWorldLabsCreateTimeoutMs(),
json: {
display_name: displayName,
model: officialMarbleModel,
permission: {
public: body.makePublic === true,
},
world_prompt: worldPrompt,
},
});
}
usedProvider = candidate;
usedExternalTaskApi = useExternalTaskApi;
providerForLog = candidate.providerId || null;
break;
} catch (error) {
@ -292,12 +427,18 @@ export async function POST(request: Request) {
if (shouldChargePoints && pointsConfig.enabled && unitCost > 0) {
try {
const usedBillingModel = usedExternalTaskApi
? WORLDLABS_EXTERNAL_TASK_MODEL
: officialMarbleModel;
const usedBillingDisplayName = usedBillingModel === WORLDLABS_EXTERNAL_TASK_MODEL
? WORLDLABS_EXTERNAL_TASK_MODEL
: worldModelDisplayName;
await consumeUserPoints({
userId: session.user.id,
amount: unitCost,
reason: "world3d_generation",
modelId: `worldlabs-3d:${resolvedWorldModel || "marble-1.1"}`,
note: `3D世界生成消耗积分（${worldModelDisplayName}）`,
modelId: `worldlabs-3d:${usedBillingModel}`,
note: `3D世界生成消耗积分（${usedBillingDisplayName}）`,
});
} catch (error) {
logger.error("points consume failed", error, {
@ -313,10 +454,13 @@ export async function POST(request: Request) {
elapsedMs: Date.now() - startedAt,
taskIdLength: taskId.length,
mode,
model: resolvedWorldModel || "marble-1.1",
model: usedExternalTaskApi
? WORLDLABS_EXTERNAL_TASK_MODEL
: officialMarbleModel,
providerId: usedProvider.providerId || null,
status: result.done ? "success" : "running",
unitCost,
externalTaskApi: usedExternalTaskApi,
});
return NextResponse.json({
@ -327,6 +471,8 @@ export async function POST(request: Request) {
assetPayload: {
providerId: usedProvider.providerId || null,
providerName: usedProvider.providerName || null,
externalTaskApi: usedExternalTaskApi,
marbleApi: !usedExternalTaskApi,
},
raw: result,
},

117
app/api/canvas/world/upload/route.ts
View File

@ -1,7 +1,11 @@
import { NextResponse } from "next/server";
import { auth } from "@/app/(auth)/auth";
import { createLogger } from "@/lib/observability/logger";
import { WorldLabsApiError, worldLabsRequest } from "@/lib/worldlabs/client";
import { uploadFile } from "@/lib/storage/service";
import {
WorldLabsApiError,
worldLabsRequest,
} from "@/lib/worldlabs/client";
import {
getWorldLabsProviderConfig,
getWorldLabsProviderConfigs,
@ -112,6 +116,81 @@ function normalizeMediaAssetId(data: WorldLabsPrepareUploadResponse) {
return String(data.media_asset?.media_asset_id || data.media_asset?.id || "").trim();
}
function readOptionalString(value: unknown) {
const text = String(value || "").trim();
return text || null;
}
function extractUploadInfoFromCurlExample(curlExample: unknown) {
const curl = String(curlExample || "");
const uploadUrl =
[...curl.matchAll(/https?:\/\/[^\s'"\\]+/g)]
.map((match) => match[0])
.find(Boolean) || null;
const requiredHeaders: Record<string, string> = {};
const headerPattern = /(?:-H|--header)\s+(?:"([^"]+)"|'([^']+)'|([^\s]+))/g;
let headerMatch: RegExpExecArray | null;
while ((headerMatch = headerPattern.exec(curl))) {
const header = String(
headerMatch[1] || headerMatch[2] || headerMatch[3] || ""
).trim();
const colonIndex = header.indexOf(":");
if (colonIndex <= 0) continue;
const name = header.slice(0, colonIndex).trim();
const value = header.slice(colonIndex + 1).trim();
if (name && value) {
requiredHeaders[name] = value;
}
}
return { uploadUrl, requiredHeaders };
}
function resolvePrepareUploadDetails(data: WorldLabsPrepareUploadResponse) {
const uploadInfo = data.upload_info || {};
const uploadInfoRecord = uploadInfo as Record<string, unknown>;
const curlDetails = extractUploadInfoFromCurlExample(uploadInfo.curl_example);
const explicitHeaders =
uploadInfo.required_headers && typeof uploadInfo.required_headers === "object"
? uploadInfo.required_headers
: {};
return {
uploadUrl:
readOptionalString(uploadInfo.upload_url) ||
readOptionalString(uploadInfoRecord.uploadUrl) ||
readOptionalString(uploadInfoRecord.url) ||
curlDetails.uploadUrl,
uploadMethod:
readOptionalString(uploadInfo.upload_method) ||
readOptionalString(uploadInfoRecord.method) ||
"PUT",
requiredHeaders: {
...curlDetails.requiredHeaders,
...explicitHeaders,
},
};
}
function isSignedUploadFailure(error: unknown) {
const message = error instanceof Error ? error.message : String(error || "");
return (
message.includes("SignatureDoesNotMatch") ||
message.includes("Access denied") ||
message.includes("Google secret key") ||
message.includes("签名")
);
}
async function uploadWorldLabsInputToStorage(file: File, extension: string) {
const buffer = await file.arrayBuffer();
const safeExtension = extension && extension !== "bin" ? extension : "bin";
const filename = `worldlabs-input-${Date.now()}-${crypto.randomUUID()}.${safeExtension}`;
const uploaded = await uploadFile(filename, buffer);
return uploaded.url;
}
function isCredentialOrBillingProviderError(error: unknown) {
return (
error instanceof WorldLabsApiError &&
@ -202,24 +281,24 @@ export async function POST(request: Request) {
);
const mediaAssetId = normalizeMediaAssetId(prepareUpload);
const uploadUrl = String(prepareUpload.upload_info?.upload_url || "").trim();
const { uploadUrl, uploadMethod, requiredHeaders } =
resolvePrepareUploadDetails(prepareUpload);
if (!mediaAssetId || !uploadUrl) {
throw new Error("World Labs 返回的上传信息不完整");
}
const uploadHeaders = new Headers();
uploadHeaders.set("Content-Type", file.type || "application/octet-stream");
const requiredHeaders = prepareUpload.upload_info?.required_headers || {};
for (const [headerName, headerValue] of Object.entries(requiredHeaders)) {
if (!headerName || headerValue === undefined || headerValue === null) continue;
uploadHeaders.set(headerName, String(headerValue));
}
if (!uploadHeaders.has("Content-Type") && !prepareUpload.upload_info?.curl_example) {
uploadHeaders.set("Content-Type", file.type || "application/octet-stream");
}
const uploadMethod = String(
prepareUpload.upload_info?.upload_method || "PUT"
).toUpperCase();
const normalizedUploadMethod = uploadMethod.toUpperCase();
const uploadResponse = await fetch(uploadUrl, {
method: uploadMethod === "PUT" ? "PUT" : uploadMethod,
method: normalizedUploadMethod === "PUT" ? "PUT" : normalizedUploadMethod,
headers: uploadHeaders,
body: file,
cache: "no-store",
@ -252,6 +331,28 @@ export async function POST(request: Request) {
});
} catch (error) {
lastError = error;
if (isSignedUploadFailure(error)) {
try {
const sourceUrl = await uploadWorldLabsInputToStorage(file, extension);
logger.warn("worldlabs signed upload failed; using storage source url", {
elapsedMs: Date.now() - startedAt,
providerId: config.providerId || null,
kind: isImage ? "image" : "video",
size: file.size,
});
return NextResponse.json({
success: true,
data: {
sourceUrl,
providerId: config.providerId || null,
providerName: config.providerName || null,
uploadFallback: "storage-source-url",
},
});
} catch (fallbackError) {
lastError = fallbackError;
}
}
if (isCredentialOrBillingProviderError(error)) {
await reportWorldLabsProviderRuntimeFailure(
config.providerId,

52
app/api/scene-fusion/route.ts
View File

@ -28,6 +28,7 @@ import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
import { createBananaImage } from "@/lib/ai/providers/bananaimage";
import { createVisionary } from "@/lib/ai/providers/visionary";
import { createAtlasCloud } from "@/lib/ai/providers/atlascloud";
import { createVip98k } from "@/lib/ai/providers/vip98k";
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
import {
canProcessSceneFusionProviderType,
@ -1565,6 +1566,18 @@ async function processSceneWithProvider(
imageGenConfig,
);
case "vip98k":
return processWithVip98k(
productImageUrl,
sceneImageUrl,
prompt,
provider,
runtimeModelId || modelId,
aspectRatio,
imageSize,
imageGenConfig,
);
case "gemini":
case "gemini-image":
return processWithGemini(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
@ -1933,6 +1946,45 @@ async function processWithAtlasCloud(
return { thumbnail, original, prompt };
}
async function processWithVip98k(
productImageUrl: string,
sceneImageUrl: string,
prompt: string,
provider: { baseUrl: string; apiKey: string },
modelId: string,
aspectRatio?: string,
imageSize?: string,
imageGenConfig?: Record<string, any> | null
): Promise<{ thumbnail: string; original: string; prompt: string }> {
logger.info("provider_processing", { providerType: "vip98k", modelId });
const vip98kProvider = createVip98k({
baseURL: provider.baseUrl,
apiKey: provider.apiKey,
});
const result = await vip98kProvider.chat(modelId, imageGenConfig || null).doGenerate({
messages: [
{
role: "user",
content: [
{ type: "image_url", image_url: { url: productImageUrl } },
{ type: "image_url", image_url: { url: sceneImageUrl } },
{ type: "text", text: prompt },
],
},
],
imageParams: {
aspectRatio: aspectRatio || "auto",
imageSize: imageSize || "2K",
imageCount: 1,
},
imageGenConfig,
});
const { thumbnail, original } = parseImageFromMarkdown(result.text);
return { thumbnail, original, prompt };
}
/**
* 使用既梦/火山模型处理
*/

200
app/canvas/[id]/CanvasEditorLegacyPageClient.tsx
View File

@ -141,6 +141,7 @@ import {
createEmptyAgentGenerationProjectWorkspaceState,
createEmptyAgentWorkflowPersistedState,
normalizeAgentGenerationProjectPersistedState,
normalizeAgentGenerationProjectReplacementResultState,
} from "@/lib/agents/generation-project-state";
import {
createEmptyAgentModeWorkspaceState,
@ -227,6 +228,7 @@ import { canvasSelectors, useCanvasStore } from "@/store/canvas";
import type {
AgentGenerationProject,
AgentGenerationProjectPersistedState,
AgentGenerationProjectReplacementResultState,
AgentGenerationProjectResultCollections,
AgentGenerationProjectWorkspaceAttachment,
AgentGenerationProjectWorkspaceState,
@ -1540,6 +1542,57 @@ function normalizeResultCollectionUrls(urls: string[] | undefined | null) {
);
}
function areReplacementResultStatesEqual(
left: AgentGenerationProjectReplacementResultState | undefined | null,
right: AgentGenerationProjectReplacementResultState | undefined | null
) {
return (
JSON.stringify(normalizeAgentGenerationProjectReplacementResultState(left)) ===
JSON.stringify(normalizeAgentGenerationProjectReplacementResultState(right))
);
}
function getReplacementResultUrlCount(
state: AgentGenerationProjectPersistedState | undefined | null
) {
const normalized = normalizeAgentGenerationProjectReplacementResultState(
state?.replacementResultState
);
return (
Object.values(normalized.detail_replacement || {}).filter((record) =>
Boolean(record.url)
).length +
Object.values(normalized.sku_replacement || {}).filter((record) =>
Boolean(record.url)
).length
);
}
function hasReplacementResultCollectionMismatch(
state: AgentGenerationProjectPersistedState | undefined | null
) {
const normalized = normalizeAgentGenerationProjectReplacementResultState(
state?.replacementResultState
);
const resultCollections = state?.resultCollections || {};
const countModeRecords = (
modeType: "detail_replacement" | "sku_replacement"
) =>
Object.values(normalized[modeType] || {}).filter((record) =>
Boolean(record.url)
).length;
const countModeCollection = (
modeType: "detail_replacement" | "sku_replacement"
) => normalizeResultCollectionUrls(resultCollections[modeType]).length;
return (["detail_replacement", "sku_replacement"] as const).some(
(modeType) => {
const recordCount = countModeRecords(modeType);
return recordCount > 0 && countModeCollection(modeType) !== recordCount;
}
);
}
const AGENT_PROJECT_DRAFT_STORAGE_KEY_PREFIX = "agent-project-draft:";
function getAgentProjectDraftStorageKey(projectId: string) {
@ -2088,6 +2141,10 @@ export default function CanvasEditorLegacyPage({
agentProjectDeletedResultCollections,
setAgentProjectDeletedResultCollections,
] = useState<AgentGenerationProjectResultCollections>({});
const [
agentProjectReplacementResultState,
setAgentProjectReplacementResultState,
] = useState<AgentGenerationProjectReplacementResultState>({});
const [isAgentProjectLoading, setIsAgentProjectLoading] = useState(false);
const [loadedAgentProjectId, setLoadedAgentProjectId] = useState<
string | null
@ -3460,6 +3517,7 @@ export default function CanvasEditorLegacyPage({
setAgentProjectThumbnail(null);
setAgentProjectResultCollections({});
setAgentProjectDeletedResultCollections({});
setAgentProjectReplacementResultState({});
setLoadedAgentProjectId(null);
}
@ -3951,8 +4009,15 @@ export default function CanvasEditorLegacyPage({
...resultCollections.sku_replacement,
])
),
replacementResultState: normalizeAgentGenerationProjectReplacementResultState(
agentProjectReplacementResultState
),
};
}, [agentProjectDeletedResultCollections, agentProjectResultCollections]);
}, [
agentProjectDeletedResultCollections,
agentProjectReplacementResultState,
agentProjectResultCollections,
]);
const buildPersistedAgentProjectState = useCallback(() => {
const persistedMaterials = appearanceRecognitionEnabled
@ -4137,6 +4202,9 @@ export default function CanvasEditorLegacyPage({
setAgentProjectDeletedResultCollections(
persistedState.deletedResultCollections || {}
);
setAgentProjectReplacementResultState(
persistedState.replacementResultState || {}
);
setLoadedAgentProjectId(params.projectId);
},
[]
@ -4311,7 +4379,9 @@ export default function CanvasEditorLegacyPage({
const result = await agentProjectService.updateProject(agentProjectId, {
title: nextTitle,
thumbnail: agentProjectThumbnail,
projectState: buildPersistedAgentProjectState(),
projectState:
persistedAgentProjectStateRef.current ||
buildPersistedAgentProjectState(),
});
if (!result.success) {
@ -4377,9 +4447,27 @@ export default function CanvasEditorLegacyPage({
const serverUpdatedAt =
Number(new Date(result.data.updatedAt).getTime()) || 0;
const serverReplacementResultCount = getReplacementResultUrlCount(
result.data.projectState
);
const localReplacementResultCount = getReplacementResultUrlCount(
localDraft?.projectState
);
const localDraftMissesServerReplacementResults =
serverReplacementResultCount > 0 && localReplacementResultCount === 0;
const localDraftReplacementResultsDifferFromServer =
serverReplacementResultCount > 0 &&
localReplacementResultCount > 0 &&
localReplacementResultCount !== serverReplacementResultCount;
const localDraftHasStaleReplacementCollections =
serverReplacementResultCount > 0 &&
hasReplacementResultCollectionMismatch(localDraft?.projectState);
const shouldUseLocalDraft =
Boolean(localDraft?.projectState) &&
Number(localDraft?.savedAt || 0) > serverUpdatedAt;
Number(localDraft?.savedAt || 0) > serverUpdatedAt &&
!localDraftMissesServerReplacementResults &&
!localDraftReplacementResultsDifferFromServer &&
!localDraftHasStaleReplacementCollections;
if (shouldUseLocalDraft && localDraft?.projectState) {
hydrateAgentProjectSnapshot({
@ -4481,6 +4569,7 @@ export default function CanvasEditorLegacyPage({
}, [
agentModeState,
agentProjectId,
agentProjectReplacementResultState,
agentProjectResultCollections,
agentProjectThumbnail,
agentProjectTitle,
@ -4771,36 +4860,70 @@ export default function CanvasEditorLegacyPage({
const handleAgentResultPreviewUrlsChange = useCallback(
(modeType: AgentGenerationResultModeType, urls: string[]) => {
const normalizedUrls = normalizeResultCollectionUrls(urls);
setAgentProjectResultCollections((current) => {
const nextCollections: AgentGenerationProjectResultCollections = {
...current,
[modeType]: normalizedUrls,
};
if (areResultCollectionsEqual(current, nextCollections)) {
return current;
}
const currentProjectState =
persistedAgentProjectStateRef.current ||
createEmptyAgentGenerationProjectPersistedState();
const currentCollections = currentProjectState.resultCollections || {};
const nextCollections: AgentGenerationProjectResultCollections = {
...currentCollections,
[modeType]: normalizedUrls,
};
if (!areResultCollectionsEqual(currentCollections, nextCollections)) {
const nextProjectState: AgentGenerationProjectPersistedState = {
...persistedAgentProjectStateRef.current,
...currentProjectState,
resultCollections: nextCollections,
resultImageUrls: getAllResultUrlsFromCollections(nextCollections),
};
persistedAgentProjectStateRef.current = nextProjectState;
writeAgentProjectDraftSnapshot(nextProjectState);
}
setAgentProjectThumbnail((currentThumbnail) => {
const nextThumbnail =
normalizedUrls[0] || getFirstResultUrlFromCollections(nextCollections);
return currentThumbnail === nextThumbnail ? currentThumbnail : nextThumbnail;
});
return nextCollections;
setAgentProjectResultCollections((current) =>
areResultCollectionsEqual(current, nextCollections)
? current
: nextCollections
);
setAgentProjectThumbnail((currentThumbnail) => {
const nextThumbnail =
normalizedUrls[0] || getFirstResultUrlFromCollections(nextCollections);
return currentThumbnail === nextThumbnail ? currentThumbnail : nextThumbnail;
});
},
[writeAgentProjectDraftSnapshot]
);
const handleAgentReplacementResultStateChange = useCallback(
(nextState: AgentGenerationProjectReplacementResultState) => {
const normalizedState =
normalizeAgentGenerationProjectReplacementResultState(nextState);
const currentProjectState =
persistedAgentProjectStateRef.current ||
createEmptyAgentGenerationProjectPersistedState();
if (
!areReplacementResultStatesEqual(
currentProjectState.replacementResultState,
normalizedState
)
) {
const nextProjectState: AgentGenerationProjectPersistedState = {
...currentProjectState,
replacementResultState: normalizedState,
};
persistedAgentProjectStateRef.current = nextProjectState;
writeAgentProjectDraftSnapshot(nextProjectState);
}
setAgentProjectReplacementResultState((current) =>
areReplacementResultStatesEqual(current, normalizedState)
? current
: normalizedState
);
},
[writeAgentProjectDraftSnapshot]
);
const handleAgentResultImageDelete = useCallback(
(modeType: AgentGenerationResultModeType, imageUrl: string) => {
const normalizedUrl = String(imageUrl || "").trim();
@ -4834,6 +4957,33 @@ export default function CanvasEditorLegacyPage({
? current
: nextDeletedCollections
);
if (modeType === "detail_replacement" || modeType === "sku_replacement") {
setAgentProjectReplacementResultState((current) => {
const normalizedState =
normalizeAgentGenerationProjectReplacementResultState(current);
const currentModeState = normalizedState[modeType] || {};
const nextModeState = Object.fromEntries(
Object.entries(currentModeState).filter(
([, record]) => record.url !== normalizedUrl
)
);
const nextState: AgentGenerationProjectReplacementResultState = {
...normalizedState,
[modeType]: nextModeState,
};
if (areReplacementResultStatesEqual(normalizedState, nextState)) {
return current;
}
const nextProjectState: AgentGenerationProjectPersistedState = {
...persistedAgentProjectStateRef.current,
replacementResultState: nextState,
};
persistedAgentProjectStateRef.current = nextProjectState;
writeAgentProjectDraftSnapshot(nextProjectState);
return nextState;
});
}
setAgentProjectThumbnail((current) => {
const nextThumbnail =
current === normalizedUrl
@ -4842,7 +4992,11 @@ export default function CanvasEditorLegacyPage({
return current === nextThumbnail ? current : nextThumbnail;
});
},
[agentProjectDeletedResultCollections, agentProjectResultCollections]
[
agentProjectDeletedResultCollections,
agentProjectResultCollections,
writeAgentProjectDraftSnapshot,
]
);
useEffect(() => {
@ -6745,6 +6899,7 @@ export default function CanvasEditorLegacyPage({
deletedResultCollections:
existingPersistedState.deletedResultCollections || {},
resultImageUrls: existingPersistedState.resultImageUrls || [],
replacementResultState: existingPersistedState.replacementResultState || {},
};
const payload = {
@ -10031,10 +10186,13 @@ export default function CanvasEditorLegacyPage({
mode: null,
});
}}
onPersistProject={() => persistAgentProject("manual")}
onReplacementResultStateChange={handleAgentReplacementResultStateChange}
onResultPreviewUrlsChange={handleAgentResultPreviewUrlsChange}
onWorkspaceStateChange={setAgentWorkspaceState}
projectId={agentProjectId}
key={agentProjectId}
replacementResultState={agentProjectReplacementResultState}
resultCollections={agentProjectResultCollections}
selectedAgentPresetId={
isAgentWorkspaceEnabled ? effectiveSelectedAgentPresetId : null

24
app/canvas/[id]/model3d/page.tsx
View File

@ -1970,15 +1970,18 @@ export default function CanvasModel3DPage() {
worldImageFile as File,
"image"
);
if (!uploadResult.success || !uploadResult.data?.mediaAssetId) {
const uploadData = uploadResult.data || null;
const mediaAssetId = uploadData?.mediaAssetId || null;
const sourceUrl = uploadData?.sourceUrl || null;
if (!uploadResult.success || !uploadData || (!mediaAssetId && !sourceUrl)) {
throw new Error(uploadResult.error || "上传世界参考图失败");
}
await submitTask({
provider: "worldlabs-3d",
payload: {
mode: "image",
mediaAssetId: uploadResult.data.mediaAssetId,
providerId: uploadResult.data.providerId,
...(mediaAssetId ? { mediaAssetId } : { sourceUrl }),
providerId: uploadData.providerId,
prompt: normalizedPrompt || undefined,
displayName: worldDisplayName.trim() || undefined,
model: worldModel,
@ -2000,15 +2003,18 @@ export default function CanvasModel3DPage() {
worldVideoFile as File,
"video"
);
if (!uploadResult.success || !uploadResult.data?.mediaAssetId) {
const uploadData = uploadResult.data || null;
const mediaAssetId = uploadData?.mediaAssetId || null;
const sourceUrl = uploadData?.sourceUrl || null;
if (!uploadResult.success || !uploadData || (!mediaAssetId && !sourceUrl)) {
throw new Error(uploadResult.error || "上传世界视频失败");
}
await submitTask({
provider: "worldlabs-3d",
payload: {
mode: "video",
mediaAssetId: uploadResult.data.mediaAssetId,
providerId: uploadResult.data.providerId,
...(mediaAssetId ? { mediaAssetId } : { sourceUrl }),
providerId: uploadData.providerId,
prompt: normalizedPrompt || undefined,
displayName: worldDisplayName.trim() || undefined,
model: worldModel,
@ -2043,11 +2049,13 @@ export default function CanvasModel3DPage() {
}
const mediaAssets = uploads.map(({ slot, result }) => {
if (!result.success || !result.data?.mediaAssetId) {
const mediaAssetId = result.data?.mediaAssetId || null;
const sourceUrl = result.data?.sourceUrl || null;
if (!result.success || (!mediaAssetId && !sourceUrl)) {
throw new Error(result.error || "上传多图世界图片失败");
}
return {
mediaAssetId: result.data.mediaAssetId,
...(mediaAssetId ? { mediaAssetId } : { sourceUrl }),
azimuth: slot.azimuth,
};
});

32
components/canvas/CanvasWorkspace.tsx
View File

@ -27,6 +27,7 @@ import {
Heart,
HelpCircle,
ImagePlus,
LayoutTemplate,
LayoutGrid,
Lightbulb,
Minus,
@ -553,8 +554,14 @@ export function CanvasWorkspace({
canvasToolbarConfig,
"imageLocalEdit"
);
const selectedImageLayoutReplaceItem = getCanvasToolbarItem(
canvasToolbarConfig,
"imageLayoutReplace"
);
// 局部编辑属于主无限画布图片的固定能力，旧配置缺少该 key 时也保持可见。
const selectedImageLocalEditEnabled = true;
// 排版替换同样是主无限画布图片的核心 AI 入口，旧配置缺少该 key 时也保持可见。
const selectedImageLayoutReplaceEnabled = true;
// Store actions
const selectNode = useCanvasStore((state) => state.selectNode);
@ -2367,6 +2374,31 @@ export function CanvasWorkspace({
{selectedImageLocalEditItem.label}
</Button>
) : null}
{selectedImageLayoutReplaceEnabled && selectedImageNodes.length === 1 ? (
<Button
variant={isWorkflowCanvas ? "outline" : "secondary"}
size="sm"
className={cn(
"h-8 gap-1.5 px-3 text-xs shadow-sm border border-border/60",
isWorkflowCanvas &&
"rounded-full border-[#dbe2ee] bg-white text-[#3b4860] shadow-[0_0_0_1px_rgba(219,226,238,0.26)] hover:bg-[#f7f9ff]"
)}
onClick={() =>
requestImageNodeAction(
selectedImageNodes[0].id,
"layout-replace"
)
}
title={selectedImageLayoutReplaceItem.label}
>
<CanvasToolbarIcon
className="h-3.5 w-3.5"
fallback={LayoutTemplate}
item={selectedImageLayoutReplaceItem}
/>
{selectedImageLayoutReplaceItem.label}
</Button>
) : null}
<Button
variant={isWorkflowCanvas ? "outline" : "secondary"}
size="sm"

561
components/canvas/nodes/ImageNode.tsx
View File

@ -18,6 +18,7 @@ import {
Globe2,
ImagePlus,
Layers,
LayoutTemplate,
Loader2,
Link2,
Maximize2,
@ -134,6 +135,7 @@ import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
@ -300,6 +302,7 @@ const AI_CLOUD_IMAGE_ACTIONS = new Set([
"layer-split",
"box-cutout",
"local-edit",
"layout-replace",
"scale-adjust",
"text-edit",
"prompt-regenerate",
@ -650,6 +653,8 @@ const CANVAS_3D_POLL_INTERVAL_MS = 2000;
const CANVAS_3D_MAX_POLL_ATTEMPTS = 360;
const AI_WASH_TASK_POLL_INTERVAL_MS = 2500;
const AI_WASH_TASK_POLL_LIMIT = 80;
const LAYOUT_REPLACE_TASK_POLL_INTERVAL_MS = 2500;
const LAYOUT_REPLACE_TASK_POLL_LIMIT = 240;
const HYDRATED_IMAGE_TASK_RECOVERY_DELAY_MS = 15 * 1000;
const HYDRATED_IMAGE_TASK_POLL_INTERVAL_MS = 2500;
const HYDRATED_IMAGE_TASK_POLL_LIMIT = 240;
@ -795,6 +800,76 @@ async function pollAiWashQuickTask(taskId: string) {
throw new Error("AI洗图任务轮询超时，请稍后重试");
}
async function pollLayoutReplaceTask(taskId: string) {
for (let attempt = 0; attempt < LAYOUT_REPLACE_TASK_POLL_LIMIT; attempt += 1) {
await sleep(LAYOUT_REPLACE_TASK_POLL_INTERVAL_MS);
const taskResult = await imageGenerationService.getImageTask(taskId);
const task = taskResult.data?.task;
const status = String(task?.status || "").toLowerCase();
if (!taskResult.success || !task) {
continue;
}
if (status === "succeeded" || status === "completed") {
const image = task.result?.images?.[0];
const imageUrl = image?.originalUrl || image?.thumbnailUrl || "";
if (!imageUrl) {
throw new Error("排版替换已完成但未返回图片");
}
return {
imageUrl,
thumbnailUrl: image?.thumbnailUrl || imageUrl,
modelId: String(task.result?.model || task.result?.modelRef || "").trim(),
prompt: String(task.result?.prompt || "").trim(),
};
}
if (["failed", "cancelled", "expired", "banned"].includes(status)) {
throw new Error(task.error || "排版替换生成失败");
}
}
throw new Error("排版替换任务等待超时，请稍后重试");
}
async function resolveImageNaturalDimensions(imageUrl: string) {
const source = String(imageUrl || "").trim();
if (!source || typeof window === "undefined") {
throw new Error("参考图地址不可用");
}
return new Promise<{ width: number; height: number }>((resolve, reject) => {
const image = new window.Image();
const cleanup = () => {
window.clearTimeout(timeoutId);
image.onload = null;
image.onerror = null;
};
const timeoutId = window.setTimeout(() => {
cleanup();
reject(new Error("读取参考图尺寸超时"));
}, 8000);
image.onload = () => {
const width = image.naturalWidth || image.width;
const height = image.naturalHeight || image.height;
cleanup();
if (width > 0 && height > 0) {
resolve({ width, height });
return;
}
reject(new Error("参考图尺寸不可用"));
};
image.onerror = () => {
cleanup();
reject(new Error("读取参考图尺寸失败"));
};
image.decoding = "async";
image.src = buildImageProxyUrl(source, { delivery: "proxy" });
});
}
const isTerminal3DFailureStatus = (status?: string | null) =>
["failed", "banned", "expired", "cancelled"].includes(String(status || ""));
@ -1347,6 +1422,10 @@ export function ImageNode({
);
const imageBoxCutoutItem = getCanvasToolbarItem(canvasToolbarConfig, "imageBoxCutout");
const imageLocalEditItem = getCanvasToolbarItem(canvasToolbarConfig, "imageLocalEdit");
const imageLayoutReplaceItem = getCanvasToolbarItem(
canvasToolbarConfig,
"imageLayoutReplace"
);
const imageRemoveWatermarkItem = getCanvasToolbarItem(
canvasToolbarConfig,
"imageRemoveWatermark"
@ -1419,6 +1498,8 @@ export function ImageNode({
// 局部编辑是无限画布图片的核心入口：即使旧后台工具栏配置里还没有该 key，
// 也必须默认展示，避免历史配置把新功能吞掉。
const imageLocalEditEnabled = true;
// 排版替换同样需要在旧配置中默认展示，避免历史配置吞掉新入口。
const imageLayoutReplaceEnabled = true;
const imageLayerSplitEnabledByToolbar = isCanvasToolbarItemEnabled(
canvasToolbarConfig,
"imageLayerSplit"
@ -1484,6 +1565,7 @@ export function ImageNode({
{ key: "textEdit", enabled: imageTextEditEnabled },
{ key: "boxCutout", enabled: imageBoxCutoutEnabled },
{ key: "localEdit", enabled: imageLocalEditEnabled },
{ key: "layoutReplace", enabled: imageLayoutReplaceEnabled },
{ key: "removeWatermark", enabled: imageRemoveWatermarkEnabled },
{ key: "resize", enabled: imageResizeEnabled },
{ key: "multiAngle", enabled: imageMultiAngleEnabled },
@ -1526,6 +1608,7 @@ export function ImageNode({
imageMultiAngleEnabled ||
imagePromptRegenerateEnabled ||
imageTextEditEnabled ||
imageLayoutReplaceEnabled ||
imageExtendEnabled ||
imageCropEnabled ||
imageAdjustEnabled ||
@ -1677,6 +1760,12 @@ export function ImageNode({
DEFAULT_CANVAS_WORLD_MODEL || "world-standard"
);
const [regeneratePrompt, setRegeneratePrompt] = useState("");
const [layoutReplaceDialogOpen, setLayoutReplaceDialogOpen] = useState(false);
const [layoutReplaceReferenceUrl, setLayoutReplaceReferenceUrl] = useState("");
const [layoutReplacePrompt, setLayoutReplacePrompt] = useState("");
const [isUploadingLayoutReplaceReference, setIsUploadingLayoutReplaceReference] =
useState(false);
const [isSubmittingLayoutReplace, setIsSubmittingLayoutReplace] = useState(false);
const [isUploading, setIsUploading] = useState(false);
const [annotatorOpen, setAnnotatorOpen] = useState(false);
const [fullscreenPreviewOpen, setFullscreenPreviewOpen] = useState(false);
@ -6052,6 +6141,23 @@ export function ImageNode({
await detectTextItems(true);
}, [data.src, data.textEditItems, data.textEditSource, detectTextItems]);
const handleOpenLayoutReplace = useCallback(() => {
setCropPanelOpen(false);
setCropInteraction(null);
setImageAdjustPanelOpen(false);
setFlipRotatePanelOpen(false);
setAnglePanelOpen(false);
setTextEditPanelOpen(false);
setNotePanelOpen(false);
setPromptDialogOpen(false);
setAiWashQuickPanelOpen(false);
resetBoxCutoutState();
resetLocalEditState();
setLayoutReplaceReferenceUrl("");
setLayoutReplacePrompt("");
setLayoutReplaceDialogOpen(true);
}, [resetBoxCutoutState, resetLocalEditState]);
useEffect(() => {
if (!imageNodeActionRequest || imageNodeActionRequest.nodeId !== node.id) {
return;
@ -6071,11 +6177,17 @@ export function ImageNode({
if (imageNodeActionRequest.action === "local-edit") {
handleEnterLocalEditMode();
return;
}
if (imageNodeActionRequest.action === "layout-replace") {
handleOpenLayoutReplace();
}
}, [
clearImageNodeActionRequest,
handleEnterBoxCutoutMode,
handleEnterLocalEditMode,
handleOpenLayoutReplace,
handleOpenTextEdit,
imageNodeActionRequest,
node.id,
@ -6365,6 +6477,248 @@ export function ImageNode({
e.target.value = "";
};
const handleLayoutReplaceReferenceUpload = useCallback(
async (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.currentTarget.files?.[0];
event.currentTarget.value = "";
if (!file) return;
if (!file.type.startsWith("image/")) {
toast.error("请上传图片文件");
return;
}
setIsUploadingLayoutReplaceReference(true);
try {
const uploaded = await uploadChatAttachmentBlob(file, file.name);
setLayoutReplaceReferenceUrl(uploaded.url);
toast.success("参考图已上传");
} catch (error) {
toast.error(error instanceof Error ? error.message : "参考图上传失败");
} finally {
setIsUploadingLayoutReplaceReference(false);
}
},
[]
);
const handleSubmitLayoutReplace = useCallback(async () => {
const sourceUrl = String(data.src || "").trim();
const referenceUrl = layoutReplaceReferenceUrl.trim();
if (!sourceUrl) {
toast.error("请先上传图片");
return;
}
if (isCanvasImageUploadPending(node) || !isHttpImageUrl(sourceUrl)) {
toast.error("图片仍在本地，上传完成后再进行 AI 处理");
return;
}
if (!isHttpImageUrl(referenceUrl)) {
toast.error("请先上传或填写可访问的参考图地址");
return;
}
if (isProcessing || isSubmittingLayoutReplace) {
toast.info("正在处理中，请稍候...");
return;
}
const actionName = "排版替换";
const resultTitle = `${displayTitle}-${actionName}`;
const displayPrompt = `${actionName}处理中...`;
const createdAt = new Date().toISOString();
setLayoutReplaceDialogOpen(false);
setIsCreatingDetachedResultNode(true);
setIsProcessing(true);
setIsSubmittingLayoutReplace(true);
setAiWashQuickPanelOpen(false);
const newNodeId = addNode({
type: "image",
position: {
x: node.position.x + node.size.width + 20,
y: node.position.y,
},
size: { width: node.size.width, height: node.size.height },
data: {
src: "",
status: "generating",
prompt: displayPrompt,
title: resultTitle,
originalSrc: sourceUrl,
metadata: {
editMode: "layout-replace",
sourceNodeId: node.id,
layoutReferenceUrl: referenceUrl,
layoutReplaceStartedAt: createdAt,
},
},
connections: [node.id],
});
selectNode(newNodeId);
let aspectRatio = guessNearestAspectRatio(node.size.width, node.size.height);
let referenceDimensions: { width: number; height: number } | null = null;
try {
try {
referenceDimensions = await resolveImageNaturalDimensions(referenceUrl);
aspectRatio = guessNearestAspectRatio(
referenceDimensions.width,
referenceDimensions.height
);
const resultSize = fitSizeWithinBox({
maxHeight: Math.max(
320,
Math.min(1800, Math.round(Math.max(node.size.height, node.size.width) * 3.2))
),
maxWidth: Math.max(160, Math.min(900, Math.round(node.size.width))),
minHeight: 120,
minWidth: 120,
sourceHeight: referenceDimensions.height,
sourceWidth: referenceDimensions.width,
});
updateNode(newNodeId, {
size: resultSize,
data: {
src: "",
status: "generating",
prompt: displayPrompt,
title: resultTitle,
originalSrc: sourceUrl,
metadata: {
editMode: "layout-replace",
sourceNodeId: node.id,
layoutReferenceUrl: referenceUrl,
layoutReferenceWidth: referenceDimensions.width,
layoutReferenceHeight: referenceDimensions.height,
layoutAspectRatio: aspectRatio,
layoutReplaceStartedAt: createdAt,
},
},
});
} catch (dimensionError) {
console.warn("[Canvas Layout Replace] 读取参考图尺寸失败:", dimensionError);
}
const submitResult = await imageGenerationService.layoutReplace({
sourceImageUrl: sourceUrl,
referenceImageUrl: referenceUrl,
supplementalPrompt: layoutReplacePrompt,
imageSize: "2K",
sourceNodeId: node.id,
referenceLabel: "画布排版参考图",
debugMeta: {
layoutReferenceUrl: referenceUrl,
layoutReferenceWidth: referenceDimensions?.width,
layoutReferenceHeight: referenceDimensions?.height,
layoutAspectRatio: aspectRatio,
},
});
if (!submitResult.success) {
throw new Error(submitResult.error || "排版替换提交失败");
}
const taskId = submitResult.data?.taskId;
if (!taskId) {
throw new Error("排版替换未返回任务ID");
}
updateNode(newNodeId, {
data: {
src: "",
status: "generating",
prompt: displayPrompt,
title: resultTitle,
originalSrc: sourceUrl,
metadata: {
editMode: "layout-replace",
sourceNodeId: node.id,
taskId,
layoutReferenceUrl: referenceUrl,
layoutReferenceWidth: referenceDimensions?.width,
layoutReferenceHeight: referenceDimensions?.height,
layoutAspectRatio: aspectRatio,
layoutReplaceStartedAt: createdAt,
},
},
});
const completed = await pollLayoutReplaceTask(taskId);
if (!completed?.imageUrl) {
throw new Error("排版替换未返回结果图片");
}
updateNode(newNodeId, {
data: {
src: completed.imageUrl,
status: "completed",
prompt: `${actionName}结果`,
title: resultTitle,
originalSrc: sourceUrl,
model: completed.modelId || undefined,
metadata: {
editMode: "layout-replace",
sourceNodeId: node.id,
taskId,
thumbnailUrl: completed.thumbnailUrl || completed.imageUrl,
layoutReferenceUrl: referenceUrl,
layoutReferenceWidth: referenceDimensions?.width,
layoutReferenceHeight: referenceDimensions?.height,
layoutAspectRatio: aspectRatio,
normalizedImageSize: referenceDimensions
? `${referenceDimensions.width}x${referenceDimensions.height}`
: undefined,
layoutReplaceStartedAt: createdAt,
layoutReplaceCompletedAt: new Date().toISOString(),
},
},
});
toast.success("排版替换结果已生成到画布");
} catch (error) {
const message = error instanceof Error ? error.message : "排版替换失败";
console.error("[Canvas Layout Replace] failed:", error);
updateNode(newNodeId, {
data: {
src: "",
status: "failed",
prompt: `${actionName}失败`,
title: `${resultTitle}-失败`,
originalSrc: sourceUrl,
metadata: {
editMode: "layout-replace",
sourceNodeId: node.id,
error: message,
layoutReferenceUrl: referenceUrl,
layoutReferenceWidth: referenceDimensions?.width,
layoutReferenceHeight: referenceDimensions?.height,
layoutAspectRatio: aspectRatio,
layoutReplaceStartedAt: createdAt,
},
},
});
toast.error(message);
} finally {
setIsCreatingDetachedResultNode(false);
setIsProcessing(false);
setIsSubmittingLayoutReplace(false);
}
}, [
addNode,
data.src,
displayTitle,
isProcessing,
isSubmittingLayoutReplace,
layoutReplacePrompt,
layoutReplaceReferenceUrl,
node,
selectNode,
updateNode,
]);
// 获取操作的中文名称
const getActionName = (action: string): string => {
const names: Record<string, string> = {
@ -6385,6 +6739,7 @@ export function ImageNode({
"prompt-regenerate": "重绘",
"box-cutout": "框选抠图",
"local-edit": "局部编辑",
"layout-replace": "排版替换",
};
return names[action] || action;
};
@ -6804,6 +7159,10 @@ export function ImageNode({
handleEnterLocalEditMode();
return;
}
if (action === "layout-replace") {
handleOpenLayoutReplace();
return;
}
if (action === "scale-adjust") {
handleEnterLocalEditMode(undefined, "scale_adjust");
return;
@ -8113,20 +8472,9 @@ export function ImageNode({
setIsCreatingDetachedResultNode(true);
setIsProcessing(true);
try {
const uploadResult = await worldLabsGenerationService.uploadMediaFromUrl(
sourceImageUrl,
"image"
);
if (!uploadResult.success || !uploadResult.data?.mediaAssetId) {
throw new Error(uploadResult.error || "上传世界参考图失败");
}
const providerId = uploadResult.data.providerId || null;
const providerName = uploadResult.data.providerName || null;
const createResult = await worldLabsGenerationService.createTask({
mode: "image",
mediaAssetId: uploadResult.data.mediaAssetId,
providerId,
sourceUrl: sourceImageUrl,
prompt: promptText || undefined,
displayName: displayTitle || promptText || "画布图生世界",
model: modelVersion,
@ -8139,8 +8487,6 @@ export function ImageNode({
const createAssetPayload = {
...(createResult.data.assetPayload || {}),
providerId,
providerName,
sourceImageUrl,
};
updateNode(newNodeId, {
@ -8182,8 +8528,8 @@ export function ImageNode({
void pollCanvasWorld3DTask({
nodeId: newNodeId,
taskId: createResult.data.taskId,
providerId,
providerName,
providerId: createAssetPayload.providerId || null,
providerName: createAssetPayload.providerName || null,
promptText,
modelVersion,
sourceImageUrl,
@ -9662,6 +10008,23 @@ export function ImageNode({
{imageTextEditItem.label}
</DropdownMenuItem>
) : null}
{imageLayoutReplaceEnabled ? (
<DropdownMenuItem
onClick={() => handleAction("layout-replace")}
>
{isSubmittingLayoutReplace ? (
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
) : (
<CanvasToolbarIcon
className="mr-2 h-4 w-4"
fallback={LayoutTemplate}
item={imageLayoutReplaceItem}
/>
)}
<span>{imageLayoutReplaceItem.label}</span>
<ImageActionCreditBadge label={localEditCostShortLabel} />
</DropdownMenuItem>
) : null}
{imageExtendEnabled ? (
<DropdownMenuItem onClick={() => handleAction("extend")}>
<CanvasToolbarIcon
@ -9813,6 +10176,127 @@ export function ImageNode({
prompt={regeneratePrompt}
/>
<Dialog
onOpenChange={(open) => {
if (!isSubmittingLayoutReplace) {
setLayoutReplaceDialogOpen(open);
}
}}
open={layoutReplaceDialogOpen}
>
<DialogContent className="sm:max-w-[760px]">
<DialogHeader>
<DialogTitle className="flex items-center gap-2 text-base">
<LayoutTemplate className="h-4 w-4" />
排版替换
</DialogTitle>
<DialogDescription className="sr-only">
选择版式参考图，保留参考图文字图标排版并仅替换产品主体
</DialogDescription>
</DialogHeader>
<div className="grid gap-4 md:grid-cols-2">
<div className="space-y-2">
<Label>当前图片</Label>
<div className="flex aspect-square items-center justify-center overflow-hidden rounded-lg border border-slate-200 bg-slate-50 dark:border-white/10 dark:bg-zinc-900">
{data.src ? (
<img
alt={displayTitle}
className="max-h-full max-w-full object-contain"
draggable={false}
src={buildImageProxyUrl(data.src, { width: 560 })}
/>
) : null}
</div>
</div>
<div className="space-y-2">
<Label htmlFor={`layout-replace-reference-url-${node.id}`}>
参考图
</Label>
<Input
id={`layout-replace-reference-url-${node.id}`}
onChange={(event) =>
setLayoutReplaceReferenceUrl(event.currentTarget.value)
}
placeholder="https://..."
value={layoutReplaceReferenceUrl}
/>
<Input
accept="image/jpeg,image/png,image/webp,image/heic,image/heif,.heic,.heif"
disabled={isUploadingLayoutReplaceReference}
onChange={(event) => {
void handleLayoutReplaceReferenceUpload(event);
}}
type="file"
/>
<div className="flex aspect-square items-center justify-center overflow-hidden rounded-lg border border-dashed border-slate-200 bg-slate-50 dark:border-white/10 dark:bg-zinc-900">
{isUploadingLayoutReplaceReference ? (
<Loader2 className="h-5 w-5 animate-spin text-muted-foreground" />
) : isHttpImageUrl(layoutReplaceReferenceUrl.trim()) ? (
<img
alt="排版参考图"
className="max-h-full max-w-full object-contain"
draggable={false}
src={buildImageProxyUrl(layoutReplaceReferenceUrl.trim(), {
width: 560,
})}
/>
) : (
<LayoutTemplate className="h-6 w-6 text-slate-300" />
)}
</div>
</div>
</div>
<div className="space-y-2">
<Label htmlFor={`layout-replace-prompt-${node.id}`}>补充要求</Label>
<Textarea
className="min-h-[92px]"
id={`layout-replace-prompt-${node.id}`}
onChange={(event) =>
setLayoutReplacePrompt(event.currentTarget.value)
}
placeholder="可选，只写产品替换细节；不会改写参考图文字、图标和排版"
value={layoutReplacePrompt}
/>
</div>
<DialogFooter>
<Button
disabled={isSubmittingLayoutReplace}
onClick={() => setLayoutReplaceDialogOpen(false)}
type="button"
variant="outline"
>
取消
</Button>
<Button
disabled={
isProcessing ||
isSubmittingLayoutReplace ||
isUploadingLayoutReplaceReference ||
!isHttpImageUrl(layoutReplaceReferenceUrl.trim())
}
onClick={() => void handleSubmitLayoutReplace()}
type="button"
>
{isSubmittingLayoutReplace ? (
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
) : (
<LayoutTemplate className="mr-2 h-4 w-4" />
)}
开始生成
<ImageActionCreditBadge
className="ml-1 bg-white/18 text-white"
compact
label={localEditCostShortLabel}
/>
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
<ThreeDGenerationDialog
costLabel={
threedGenerationDialogMode === "world"
@ -9972,6 +10456,7 @@ export function ImageNode({
!isEditingTitle &&
!sceneFusionDialogOpen &&
!promptDialogOpen &&
!layoutReplaceDialogOpen &&
!threedGenerationDialogMode &&
!notePanelOpen &&
!boxCutoutMode &&
@ -10369,7 +10854,7 @@ export function ImageNode({
</Button>
) : null}
{isSelectedImageActionInline("localEdit") ? (
{isSelectedImageActionInline("localEdit") ? (
<Button
className="group relative h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
disabled={isProcessing || isSubmittingLocalEdit}
@ -10390,6 +10875,31 @@ export function ImageNode({
</Button>
) : null}
{isSelectedImageActionInline("layoutReplace") ? (
<Button
className="group relative h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
disabled={isProcessing || isSubmittingLayoutReplace}
onClick={() => handleAction("layout-replace")}
title={imageLayoutReplaceItem.label}
variant="ghost"
>
{isSubmittingLayoutReplace ? (
<Loader2 className="mr-1.5 h-4 w-4 animate-spin" />
) : (
<CanvasToolbarIcon
className="mr-1.5 h-4 w-4"
fallback={LayoutTemplate}
item={imageLayoutReplaceItem}
/>
)}
{imageLayoutReplaceItem.label}
<ImageActionCreditBadge
compact
label={localEditCostShortLabel}
/>
</Button>
) : null}
{isSelectedImageActionInline("removeWatermark") ? (
<Button
className="group relative h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
@ -10664,6 +11174,23 @@ export function ImageNode({
<ImageActionCreditBadge label={localEditCostShortLabel} />
</DropdownMenuItem>
) : null}
{isSelectedImageActionInMore("layoutReplace") ? (
<DropdownMenuItem
onClick={() => handleAction("layout-replace")}
>
{isSubmittingLayoutReplace ? (
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
) : (
<CanvasToolbarIcon
className="mr-2 h-4 w-4"
fallback={LayoutTemplate}
item={imageLayoutReplaceItem}
/>
)}
<span>{imageLayoutReplaceItem.label}</span>
<ImageActionCreditBadge label={localEditCostShortLabel} />
</DropdownMenuItem>
) : null}
{isSelectedImageActionInMore("removeWatermark") ? (
<DropdownMenuItem onClick={() => handleAction("watermark")}>
<CanvasToolbarIcon

11
components/canvas/nodes/image-node/image-node-local-edit.ts
View File

@ -61,6 +61,17 @@ export const createLocalEditRegionId = () =>
export const createLocalEditReferenceImageId = () =>
`local-edit-ref-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
export const hasScaleAdjustTargetChanged = (
sourceRect: NormalizedCropRect,
targetRect: NormalizedCropRect | null | undefined
) => {
if (!targetRect) return false;
const minimumNormalizedChange = 0.001;
return (["x", "y", "width", "height"] as const).some(
(key) => Math.abs(targetRect[key] - sourceRect[key]) >= minimumNormalizedChange
);
};
export const normalizeLocalEditRect = (value: unknown): NormalizedCropRect | null => {
if (!value || typeof value !== "object") return null;
const rect = value as Partial<NormalizedCropRect>;

1
components/image-generation-loading.tsx
View File

@ -29,6 +29,7 @@ const PROVIDER_MODEL_LABEL: Record<string, string> = {
bananaimage: "BananaImage 图片",
visionary: "Visionary 图片",
atlascloud: "Atlas Cloud 图片",
vip98k: "VIP98K 图片",
"stepfun-image": "StepFun 图片",
midjourney: "Midjourney",
"volcano-vision": "即梦 4.5",

1
components/image-params-bar.tsx
View File

@ -177,6 +177,7 @@ export function ImageParamsBar({
providerType === "bananaimage" ||
providerType === "visionary" ||
providerType === "atlascloud" ||
providerType === "vip98k" ||
providerType === "stepfun-image" ||
providerType === "midjourney"
) {

343
docs/worldlabs-proxy-provider-requirements.md
View File

@ -1,316 +1,93 @@
# World Labs 3D 世界生成中转接口兼容要求
# 3D 世界生成 API 调用规范
本文档用于说明我方系统接入 World Labs 3D 世界生成功能时，对第三方中转服务的接口兼容要求。
## 核心原则
我方系统会按照 World Labs 官方 API 协议发起请求：
当前 3D 世界生成通过智枢外部任务接口调用 World Labs 能力。Base URL 只填写根地址：
```text
{baseUrl} + /marble/v1/...
https://api.vip98k.com
```
中转服务需要兼容官方路径、请求体、响应体和错误格式。请不要要求我方把 `baseUrl` 配置成某个完整接口地址。
不要填写 `/v1`，也不要填写旧的 `/marble/v1`。
正确示例：
## 鉴权方式
```text
http://your-proxy-domain.com
```
错误示例：
```text
http://your-proxy-domain.com/marble/v1/worlds:generate
```
## 我方请求方式
我方后台会在请求中携带：
所有请求都必须携带本站发出的智枢 API Key：
```http
WLT-Api-Key: {配置的 API Key}
Authorization: Bearer <API_KEY>
Accept: application/json
Content-Type: application/json
```
中转服务应支持 `WLT-Api-Key` 请求头。如果中转服务内部自行管理官方 Key，也可以将该字段作为租户标识或鉴权凭据，但不应要求我方改成其他 header。
不要使用管理员 Key、其他平台 Key，也不要使用旧的 `WLT-Api-Key` 作为主链路鉴权。
## 必须兼容的接口
## 模型目录
### 1. 准备上传素材
正式创建任务前，可以先确认这把 Key 是否开通空间生成分组：
```http
POST {baseUrl}/marble/v1/media-assets:prepare_upload
```bash
curl "https://api.vip98k.com/api/external/task/models" \
-H "Authorization: Bearer <API_KEY>"
```
请求体示例：
返回内容里应能找到：
```json
{
"file_name": "input.jpg",
"extension": "jpg",
"kind": "image",
"metadata": {
"mime_type": "image/jpeg",
"uploaded_from": "canvas_world_mode"
"modelId": "world-generation"
}
```
如果找不到，说明这把 Key 没有勾选空间生成分组。
## 创建任务
```bash
curl "https://api.vip98k.com/api/external/task/generate" \
-X POST \
-H "Authorization: Bearer <API_KEY>" \
-H "Content-Type: application/json" \
-d '{
"model": "world-generation",
"displayName": "Test scene",
"prompt": "A quiet sunlit courtyard with a small stone fountain."
}'
```
正常会先返回 `accepted` 或 `running`，这不是失败：
```json
{
"data": {
"taskId": "任务ID",
"status": "accepted",
"done": false,
"resultPath": "/api/external/task/result?taskId=任务ID"
}
}
```
响应必须包含可用的上传信息：
## 查询结果
```bash
curl "https://api.vip98k.com/api/external/task/result?taskId=<任务ID>" \
-H "Authorization: Bearer <API_KEY>"
```
建议每隔 5 到 10 秒查询一次，最长等待 10 分钟。直到返回：
```json
{
"media_asset": {},
"upload_info": {
"upload_url": "https://...",
"upload_method": "PUT",
"required_headers": {}
"data": {
"status": "succeeded",
"done": true
}
}
```
要求：
## 常见错误
- `upload_url` 必须可由我方服务器直接访问。
- `upload_method` 如为 `PUT`，必须支持 `PUT` 上传文件二进制。
- `required_headers` 中返回的 header，我方会原样带上。
- 上传成功时，上传地址必须返回 `2xx`。
### 2. 创建世界生成任务
```http
POST {baseUrl}/marble/v1/worlds:generate
```
请求体示例：
```json
{
"display_name": "",
"model": "marble-1.1",
"permission": {
"public": false
},
"world_prompt": {
"type": "image",
"image_prompt": {
"source": "media_asset",
"media_asset_id": "media_asset_id_here"
}
}
}
```
响应需兼容官方 operation 结构，至少包含：
```json
{
"operation_id": "task-id",
"done": false,
"metadata": {
"progress": {
"status": "RUNNING"
}
}
}
```
要求：
- `operation_id` 必须稳定，后续查询状态必须使用同一个 id。
- 不要在创建成功但任务未完成时返回业务失败。
- 如果余额不足、Key 无效，应返回明确 HTTP 状态码和错误信息。
### 3. 查询任务状态
```http
GET {baseUrl}/marble/v1/operations/{operation_id}
```
运行中响应示例：
```json
{
"operation_id": "task-id",
"done": false,
"metadata": {
"progress": {
"status": "RUNNING",
"description": "World generation in progress"
}
}
}
```
成功响应示例：
```json
{
"operation_id": "task-id",
"done": true,
"error": null,
"metadata": {
"progress": {
"status": "SUCCEEDED"
},
"world_id": "world-id"
},
"response": {
"world_id": "world-id",
"world_marble_url": "https://marble.worldlabs.ai/world/...",
"assets": {
"mesh": {
"collider_mesh_url": "https://cdn.marble.worldlabs.ai/.../model.glb"
},
"imagery": {
"pano_url": "https://cdn.marble.worldlabs.ai/.../rgb_0.png"
},
"thumbnail_url": "https://cdn.marble.worldlabs.ai/.../thumbnail.webp",
"splats": {
"spz_urls": {
"100k": "https://cdn.marble.worldlabs.ai/...100k.spz",
"150k": "https://cdn.marble.worldlabs.ai/...150k.spz",
"500k": "https://cdn.marble.worldlabs.ai/...500k.spz",
"full_res": "https://cdn.marble.worldlabs.ai/...full.spz"
}
}
}
}
}
```
失败响应示例：
```json
{
"operation_id": "task-id",
"done": true,
"error": {
"message": "Insufficient account balance"
}
}
```
要求：
- `done=false` 表示任务仍在运行。
- `done=true` 且 `error=null` 表示任务成功。
- `done=true` 且 `error.message` 存在表示任务失败。
- 不要把运行中的任务返回为 HTTP 404。
- 不要因为任务未完成就返回 `Operation not found`。
### 4. 查询世界详情
```http
GET {baseUrl}/marble/v1/worlds/{world_id}
```
当 `operations` 成功响应中资产不完整时，我方会调用该接口补充详情。响应应包含与 `response` 内 world 对象兼容的数据结构。
## 错误码要求
建议按以下语义返回：
| 场景 | HTTP 状态码 | message 示例 |
| --- | --- | --- |
| API Key 缺失或无效 | `401` | `Unauthorized` |
| 权限不足 | `403` | `Forbidden` |
| 余额不足 | `402` 或 `403` | `Insufficient account balance` |
| 请求体错误 | `400` | `Invalid request body` |
| 任务不存在 | `404` | `Operation not found` |
| 上游超时 | `504` | `Upstream timeout` |
| 上游异常 | `502` | `Upstream error` |
响应体建议：
```json
{
"message": "Insufficient account balance"
}
```
或：
```json
{
"error": {
"message": "Insufficient account balance"
}
}
```
## 超时与稳定性要求
- `prepare_upload` 建议 30 秒内返回。
- `worlds:generate` 建议 60 秒内返回 operation，而不是等待生成完成。
- `operations/{id}` 建议 10 秒内返回当前状态。
- 生成任务本身可异步运行，前端会轮询状态。
- 同一个 `operation_id` 多次查询必须返回同一个任务状态，不应随机切换账号或路由。
## 上传地址要求
`prepare_upload` 返回的 `upload_url` 可以是官方签名地址，也可以是中转服务自己的上传地址，但必须满足：
- 我方服务器可直接访问。
- 支持返回的 `upload_method`。
- 支持上传原始文件二进制。
- 上传成功返回 `2xx`。
- 上传完成后，后续 `worlds:generate` 使用的 `media_asset_id` 必须可用。
## 安全要求
- 不要在错误信息中返回完整官方 API Key。
- 日志中建议只记录 Key 前后少量字符，例如 `sk-****abcd`。
- 中转服务如保存官方 Key，应加密存储。
- 上传 URL 如有签名，应设置合理过期时间。
- 不要允许用户通过 `baseUrl` 或 path 注入访问内网地址。
## 我方验收用例
中转服务上线前，请至少通过以下链路：
1. `POST /marble/v1/media-assets:prepare_upload` 返回 `media_asset_id` 和可上传 URL。
2. 使用返回的 `upload_url` 上传图片，返回 `2xx`。
3. `POST /marble/v1/worlds:generate` 返回稳定 `operation_id`。
4. `GET /marble/v1/operations/{operation_id}` 在运行中返回 `done=false`。
5. 任务完成后返回 `done=true`、`error=null`、`world_id` 和资产 URL。
6. 资产 URL 中至少应包含 thumbnail、pano、mesh 或 spz 中的一种，最好完整返回。
7. `GET /marble/v1/worlds/{world_id}` 可返回完整 world 详情。
## 配置示例
我方系统中的 API 地址应填写中转根地址：
```text
http://23.169.168.133:8080
```
最终我方会请求：
```text
http://23.169.168.133:8080/marble/v1/media-assets:prepare_upload
http://23.169.168.133:8080/marble/v1/worlds:generate
http://23.169.168.133:8080/marble/v1/operations/{operation_id}
http://23.169.168.133:8080/marble/v1/worlds/{world_id}
```
如果切回官方接口，配置为：
```text
https://api.worldlabs.ai
```
最终请求：
```text
https://api.worldlabs.ai/marble/v1/media-assets:prepare_upload
https://api.worldlabs.ai/marble/v1/worlds:generate
https://api.worldlabs.ai/marble/v1/operations/{operation_id}
https://api.worldlabs.ai/marble/v1/worlds/{world_id}
```
## 结论
中转服务需要做的是兼容 World Labs 官方 API 协议，而不是要求我方修改接口路径。只要中转根地址完整支持 `/marble/v1/...`，我方系统就可以在官方接口和中转接口之间无缝切换。
- 不能用 `/v1/chat/completions`。
- 不能把模型写成 `marble-1.1`。
- 不能把 `accepted` 或 `running` 当作调用失败。
- 不能只等待几十秒就判定超时。
- 必须携带 `Authorization: Bearer <API_KEY>`。

16
lib/ai/dynamic-provider.ts
View File

@ -27,6 +27,7 @@ import { createOpenAIImage } from "./providers/openai-image";
import { createBananaImage } from "./providers/bananaimage";
import { createVisionary } from "./providers/visionary";
import { createAtlasCloud } from "./providers/atlascloud";
import { createVip98k } from "./providers/vip98k";
import { createStepfunImage } from "./providers/stepfun-image";
import { createMidjourney } from "./providers/midjourney";
import { createFalVideo } from "./providers/fal-video";
@ -324,6 +325,15 @@ function getOrCreateProvider(provider: Pick<AiProvider, "id" | "type" | "baseUrl
return atlasCloudProvider as any;
}
case "vip98k": {
const vip98kProvider = createVip98k({
baseURL: resolvedBaseUrl,
apiKey: provider.apiKey,
});
logProviderCreated("vip98k", resolvedBaseUrl);
return vip98kProvider as any;
}
case "stepfun-image": {
const stepfunImageProvider = createStepfunImage({
baseURL: resolvedBaseUrl,
@ -501,6 +511,12 @@ export async function getDynamicModel(
} else if (provider.type === "atlascloud") {
baseModel = providerInstance.chat(resolvedModelId, model.imageGenConfig || null);
logChatModelResult("atlascloud", baseModel);
} else if (provider.type === "vip98k") {
baseModel = providerInstance.chat(resolvedModelId, {
capabilities: model.capabilities || null,
imageGenConfig: model.imageGenConfig || null,
});
logChatModelResult("vip98k", baseModel);
} else if (provider.type === "stepfun-image") {
baseModel = providerInstance.chat(resolvedModelId);
logChatModelResult("stepfun-image", baseModel);

7
lib/ai/model-classification.ts
View File

@ -76,6 +76,8 @@ const THREE_D_MODE_KEYS = new Set([
"mesh-generation",
"mesh_generation",
"cad",
"world-generation",
"world_generation",
]);
const THREE_D_KIND_KEYS = new Set([
@ -86,6 +88,11 @@ const THREE_D_KIND_KEYS = new Set([
"mesh-generation",
"mesh_generation",
"cad",
"world",
"world-generation",
"world_generation",
"3d-world",
"3d_world",
]);
const TEXT_TO_IMAGE_MODE_KEYS = new Set([

37
lib/ai/model-router.ts
View File

@ -9,6 +9,7 @@ import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
import {
isImageGenerationModelDescriptor,
modelSupportsTextToImageDescriptor,
is3DModelDescriptor,
isVideoModelDescriptor,
} from "@/lib/ai/model-classification";
import { isKnownReferenceImageModel } from "@/lib/ai/reference-image-capability";
@ -51,6 +52,7 @@ const CHAT_PROVIDER_TYPES = [
"gemini",
"huanke-gemini",
"volcano-multimodal",
"vip98k",
] as const;
const IMAGE_ONLY_PROVIDER_TYPES = new Set([
@ -91,6 +93,25 @@ function isConversationalCandidate(
const providerType = String(item.provider.type || "");
if (!item.model.isEnabled || item.model.isSubModel) return false;
if (isVideoProviderType(providerType, item.model)) return false;
if (
is3DModelDescriptor({
providerType,
modelId: item.model.modelId,
capabilities: item.model.capabilities,
})
) {
return false;
}
if (
isImageGenerationModelDescriptor({
providerType,
modelId: item.model.modelId,
capabilities: item.model.capabilities,
supportsImageGen: item.model.supportsImageGen,
})
) {
return false;
}
const requireVision = Boolean(options?.requireVision);
const matchesPreferredProvider = CHAT_PROVIDER_TYPES.includes(
@ -175,7 +196,7 @@ const INTENT_MODEL_MAP: Record<
> = {
// 图片编辑：优先 alibaba-image（灵魂4.5，支持提示词扩展）
image_edit: {
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "stepfun-image", "runway", "krea", "gemini", "huanke-gemini"],
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "vip98k", "stepfun-image", "runway", "krea", "gemini", "huanke-gemini"],
requireVision: true,
requireImageGen: true,
description: "图片编辑",
@ -190,20 +211,20 @@ const INTENT_MODEL_MAP: Record<
// 图片生成：优先灵魂4.5
image_gen: {
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "stepfun-image", "runway", "krea", "gemini", "huanke-gemini", "volcano-vision"],
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "vip98k", "stepfun-image", "runway", "krea", "gemini", "huanke-gemini", "volcano-vision"],
requireImageGen: true,
description: "图片生成",
},
// 视频生成
video_gen: {
providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video", "runway", "krea", "custom"],
providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video", "runway", "krea", "vip98k", "custom"],
description: "视频生成",
},
// 普通对话：优先千问3
chat: {
providerTypes: ["alibaba", "zhipu", "openai", "custom", "local", "gemini", "huanke-gemini", "volcano-multimodal"],
providerTypes: ["alibaba", "zhipu", "openai", "custom", "local", "gemini", "huanke-gemini", "volcano-multimodal", "vip98k"],
description: "对话",
},
};
@ -218,25 +239,25 @@ const SUB_INTENT_OVERRIDES: Record<
> = {
// 抠图：alibaba-image 效果最好
remove_background: {
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "stepfun-image", "runway", "krea", "gemini"],
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "vip98k", "stepfun-image", "runway", "krea", "gemini"],
reason: "抠图首选灵魂4.5",
},
// 换背景：alibaba-image 支持场景生成
replace_background: {
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "stepfun-image", "runway", "krea", "gemini"],
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "vip98k", "stepfun-image", "runway", "krea", "gemini"],
reason: "换背景首选灵魂4.5",
},
// 图片增强
enhance: {
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "stepfun-image", "runway", "krea", "gemini", "volcano-vision"],
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "vip98k", "stepfun-image", "runway", "krea", "gemini", "volcano-vision"],
reason: "增强首选灵魂4.5",
},
// 风格迁移：gemini 可能更有创意
style_transfer: {
providerTypes: ["gemini", "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "stepfun-image", "runway", "krea"],
providerTypes: ["gemini", "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "vip98k", "stepfun-image", "runway", "krea"],
reason: "风格迁移优选Gemini",
},
};

3
lib/ai/provider-defaults.ts
View File

@ -20,12 +20,13 @@ const PROVIDER_DEFAULT_BASE_URL: Partial<Record<AiProvider["type"], string>> = {
bananaimage: "https://bananaimage.aigenmedia.art",
visionary: "https://api.visionary.beer",
atlascloud: "https://api.atlascloud.ai",
vip98k: "https://api.vip98k.com",
"stepfun-image": "https://api.stepfun.com",
"bfl-image": "https://api.bfl.ai",
midjourney: "https://your-midjourney-api.example.com",
"fal-video": "https://fal.run",
"tripo-3d": "https://api.tripo3d.ai/v2/openapi",
"worldlabs-3d": "https://api.worldlabs.ai",
"worldlabs-3d": "https://api.vip98k.com",
"kittycad-cad": "https://api.zoo.dev",
runway: "https://api.dev.runwayml.com",
krea: "https://api.krea.ai",

674
lib/ai/providers/vip98k.ts Normal file
View File

@ -0,0 +1,674 @@
import "server-only";
import { createOpenAI } from "@ai-sdk/openai";
import sharp from "sharp";
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
import { VIP98K_BASE_URL, normalizeVip98kBaseUrl } from "@/lib/ai/vip98k-models";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service";
interface Vip98kConfig {
apiKey: string;
baseURL?: string | null;
}
interface Vip98kTaskParams {
apiKey: string;
baseURL?: string | null;
modelId: string;
kind: string;
prompt: string;
imageUrls: string[];
videoUrls: string[];
videoParams?: Record<string, unknown>;
imageParams?: Record<string, unknown>;
modelConfig: Vip98kModelRuntimeConfig;
}
type Vip98kModelRuntimeConfig = {
capabilities?: Record<string, unknown> | null;
imageGenConfig?: Record<string, unknown> | null;
};
type UploadedImage = {
thumbnail: string;
original: string;
};
const DOWNLOAD_TIMEOUT_MS = 120_000;
const DEFAULT_POLL_INTERVAL_SECONDS = 8;
const DEFAULT_MAX_POLL_SECONDS = 600;
function toRecord(value: unknown): Record<string, unknown> | null {
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
return value as Record<string, unknown>;
}
function normalizeOpenAIBaseUrl(baseURL?: string | null) {
const normalized = normalizeVip98kBaseUrl(baseURL || VIP98K_BASE_URL);
return /\/v1$/i.test(normalized) ? normalized : `${normalized}/v1`;
}
function parseRuntimeConfig(value?: unknown): Vip98kModelRuntimeConfig {
const record = toRecord(value);
if (!record) return {};
if ("capabilities" in record || "imageGenConfig" in record) {
return {
capabilities: toRecord(record.capabilities),
imageGenConfig: toRecord(record.imageGenConfig),
};
}
return {
capabilities: null,
imageGenConfig: record,
};
}
function getStringField(source: unknown, key: string) {
const record = toRecord(source);
return String(record?.[key] || "").trim().toLowerCase();
}
function getVip98kMetadata(config: Vip98kModelRuntimeConfig) {
const capabilitiesVip98k = toRecord(toRecord(config.capabilities)?.vip98k);
const imageConfigVip98k = toRecord(toRecord(config.imageGenConfig)?.vip98k);
return capabilitiesVip98k || imageConfigVip98k || {};
}
function inferKind(modelId: string, config: Vip98kModelRuntimeConfig) {
const capabilities = config.capabilities;
const metadata = getVip98kMetadata(config);
const configuredKind =
getStringField(capabilities, "kind") ||
getStringField(capabilities, "modelType") ||
getStringField(capabilities, "mediaType") ||
getStringField(metadata, "modelType");
if (configuredKind.includes("world")) return "world";
if (configuredKind.includes("video")) return "video";
if (configuredKind.includes("image-tool")) return "image-tool";
if (configuredKind.includes("image")) return "image";
if (configuredKind.includes("audio")) return "audio";
if (configuredKind.includes("document")) return "document-tool";
if (configuredKind.includes("chat") || configuredKind.includes("model")) return "chat";
const normalizedModelId = modelId.trim().toLowerCase();
if (normalizedModelId === "world-generation") return "world";
if (/^veo[-_.]/.test(normalizedModelId) || normalizedModelId.startsWith("veo-")) {
return "video";
}
if (
normalizedModelId.startsWith("gpt-image") ||
normalizedModelId.startsWith("nano-banana")
) {
return "image";
}
return "chat";
}
function delay(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function parseJsonResponse(response: Response) {
const text = await response.text();
if (!text.trim()) return null;
try {
return JSON.parse(text);
} catch {
return text;
}
}
function extractErrorMessage(payload: unknown, fallback = "VIP98K request failed") {
if (typeof payload === "string" && payload.trim()) return payload.trim();
const record = toRecord(payload);
const data = toRecord(record?.data);
const error = toRecord(record?.error);
const value =
data?.message ||
data?.errorMessage ||
data?.error ||
error?.message ||
record?.message ||
record?.error ||
fallback;
return typeof value === "string" && value.trim() ? value.trim() : fallback;
}
function unwrapData(payload: unknown) {
const record = toRecord(payload);
return record?.data && typeof record.data === "object" ? record.data : payload;
}
function extractTaskId(payload: unknown) {
const data = unwrapData(payload);
const record = toRecord(data);
return String(
record?.taskId ||
record?.task_id ||
record?.id ||
record?.requestId ||
record?.request_id ||
"",
).trim();
}
function extractResultPath(payload: unknown) {
const data = unwrapData(payload);
const record = toRecord(data);
return String(record?.resultPath || record?.result_path || "").trim();
}
function extractStatus(payload: unknown) {
const data = unwrapData(payload);
const record = toRecord(data);
return String(record?.status || record?.state || record?.taskStatus || "").trim().toLowerCase();
}
function extractDone(payload: unknown) {
const data = unwrapData(payload);
const record = toRecord(data);
if (typeof record?.done === "boolean") return record.done;
if (typeof record?.isDone === "boolean") return record.isDone;
if (typeof record?.finished === "boolean") return record.finished;
return false;
}
function collectHttpUrls(value: unknown, urls = new Set<string>()) {
if (!value) return urls;
if (typeof value === "string") {
const rawUrlMatches = value.match(/https?:\/\/[^\s"'<>()[\]{}]+/g) || [];
for (const match of rawUrlMatches) {
urls.add(match.replace(/[),.;]+$/g, ""));
}
if (/^data:image\/[a-zA-Z0-9.+-]+;base64,/i.test(value)) urls.add(value);
return urls;
}
if (Array.isArray(value)) {
value.forEach((item) => collectHttpUrls(item, urls));
return urls;
}
if (typeof value !== "object") return urls;
Object.values(value as Record<string, unknown>).forEach((nested) =>
collectHttpUrls(nested, urls),
);
return urls;
}
function collectOutputUrls(payload: unknown) {
const data = unwrapData(payload);
const candidateRecord = toRecord(data);
const preferred = [
candidateRecord?.result,
candidateRecord?.results,
candidateRecord?.output,
candidateRecord?.outputs,
candidateRecord?.url,
candidateRecord?.urls,
candidateRecord?.imageUrl,
candidateRecord?.image_url,
candidateRecord?.videoUrl,
candidateRecord?.video_url,
candidateRecord?.assetUrl,
candidateRecord?.asset_url,
candidateRecord?.files,
candidateRecord?.assets,
data,
];
const urls = new Set<string>();
preferred.forEach((candidate) => collectHttpUrls(candidate, urls));
return Array.from(urls);
}
function parseDataUrl(value: string) {
const match = value.match(/^data:([^;,]+);base64,(.+)$/s);
if (!match) throw new Error("VIP98K returned invalid Base64 media");
return {
mimeType: match[1] || "image/png",
buffer: Buffer.from(match[2] || "", "base64"),
};
}
function detectExtension(mimeType: string) {
if (/jpe?g/i.test(mimeType)) return "jpg";
if (/webp/i.test(mimeType)) return "webp";
if (/gif/i.test(mimeType)) return "gif";
if (/mp4/i.test(mimeType)) return "mp4";
if (/webm/i.test(mimeType)) return "webm";
if (/quicktime/i.test(mimeType)) return "mov";
return "png";
}
function toArrayBuffer(buffer: Buffer) {
return buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength,
) as ArrayBuffer;
}
async function uploadBuffer(buffer: Buffer, mimeType: string) {
const extension = detectExtension(mimeType);
const thumbnailBuffer = await sharp(buffer, { failOn: "none" })
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
.jpeg({ quality: 82 })
.toBuffer()
.catch(() => buffer);
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
const [original, thumbnail] = await Promise.all([
uploadFile(`vip98k-orig-${suffix}.${extension}`, toArrayBuffer(buffer)),
uploadFile(`vip98k-thumb-${suffix}.jpg`, toArrayBuffer(thumbnailBuffer)),
]);
return { original: original.url, thumbnail: thumbnail.url };
}
async function archiveOutputImage(imageUrl: string): Promise<UploadedImage> {
if (imageUrl.startsWith("data:")) {
const parsed = parseDataUrl(imageUrl);
return uploadBuffer(parsed.buffer, parsed.mimeType);
}
const deferred = await createDeferredUploadedImage(imageUrl, {
providerType: "vip98k",
});
if (deferred) return deferred;
const response = await fetch(imageUrl, {
cache: "no-store",
redirect: "follow",
headers: { "User-Agent": "yedir-ai-vip98k/1.0" },
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),
});
if (!response.ok) {
throw new Error(`VIP98K result image download failed (HTTP ${response.status})`);
}
return uploadBuffer(
Buffer.from(await response.arrayBuffer()),
response.headers.get("content-type") || "image/png",
);
}
async function ensurePublicMediaUrl(mediaUrl: string) {
const normalized = String(mediaUrl || "").trim();
if (/^https?:\/\//i.test(normalized)) return normalized;
if (!normalized.startsWith("data:")) return normalized;
const parsed = parseDataUrl(normalized);
const extension = detectExtension(parsed.mimeType);
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
const uploaded = await uploadFile(
`vip98k-ref-${suffix}.${extension}`,
toArrayBuffer(parsed.buffer),
);
return uploaded.url;
}
function extractPromptAndMedia(messages: Array<{ role: string; content?: any; parts?: any[] }>) {
const message = messages.filter((item) => item.role === "user").pop();
if (!message) throw new Error("No user message found");
const texts: string[] = [];
const imageUrls: string[] = [];
const videoUrls: string[] = [];
const pushText = (value: unknown) => {
if (typeof value === "string" && value.trim()) texts.push(value.trim());
};
const pushMedia = (value: unknown, mediaType?: unknown) => {
const url = String(
typeof value === "string"
? value
: (value as any)?.url ||
(value as any)?.imageUrl ||
(value as any)?.fileUrl ||
(value as any)?.src ||
"",
).trim();
if (!url || !/^(?:https?:\/\/|data:)/i.test(url)) return;
const normalizedMediaType = String(mediaType || "").toLowerCase();
const isVideo =
normalizedMediaType.startsWith("video/") ||
url.startsWith("data:video/") ||
/\.(?:mp4|mov|webm|m4v)(?:[?#].*)?$/i.test(url);
const target = isVideo ? videoUrls : imageUrls;
if (!target.includes(url)) target.push(url);
};
const visitPart = (part: any) => {
if (!part) return;
if (typeof part === "string") return pushText(part);
if (part.type === "text" || typeof part.text === "string") pushText(part.text);
if (part.type === "image_url") pushMedia(part.image_url, "image/jpeg");
if (part.type === "image") pushMedia(part.image || part.url, "image/jpeg");
if (part.type === "file") {
pushMedia(
part.url || part.file || part.data,
part.mediaType || part.contentType || part.mimeType,
);
}
if (part.inlineData?.data) {
pushMedia(
`data:${part.inlineData.mimeType || "image/png"};base64,${part.inlineData.data}`,
part.inlineData.mimeType,
);
}
};
if (typeof message.content === "string") pushText(message.content);
else if (Array.isArray(message.content)) message.content.forEach(visitPart);
if (Array.isArray(message.parts)) message.parts.forEach(visitPart);
const prompt = texts.join("\n").trim() || "Generate media";
return { prompt, imageUrls, videoUrls };
}
function getConfiguredPollSeconds(config: Vip98kModelRuntimeConfig, key: string, fallback: number) {
const metadata = getVip98kMetadata(config);
const value = Number(metadata[key]);
if (!Number.isFinite(value) || value <= 0) return fallback;
return Math.min(Math.max(value, 1), 3600);
}
async function submitTask(params: {
apiKey: string;
baseURL?: string | null;
body: Record<string, unknown>;
}) {
const baseURL = normalizeVip98kBaseUrl(params.baseURL);
const response = await fetch(`${baseURL}/api/external/task/generate`, {
method: "POST",
headers: {
Authorization: `Bearer ${params.apiKey}`,
"Content-Type": "application/json",
},
body: JSON.stringify(params.body),
signal: AbortSignal.timeout(120_000),
});
const payload = await parseJsonResponse(response);
if (!response.ok) {
throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`);
}
return {
payload,
taskId: extractTaskId(payload),
resultPath: extractResultPath(payload),
outputs: collectOutputUrls(payload),
};
}
async function waitForTask(params: {
apiKey: string;
baseURL?: string | null;
taskId: string;
resultPath?: string;
modelConfig: Vip98kModelRuntimeConfig;
}) {
const baseURL = normalizeVip98kBaseUrl(params.baseURL);
const intervalMs =
getConfiguredPollSeconds(
params.modelConfig,
"pollIntervalSeconds",
DEFAULT_POLL_INTERVAL_SECONDS,
) * 1000;
const timeoutMs =
getConfiguredPollSeconds(params.modelConfig, "maxPollSeconds", DEFAULT_MAX_POLL_SECONDS) *
1000;
const startedAt = Date.now();
while (Date.now() - startedAt < timeoutMs) {
await delay(intervalMs);
const resultUrl = params.resultPath
? params.resultPath.startsWith("http")
? params.resultPath
: `${baseURL}${params.resultPath.startsWith("/") ? "" : "/"}${params.resultPath}`
: `${baseURL}/api/external/task/result?taskId=${encodeURIComponent(params.taskId)}`;
const response = await fetch(resultUrl, {
cache: "no-store",
headers: { Authorization: `Bearer ${params.apiKey}` },
signal: AbortSignal.timeout(60_000),
});
const payload = await parseJsonResponse(response);
if (!response.ok) {
throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`);
}
const status = extractStatus(payload);
const done = extractDone(payload);
const outputs = collectOutputUrls(payload);
if (
["succeeded", "success", "completed", "done", "finished"].includes(status) ||
(done && outputs.length > 0)
) {
return { payload, outputs };
}
if (["failed", "error", "timeout", "canceled", "cancelled"].includes(status)) {
throw new Error(extractErrorMessage(payload, "VIP98K task failed"));
}
}
throw new Error("VIP98K task timed out; it may still be running");
}
function normalizeRatio(value: unknown) {
const normalized = String(value || "").trim();
return normalized && normalized !== "auto" ? normalized : undefined;
}
function normalizeDuration(value: unknown) {
const numeric = Number(value);
if (!Number.isFinite(numeric) || numeric <= 0) return undefined;
return Math.round(numeric);
}
function buildTaskBody(params: Vip98kTaskParams) {
const videoParams = params.videoParams || {};
const imageParams = params.imageParams || {};
const promptTitle = params.prompt.split(/\s+/).slice(0, 8).join(" ").slice(0, 80);
const body: Record<string, unknown> = {
model: params.modelId,
displayName: promptTitle || params.modelId,
prompt: params.prompt,
};
const aspectRatio =
normalizeRatio(videoParams.ratio) ||
normalizeRatio(videoParams.aspectRatio) ||
normalizeRatio(imageParams.aspectRatio);
const duration = normalizeDuration(videoParams.duration);
const resolution = String(videoParams.resolution || imageParams.imageSize || "").trim();
const negativePrompt = String(videoParams.negativePrompt || imageParams.negativePrompt || "").trim();
if (aspectRatio) {
body.ratio = aspectRatio;
body.aspectRatio = aspectRatio;
body.aspect_ratio = aspectRatio;
}
if (duration) body.duration = duration;
if (resolution) {
body.resolution = resolution;
body.imageSize = resolution;
body.image_size = resolution;
}
if (negativePrompt) {
body.negativePrompt = negativePrompt;
body.negative_prompt = negativePrompt;
}
if (typeof videoParams.generateAudio === "boolean") {
body.generateAudio = videoParams.generateAudio;
body.generate_audio = videoParams.generateAudio;
}
if (params.imageUrls.length > 0) {
body.images = params.imageUrls;
body.imageUrls = params.imageUrls;
body.referenceImages = params.imageUrls;
body.reference_images = params.imageUrls;
body.image = params.imageUrls[0];
body.imageUrl = params.imageUrls[0];
body.image_url = params.imageUrls[0];
body.firstFrameImage = params.imageUrls[0];
body.first_frame_image = params.imageUrls[0];
body.firstFrameImageUrl = params.imageUrls[0];
body.first_frame_image_url = params.imageUrls[0];
if (params.imageUrls[1]) {
body.lastFrameImage = params.imageUrls[1];
body.last_frame_image = params.imageUrls[1];
body.endImageUrl = params.imageUrls[1];
body.end_image_url = params.imageUrls[1];
}
}
if (params.videoUrls.length > 0) {
body.videos = params.videoUrls;
body.videoUrls = params.videoUrls;
body.referenceVideos = params.videoUrls;
}
if (params.kind === "world") {
body.mode = "world-generation";
} else if (params.kind === "video") {
body.mode = params.imageUrls.length > 0 ? "image-to-video" : "text-to-video";
}
return body;
}
function formatImageMarkdown(images: UploadedImage[], imageSize: string) {
return images
.map((image) => `![thumb|||${image.original}|||${imageSize}](${image.thumbnail})`)
.join("\n\n");
}
function formatVideoMarkdown(videoUrl: string) {
return `<video src="${videoUrl}" controls style="max-width: 100%; border-radius: 8px; margin-bottom: 12px;"></video>\n\n[下载视频](${videoUrl})`;
}
function formatWorldMarkdown(urls: string[]) {
if (urls.length === 0) return "3D 世界生成成功，但未返回可访问的结果地址";
return urls.map((url, index) => `[查看 3D 世界 ${index + 1}](${url})`).join("\n\n");
}
async function runTask(params: Vip98kTaskParams) {
const body = buildTaskBody(params);
const submitted = await submitTask({
apiKey: params.apiKey,
baseURL: params.baseURL,
body,
});
const outputUrls = submitted.outputs.length
? submitted.outputs
: await waitForTask({
apiKey: params.apiKey,
baseURL: params.baseURL,
taskId: submitted.taskId,
resultPath: submitted.resultPath,
modelConfig: params.modelConfig,
}).then((result) => result.outputs);
if (params.kind === "video") {
const videoUrl =
outputUrls.find((url) => /\.(mp4|webm|mov|m4v)(?:[?#].*)?$/i.test(url)) ||
outputUrls[0];
if (!videoUrl) throw new Error("VIP98K task completed but returned no video URL");
return formatVideoMarkdown(videoUrl);
}
if (params.kind === "image" || params.kind === "image-tool") {
if (outputUrls.length === 0) {
throw new Error("VIP98K task completed but returned no image URL");
}
const imageSize = String(params.imageParams?.imageSize || "2K").trim() || "2K";
const archived = await Promise.all(outputUrls.slice(0, 1).map(archiveOutputImage));
return formatImageMarkdown(archived, imageSize);
}
if (params.kind === "world") {
return formatWorldMarkdown(outputUrls);
}
return outputUrls.length ? outputUrls.join("\n") : "VIP98K task completed";
}
export function createVip98k({ apiKey, baseURL }: Vip98kConfig) {
const rootBaseURL = normalizeVip98kBaseUrl(baseURL);
const openAIBaseURL = normalizeOpenAIBaseUrl(rootBaseURL);
const openAITextProvider = createOpenAI({
apiKey,
baseURL: openAIBaseURL,
});
const openAIImageProvider = createOpenAIImage({
apiKey,
baseURL: openAIBaseURL,
});
return {
provider: "vip98k",
chat: (modelId: string, defaultModelConfig?: unknown): any => {
const modelConfig = parseRuntimeConfig(defaultModelConfig);
const kind = inferKind(modelId, modelConfig);
if (kind === "image") {
return openAIImageProvider.chat(modelId) as any;
}
if (kind === "chat") {
return openAITextProvider.chat(modelId) as any;
}
return {
provider: "vip98k",
doGenerate: async (options: {
messages: Array<{ role: string; content?: any; parts?: any[] }>;
imageParams?: Record<string, unknown>;
videoParams?: Record<string, unknown>;
imageGenConfig?: unknown;
overrideModelId?: string;
}) => {
const runtimeConfig =
options.imageGenConfig !== undefined
? parseRuntimeConfig({
capabilities: modelConfig.capabilities,
imageGenConfig: options.imageGenConfig,
})
: modelConfig;
const activeModelId = String(options.overrideModelId || modelId || "").trim();
if (!activeModelId) throw new Error("VIP98K model ID is not configured");
const activeKind = inferKind(activeModelId, runtimeConfig);
const { prompt, imageUrls, videoUrls } = extractPromptAndMedia(options.messages);
const [publicImageUrls, publicVideoUrls] = await Promise.all([
Promise.all(imageUrls.map(ensurePublicMediaUrl)),
Promise.all(videoUrls.map(ensurePublicMediaUrl)),
]);
const text = await runTask({
apiKey,
baseURL: rootBaseURL,
modelId: activeModelId,
kind: activeKind,
prompt,
imageUrls: publicImageUrls,
videoUrls: publicVideoUrls,
videoParams: options.videoParams,
imageParams: options.imageParams,
modelConfig: runtimeConfig,
});
return {
text,
finishReason: "stop" as const,
usage: {
promptTokens: 0,
completionTokens: text.length,
},
};
},
doStream: async function* (options: {
messages: Array<{ role: string; content?: any; parts?: any[] }>;
imageParams?: Record<string, unknown>;
videoParams?: Record<string, unknown>;
imageGenConfig?: unknown;
overrideModelId?: string;
}) {
const result = await this.doGenerate(options);
yield { type: "text-delta", textDelta: result.text };
},
};
},
};
}

156
lib/ai/tools/generate-video.ts
View File

@ -13,6 +13,7 @@ import {
import { getConfiguredVideoDurationOptions } from "@/lib/ai/video-model-config";
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
import { getKreaModelPresetById } from "@/lib/ai/krea-models";
import { isVideoModelDescriptor } from "@/lib/ai/model-classification";
import type {
VideoGenerationMode,
VideoModelCapabilities,
@ -476,6 +477,121 @@ async function resolvePreferredVideoParentModelId(preferredModelRef?: string) {
return matched.model.id;
}
async function resolvePreferredGenericVideoModel(
preferredModelRef: string | undefined,
mode: VideoGenerationMode,
params: Partial<VideoGenerationOptions>,
) {
const normalizedRef = String(preferredModelRef || "").trim();
if (!normalizedRef) return null;
const models = await getEnabledAiModelsWithProvider();
const matched = models.find(
(item) =>
item.provider.type === "vip98k" &&
item.model.isEnabled &&
!item.model.isSubModel &&
(item.model.id === normalizedRef ||
item.model.modelId === normalizedRef ||
item.model.displayName === normalizedRef) &&
isVideoModelDescriptor({
providerType: item.provider.type,
modelId: item.model.modelId,
capabilities: item.model.capabilities,
supportsImageGen: item.model.supportsImageGen,
}),
);
if (!matched) return null;
const capabilities = withConfiguredVideoCapabilities(
matched.model.capabilities as VideoModelCapabilities | undefined,
(matched.model as any).imageGenConfig,
);
const supportedModes = getSupportedModes(capabilities, matched.model.modelId);
if (!supportedModes.includes(mode)) {
throw new Error(`模型 ${matched.model.displayName} 不支持 ${mode} 模式`);
}
const resolvedCapabilities = {
...capabilities,
supportedModes,
} as VideoModelCapabilities;
validateParamsAgainstCapabilities(params, resolvedCapabilities);
return {
model: matched,
capabilities: resolvedCapabilities,
};
}
async function executeGenericProviderVideoGeneration(params: {
request: ExecuteDirectVideoGenerationParams;
prompt: string;
selected: { model: any; capabilities: VideoModelCapabilities };
generationParams: Partial<VideoGenerationOptions>;
}): Promise<ExecuteDirectVideoGenerationResult> {
const selected = params.selected.model;
const languageModel = await getLanguageModelById(selected.model.id);
if (!languageModel || typeof (languageModel as any).doGenerate !== "function") {
throw new Error("所选视频模型初始化失败");
}
await params.request.onProgress?.(
`正在调用 ${selected.model.displayName || selected.model.modelId} 生成视频...`,
);
const imageParts = (params.request.images || []).map((url) => ({
type: "image_url" as const,
image_url: { url },
}));
const result = await (languageModel as any).doGenerate({
messages: [
{
role: "user",
content: [{ type: "text", text: params.prompt }, ...imageParts],
},
],
overrideModelId: selected.model.modelId,
videoParams: {
mode: params.request.mode,
resolution: params.generationParams.resolution || "720p",
ratio: params.generationParams.ratio || "16:9",
duration: params.generationParams.duration || 8,
generateAudio: params.generationParams.generateAudio ?? true,
watermark: params.generationParams.watermark,
cameraFixed: params.generationParams.cameraFixed,
seed: params.generationParams.seed,
returnLastFrame: params.generationParams.returnLastFrame,
},
});
if (result?.finishReason === "error") {
throw new Error(String(result?.text || "视频生成失败"));
}
const videoUrl = extractVideoUrlFromGeneratedText(result?.text);
if (!videoUrl) {
throw new Error("所选视频模型未返回可用视频地址");
}
return {
success: true,
videoUrl,
metadata: {
provider: selected.provider.type,
temporaryVideoUrl: videoUrl,
mode: params.request.mode,
resolution: params.generationParams.resolution,
ratio: params.generationParams.ratio,
duration: params.generationParams.duration,
},
modelDisplayName: String(selected.model.displayName || selected.model.modelId),
modelId: String(selected.model.modelId || selected.model.id),
modelRef: String(selected.model.id || selected.model.modelId),
};
}
export async function executeDirectVideoGeneration(
params: ExecuteDirectVideoGenerationParams,
): Promise<ExecuteDirectVideoGenerationResult> {
@ -497,6 +613,33 @@ export async function executeDirectVideoGeneration(
});
}
const generationParams = {
mode: params.mode,
resolution: params.resolution || "720p",
ratio: params.ratio || "16:9",
duration: params.duration || 5,
generateAudio: params.generateAudio ?? true,
watermark: params.watermark ?? true,
cameraFixed: params.cameraFixed ?? false,
seed: params.seed,
returnLastFrame: params.returnLastFrame ?? false,
referenceImages: params.images,
} satisfies Partial<VideoGenerationOptions>;
const preferredGenericModel = await resolvePreferredGenericVideoModel(
params.preferredModelRef,
params.mode,
generationParams,
);
if (preferredGenericModel) {
return executeGenericProviderVideoGeneration({
request: params,
prompt,
selected: preferredGenericModel,
generationParams,
});
}
const preferredParentModelId = await resolvePreferredVideoParentModelId(
params.preferredModelRef,
);
@ -526,19 +669,6 @@ export async function executeDirectVideoGeneration(
)?.model.parentModelId ||
undefined;
const generationParams = {
mode: params.mode,
resolution: params.resolution || "720p",
ratio: params.ratio || "16:9",
duration: params.duration || 5,
generateAudio: params.generateAudio ?? true,
watermark: params.watermark ?? true,
cameraFixed: params.cameraFixed ?? false,
seed: params.seed,
returnLastFrame: params.returnLastFrame ?? false,
referenceImages: params.images,
} satisfies Partial<VideoGenerationOptions>;
let selectedModelConfig: { model: any; capabilities: VideoModelCapabilities };
try {
selectedModelConfig = await getCompatibleModel(

626
lib/ai/vip98k-models.ts Normal file
View File

@ -0,0 +1,626 @@
import "server-only";
export const VIP98K_BASE_URL = "https://api.vip98k.com";
export type Vip98kCatalogModelType =
| "image"
| "video"
| "image_tool"
| "document_tool"
| "model"
| "audio"
| "world"
| string;
export interface Vip98kCatalogModel {
modelId?: string;
id?: string;
name?: string;
displayName?: string;
display_name?: string;
description?: string;
type?: Vip98kCatalogModelType;
modelType?: Vip98kCatalogModelType;
model_type?: Vip98kCatalogModelType;
category?: Vip98kCatalogModelType;
capabilities?: unknown;
inputModes?: unknown;
input_modes?: unknown;
outputModes?: unknown;
output_modes?: unknown;
supportedResolutions?: unknown;
supported_resolutions?: unknown;
resolutions?: unknown;
durations?: unknown;
durationOptions?: unknown;
duration_options?: unknown;
defaultDuration?: unknown;
default_duration?: unknown;
supportsAsync?: unknown;
supports_async?: unknown;
pollIntervalSeconds?: unknown;
poll_interval_seconds?: unknown;
maxPollSeconds?: unknown;
max_poll_seconds?: unknown;
resultTTLSeconds?: unknown;
result_ttl_seconds?: unknown;
billingMode?: string;
billing_mode?: string;
pricing?: unknown;
price?: unknown;
status?: string;
[key: string]: unknown;
}
export interface Vip98kModelPreset {
modelId: string;
displayName: string;
description: string;
type: Vip98kCatalogModelType;
supportsVision: boolean;
supportsImageGen: boolean;
isHidden: boolean;
capabilities: Record<string, unknown>;
defaultParams: Record<string, unknown>;
imageGenConfig: Record<string, unknown> | null;
sortOrder: number;
}
const DEFAULT_ASPECT_RATIOS = [
"auto",
"1:1",
"4:3",
"3:4",
"16:9",
"9:16",
"3:2",
"2:3",
"4:5",
"5:4",
] as const;
const DEFAULT_VIDEO_RATIOS = ["16:9", "9:16", "1:1", "4:3", "3:4"] as const;
const DEFAULT_IMAGE_SIZES = ["1K", "2K", "4K"] as const;
const TYPE_SORT_ORDER: Record<string, number> = {
image: 100,
video: 200,
world: 300,
model: 400,
image_tool: 800,
audio: 900,
document_tool: 950,
};
function toRecord(value: unknown): Record<string, unknown> | null {
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
return value as Record<string, unknown>;
}
function parseStringArray(value: unknown): string[] {
if (!Array.isArray(value)) return [];
return Array.from(
new Set(
value
.map((item) => String(item || "").trim())
.filter(Boolean),
),
);
}
function parseNumberArray(value: unknown): number[] {
if (!Array.isArray(value)) return [];
return Array.from(
new Set(
value
.map((item) => Number(item))
.filter((item) => Number.isFinite(item) && item > 0),
),
).sort((a, b) => a - b);
}
function parseNumber(value: unknown): number | undefined {
const numeric = Number(value);
return Number.isFinite(numeric) ? numeric : undefined;
}
function normalizeCatalogType(model: Vip98kCatalogModel): Vip98kCatalogModelType {
return String(
model.type || model.modelType || model.model_type || model.category || "model",
)
.trim()
.toLowerCase();
}
function normalizeImageSize(value: string): string | null {
const normalized = value.trim().toUpperCase();
if (normalized === "1K" || normalized === "2K" || normalized === "4K") {
return normalized;
}
const kMatch = normalized.match(/^(\d+(?:\.\d+)?)\s*K$/);
if (kMatch?.[1]) return `${Number(kMatch[1]).toString()}K`;
return null;
}
function normalizeVideoResolution(value: string): string | null {
const normalized = value.trim().toLowerCase();
if (normalized === "720p" || normalized === "1080p" || normalized === "4k") {
return normalized;
}
if (normalized === "720" || normalized === "720P".toLowerCase()) return "720p";
if (normalized === "1080" || normalized === "1080P".toLowerCase()) return "1080p";
if (normalized === "2160p" || normalized === "2160" || normalized === "4K".toLowerCase()) {
return "4k";
}
return normalized || null;
}
function getSupportedResolutions(model: Vip98kCatalogModel) {
return parseStringArray(
model.supportedResolutions || model.supported_resolutions || model.resolutions,
);
}
function getSupportedImageSizes(model: Vip98kCatalogModel) {
const sizes = getSupportedResolutions(model)
.map(normalizeImageSize)
.filter((value): value is string => Boolean(value));
return sizes.length ? sizes : [...DEFAULT_IMAGE_SIZES];
}
function getSupportedVideoResolutions(model: Vip98kCatalogModel) {
const resolutions = getSupportedResolutions(model)
.map(normalizeVideoResolution)
.filter((value): value is string => Boolean(value));
return resolutions.length ? resolutions : ["720p"];
}
function getDurations(model: Vip98kCatalogModel) {
return parseNumberArray(
model.durations || model.durationOptions || model.duration_options,
);
}
function getCapabilities(model: Vip98kCatalogModel) {
return parseStringArray(model.capabilities).map((item) =>
item.trim().toLowerCase().replace(/_/g, "-"),
);
}
function getInputModes(model: Vip98kCatalogModel) {
return parseStringArray(model.inputModes || model.input_modes).map((item) =>
item.trim().toLowerCase(),
);
}
function getOutputModes(model: Vip98kCatalogModel) {
return parseStringArray(model.outputModes || model.output_modes).map((item) =>
item.trim().toLowerCase(),
);
}
function getDisplayName(model: Vip98kCatalogModel, modelId: string) {
return String(
model.displayName || model.display_name || model.name || modelId,
).trim();
}
function getModelId(model: Vip98kCatalogModel) {
return String(model.modelId || model.id || model.name || "").trim();
}
function getEffectivePrice(model: Vip98kCatalogModel) {
const pricing = toRecord(model.pricing);
return parseNumber(
model.price ||
pricing?.effectivePrice ||
pricing?.effective_price ||
pricing?.price ||
pricing?.requestPrice ||
pricing?.request_price,
);
}
function getBillingMode(model: Vip98kCatalogModel) {
return String(model.billingMode || model.billing_mode || "per_request").trim();
}
function getVip98kMetadata(model: Vip98kCatalogModel, type: string) {
return {
modelType: type,
taskApi: "external-task",
supportsAsync: Boolean(model.supportsAsync ?? model.supports_async ?? true),
pollIntervalSeconds: parseNumber(
model.pollIntervalSeconds || model.poll_interval_seconds,
) || 8,
maxPollSeconds: parseNumber(model.maxPollSeconds || model.max_poll_seconds) || 600,
resultTTLSeconds: parseNumber(
model.resultTTLSeconds || model.result_ttl_seconds,
),
billingMode: getBillingMode(model),
effectivePrice: getEffectivePrice(model),
pricing: model.pricing || null,
status: model.status || null,
};
}
function createImagePreset(
model: Vip98kCatalogModel,
index: number,
): Vip98kModelPreset {
const modelId = getModelId(model);
const displayName = getDisplayName(model, modelId);
const capabilities = getCapabilities(model);
const inputModes = getInputModes(model);
const outputModes = getOutputModes(model);
const supportedImageSizes = getSupportedImageSizes(model);
const supportsReferenceImages =
capabilities.includes("image-to-image") ||
inputModes.includes("url") ||
inputModes.includes("file");
const supportedModes = [
...(capabilities.includes("text-to-image") || capabilities.length === 0
? ["text-to-image"]
: []),
...(supportsReferenceImages ? ["image-to-image", "reference-images"] : []),
];
const maxReferenceImages = supportsReferenceImages ? 8 : 0;
const defaultImageSize =
supportedImageSizes.find((size) => size === "2K") ||
supportedImageSizes[0] ||
"1K";
return {
modelId,
displayName,
description: model.description || `VIP98K image model (${modelId})`,
type: "image",
supportsVision: supportsReferenceImages,
supportsImageGen: true,
isHidden: false,
capabilities: {
provider: "vip98k",
kind: "image",
mediaType: "image",
modelType: "image",
outputType: "image",
supportedModes,
inputModes,
outputModes,
supportedImageSizes,
supportedAspectRatios: [...DEFAULT_ASPECT_RATIOS],
supportsMultiImageReference: supportsReferenceImages,
maxReferenceImages,
vip98k: getVip98kMetadata(model, "image"),
},
defaultParams: {
provider: "vip98k",
kind: "image",
endpoint: modelId,
apiFormat: "openai-images-compatible",
taskApi: "openai-images",
supportedModes,
imageSize: defaultImageSize,
aspectRatio: "auto",
imageCount: 1,
},
imageGenConfig: {
maxImages: 1,
maxReferenceImages,
supportsSize: true,
supports1K: supportedImageSizes.includes("1K"),
supports2K: supportedImageSizes.includes("2K"),
supports4K: supportedImageSizes.includes("4K"),
supportedImageSizes,
supportedAspectRatios: [...DEFAULT_ASPECT_RATIOS],
supportsPromptExtend: false,
supportsNegativePrompt: false,
supportsWatermark: false,
supportsSeed: false,
creditCostPerImage: 0,
creditCostMultiplierBySize: {
"1K": 1,
"2K": 1,
"4K": 1,
},
enableLoadingPlaceholder: true,
loadingColor: "yellow",
vip98k: getVip98kMetadata(model, "image"),
},
sortOrder: TYPE_SORT_ORDER.image + index,
};
}
function createVideoPreset(
model: Vip98kCatalogModel,
index: number,
): Vip98kModelPreset {
const modelId = getModelId(model);
const displayName = getDisplayName(model, modelId);
const capabilities = getCapabilities(model);
const inputModes = getInputModes(model);
const outputModes = getOutputModes(model);
const supportedResolutions = getSupportedVideoResolutions(model);
const durations = getDurations(model);
const defaultDuration =
parseNumber(model.defaultDuration || model.default_duration) ||
durations[durations.length - 1] ||
8;
const supportsReferenceImages =
capabilities.includes("image-to-video") ||
inputModes.includes("url") ||
inputModes.includes("file");
const supportedModes = [
...(capabilities.includes("text-to-video") || capabilities.length === 0
? ["text-to-video"]
: []),
...(supportsReferenceImages ? ["image-to-video", "first-frame"] : []),
];
return {
modelId,
displayName,
description: model.description || `VIP98K video model (${modelId})`,
type: "video",
supportsVision: supportsReferenceImages,
supportsImageGen: false,
isHidden: false,
capabilities: {
provider: "vip98k",
kind: "video",
mediaType: "video",
modelType: "video",
outputType: "video",
supportedModes,
inputModes,
outputModes,
supportedResolutions,
supportedRatios: [...DEFAULT_VIDEO_RATIOS],
durationOptions: durations,
durationRange: durations.length
? { min: durations[0], max: durations[durations.length - 1] }
: { min: defaultDuration, max: defaultDuration },
defaultDuration,
supportWatermark: false,
supportCameraFixed: false,
maxReferenceImages: supportsReferenceImages ? 4 : 0,
vip98k: getVip98kMetadata(model, "video"),
},
defaultParams: {
provider: "vip98k",
kind: "video",
endpoint: modelId,
taskApi: "external-task",
supportedModes,
duration: defaultDuration,
resolution: supportedResolutions[0] || "720p",
ratio: "16:9",
},
imageGenConfig: {
creditCostPerVideo: 0,
videoDurationOptions: durations,
creditCostPerVideoByDuration: Object.fromEntries(
durations.map((duration) => [String(duration), 0]),
),
creditCostPerVideoByResolution: Object.fromEntries(
supportedResolutions.map((resolution) => [resolution, 0]),
),
enableLoadingPlaceholder: true,
loadingColor: "blue",
vip98k: getVip98kMetadata(model, "video"),
},
sortOrder: TYPE_SORT_ORDER.video + index,
};
}
function createWorldPreset(
model: Vip98kCatalogModel,
index: number,
): Vip98kModelPreset {
const modelId = getModelId(model);
const displayName = getDisplayName(model, modelId);
return {
modelId,
displayName,
description: model.description || `VIP98K 3D world model (${modelId})`,
type: "world",
supportsVision: false,
supportsImageGen: false,
isHidden: false,
capabilities: {
provider: "vip98k",
kind: "world",
mediaType: "world",
modelType: "world-generation",
outputType: "3d",
supportedModes: ["world-generation", "text-to-3d"],
vip98k: getVip98kMetadata(model, "world"),
},
defaultParams: {
provider: "vip98k",
kind: "world",
endpoint: modelId,
taskApi: "external-task",
},
imageGenConfig: null,
sortOrder: TYPE_SORT_ORDER.world + index,
};
}
function createChatPreset(
model: Vip98kCatalogModel,
index: number,
): Vip98kModelPreset {
const modelId = getModelId(model);
const displayName = getDisplayName(model, modelId);
return {
modelId,
displayName,
description: model.description || `VIP98K text model (${modelId})`,
type: "model",
supportsVision: false,
supportsImageGen: false,
isHidden: false,
capabilities: {
provider: "vip98k",
kind: "chat",
mediaType: "text",
modelType: "model",
outputType: "text",
apiFormat: "openai-compatible",
vip98k: getVip98kMetadata(model, "model"),
},
defaultParams: {
provider: "vip98k",
kind: "chat",
endpoint: modelId,
apiFormat: "openai-compatible",
},
imageGenConfig: null,
sortOrder: TYPE_SORT_ORDER.model + index,
};
}
function createToolPreset(
model: Vip98kCatalogModel,
type: string,
index: number,
): Vip98kModelPreset {
const modelId = getModelId(model);
const displayName = getDisplayName(model, modelId);
const inputModes = getInputModes(model);
const outputModes = getOutputModes(model);
const isImageTool = type === "image_tool";
return {
modelId,
displayName,
description: model.description || `VIP98K ${type} model (${modelId})`,
type,
supportsVision: isImageTool,
supportsImageGen: false,
isHidden: true,
capabilities: {
provider: "vip98k",
kind: type.replace(/_/g, "-"),
mediaType: isImageTool ? "image" : type === "audio" ? "audio" : "document",
modelType: type,
outputType: outputModes[0] || (isImageTool ? "image" : "text"),
inputModes,
outputModes,
supportedModes: getCapabilities(model),
vip98k: getVip98kMetadata(model, type),
},
defaultParams: {
provider: "vip98k",
kind: type.replace(/_/g, "-"),
endpoint: modelId,
taskApi: type === "audio" ? "openai-compatible" : "external-task",
},
imageGenConfig: null,
sortOrder: (TYPE_SORT_ORDER[type] || 990) + index,
};
}
export function buildVip98kModelPresets(
models: Vip98kCatalogModel[],
): Vip98kModelPreset[] {
return models
.map((model, index) => {
const modelId = getModelId(model);
if (!modelId) return null;
const type = normalizeCatalogType(model);
if (type === "image") return createImagePreset(model, index);
if (type === "video") return createVideoPreset(model, index);
if (type === "world") return createWorldPreset(model, index);
if (type === "model") return createChatPreset(model, index);
return createToolPreset(model, type, index);
})
.filter((item): item is Vip98kModelPreset => Boolean(item))
.sort((a, b) => a.sortOrder - b.sortOrder || a.displayName.localeCompare(b.displayName));
}
export function normalizeVip98kBaseUrl(baseURL?: string | null) {
const normalized = String(baseURL || VIP98K_BASE_URL).trim().replace(/\/+$/, "");
if (!normalized) return VIP98K_BASE_URL;
if (/\/v1$/i.test(normalized)) return normalized.replace(/\/v1$/i, "");
if (/\/api\/external\/task$/i.test(normalized)) {
return normalized.replace(/\/api\/external\/task$/i, "");
}
return normalized;
}
function extractCatalogModels(payload: unknown): Vip98kCatalogModel[] {
const record = toRecord(payload);
const data = record?.data;
const dataRecord = toRecord(data);
const candidates = [
dataRecord?.models,
dataRecord?.items,
dataRecord?.list,
data,
record?.models,
record?.items,
];
for (const candidate of candidates) {
if (Array.isArray(candidate)) return candidate as Vip98kCatalogModel[];
}
return [];
}
async function parseJsonResponse(response: Response) {
const text = await response.text();
if (!text.trim()) return null;
try {
return JSON.parse(text);
} catch {
return text;
}
}
function extractErrorMessage(payload: unknown, fallback = "VIP98K model catalog request failed") {
if (typeof payload === "string" && payload.trim()) return payload.trim();
const record = toRecord(payload);
const data = toRecord(record?.data);
const error = toRecord(record?.error);
const value =
data?.message ||
data?.error ||
error?.message ||
record?.message ||
record?.error ||
fallback;
return typeof value === "string" && value.trim() ? value.trim() : fallback;
}
export async function fetchVip98kCatalog(params: {
baseURL?: string | null;
apiKey: string;
}): Promise<Vip98kCatalogModel[]> {
const baseURL = normalizeVip98kBaseUrl(params.baseURL);
const response = await fetch(`${baseURL}/api/external/task/models`, {
cache: "no-store",
headers: {
Authorization: `Bearer ${params.apiKey}`,
Accept: "application/json",
},
signal: AbortSignal.timeout(60_000),
});
const payload = await parseJsonResponse(response);
if (!response.ok) {
throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`);
}
const models = extractCatalogModels(payload);
if (models.length === 0) {
throw new Error("VIP98K model catalog is empty or has an unsupported response shape");
}
return models;
}
export async function fetchVip98kModelPresets(params: {
baseURL?: string | null;
apiKey: string;
}) {
const models = await fetchVip98kCatalog(params);
return buildVip98kModelPresets(models);
}

8
lib/canvas-toolbar-config.ts
View File

@ -1,5 +1,5 @@
export const CANVAS_TOOLBAR_CONFIG_SETTING_KEY = "canvas_toolbar_config";
export const DEFAULT_SELECTED_IMAGE_INLINE_ACTION_COUNT = 7;
export const DEFAULT_SELECTED_IMAGE_INLINE_ACTION_COUNT = 8;
export const MIN_SELECTED_IMAGE_INLINE_ACTION_COUNT = 3;
export const MAX_SELECTED_IMAGE_INLINE_ACTION_COUNT = 12;
@ -220,6 +220,12 @@ export const DEFAULT_CANVAS_TOOLBAR_ITEMS = [
label: "局部编辑",
description: "框选多个彩色区域并按区域提示词局部改图",
},
{
key: "imageLayoutReplace",
group: "节点工具栏",
label: "排版替换",
description: "参考详情图的文字排版、视觉层级和尺寸比例生成替换结果",
},
{
key: "imageRemoveWatermark",
group: "节点工具栏",

28
lib/canvas/generate-image.ts
View File

@ -13,6 +13,7 @@ import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
import { createBananaImage } from "@/lib/ai/providers/bananaimage";
import { createVisionary } from "@/lib/ai/providers/visionary";
import { createAtlasCloud } from "@/lib/ai/providers/atlascloud";
import { createVip98k } from "@/lib/ai/providers/vip98k";
import { createStepfunImage } from "@/lib/ai/providers/stepfun-image";
import { createMidjourney } from "@/lib/ai/providers/midjourney";
import { createRunway } from "@/lib/ai/providers/runway";
@ -237,6 +238,7 @@ const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
"bananaimage",
"visionary",
"atlascloud",
"vip98k",
"stepfun-image",
"midjourney",
"runway",
@ -3164,6 +3166,7 @@ function getProviderBatchLimit(providerType: string) {
case "bananaimage":
case "visionary":
case "atlascloud":
case "vip98k":
case "stepfun-image":
case "midjourney":
case "runway":
@ -4112,6 +4115,29 @@ export async function generateCanvasImage(
});
return result.text || "";
};
} else if (provider.type === "vip98k") {
const vip98kProvider = createVip98k({
apiKey: provider.apiKey,
baseURL: provider.baseUrl,
});
const runtimeModelId = resolveModelRuntimeId(selectedModel.model, {
providerType: provider.type,
});
const modelInstance = vip98kProvider.chat(runtimeModelId, {
capabilities: selectedModel.model.capabilities || null,
imageGenConfig: selectedModel.model.imageGenConfig || null,
});
executeGenerateBatch = async (batchImageCount) => {
const result = await modelInstance.doGenerate({
messages,
imageParams: {
...imageParams,
imageCount: batchImageCount,
},
imageGenConfig: selectedModel.model.imageGenConfig || null,
});
return result.text || "";
};
} else if (provider.type === "stepfun-image") {
const stepfunProvider = createStepfunImage({
apiKey: provider.apiKey,
@ -4188,7 +4214,7 @@ export async function generateCanvasImage(
};
} else {
throw new Error(
`不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/Grsai GPT Image/火山 Seedream/阿里/BFL/RUXA/FAL/SnapGen/OpenAI/BananaImage/Visionary/Atlas Cloud/StepFun/Midjourney/Runway/Krea 图片模型`
`不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/Grsai GPT Image/火山 Seedream/阿里/BFL/RUXA/FAL/SnapGen/OpenAI/BananaImage/Visionary/Atlas Cloud/VIP98K/StepFun/Midjourney/Runway/Krea 图片模型`
);
}

27
lib/canvas/image-task-queue.ts
View File

@ -20,9 +20,17 @@ import {
type StyleProTaskRequest,
type StyleProTaskResult,
} from "@/lib/style-pro/task";
import {
executeCanvasLayoutReplaceTask,
isCanvasLayoutReplaceTaskRequest,
type CanvasLayoutReplaceTaskRequest,
} from "@/lib/canvas/layout-replace-task";
type CanvasImageTaskStatus = "pending" | "running" | "succeeded" | "failed";
type CanvasImageTaskRequest = CanvasGenerateImageRequest | StyleProTaskRequest;
type CanvasImageTaskRequest =
| CanvasGenerateImageRequest
| StyleProTaskRequest
| CanvasLayoutReplaceTaskRequest;
type CanvasImageTaskResult = CanvasGenerateImageResult | StyleProTaskResult;
export type CanvasImageTaskQueueStatus = {
@ -95,6 +103,10 @@ function normalizeTaskModelKey(request: unknown) {
return normalizeCanvasImageModelKey(getStyleProTaskPrimaryModelKey(request));
}
if (isCanvasLayoutReplaceTaskRequest(request)) {
return normalizeCanvasImageModelKey(request.model || "__default__");
}
if (!request || typeof request !== "object" || Array.isArray(request)) {
return "__default__";
}
@ -468,10 +480,15 @@ async function runSingleTask(entry: PendingTaskEntry) {
request,
userId: String(task.userId || entry.userId),
})
: generateCanvasImage({
...(request as CanvasGenerateImageRequest),
ownerKey: task.userId,
});
: isCanvasLayoutReplaceTaskRequest(request)
? executeCanvasLayoutReplaceTask({
request,
userId: String(task.userId || entry.userId),
})
: generateCanvasImage({
...(request as CanvasGenerateImageRequest),
ownerKey: task.userId,
});
const result = await withCanvasImageTaskExecutionTimeout(
execution,
runningTimeoutMs,

633
lib/canvas/layout-replace-task.ts Normal file
View File

@ -0,0 +1,633 @@
import "server-only";
import {
analyzeReplacementExecutionPlan,
buildAgentReplacementPrompt,
createAgentImageThumbnailBuffer,
normalizeReplacementResultImage,
resolveDirectReplacementModelRef,
resolveReplacementReferenceGeometry,
resolveReplacementScenePlanningRuntime,
type AgentDetailReplacementStrategy,
} from "@/app/(chat)/api/chat/_lib/detail-replacement";
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
import {
generateCanvasImage,
type CanvasGenerateImageResult,
} from "@/lib/canvas/generate-image";
import { uploadFile } from "@/lib/storage/service";
export interface CanvasLayoutReplaceTaskRequest {
taskKind: "canvas-layout-replace";
sourceImageUrl: string;
referenceImageUrl: string;
supplementalPrompt?: string;
model?: string;
imageSize?: string;
sourceNodeId?: string;
referenceLabel?: string;
debugMeta?: Record<string, unknown>;
}
function normalizeHttpUrl(value: unknown) {
const url = String(value || "").trim();
return /^https?:\/\//i.test(url) ? url : "";
}
export function isCanvasLayoutReplaceTaskRequest(
request: unknown,
): request is CanvasLayoutReplaceTaskRequest {
return Boolean(
request &&
typeof request === "object" &&
!Array.isArray(request) &&
(request as { taskKind?: unknown }).taskKind === "canvas-layout-replace",
);
}
function normalizeLayoutReplacePromptSupplement(value: unknown) {
return String(value || "")
.trim()
.replace(/\s+/g, " ")
.slice(0, 600);
}
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
return buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength,
) as ArrayBuffer;
}
function selectLayoutPatchMask(params: {
generated: Buffer;
height: number;
reference: Buffer;
width: number;
}) {
const { generated, height, reference, width } = params;
const pixelCount = width * height;
const rawMask = new Uint8Array(pixelCount);
const diffThreshold = 34;
for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) {
const offset = pixelIndex * 3;
const diff = Math.max(
Math.abs((generated[offset] || 0) - (reference[offset] || 0)),
Math.abs((generated[offset + 1] || 0) - (reference[offset + 1] || 0)),
Math.abs((generated[offset + 2] || 0) - (reference[offset + 2] || 0)),
);
if (diff >= diffThreshold) {
rawMask[pixelIndex] = 1;
}
}
const visited = new Uint8Array(pixelCount);
const queue = new Int32Array(pixelCount);
const components: Array<{
area: number;
maxX: number;
maxY: number;
minX: number;
minY: number;
}> = [];
for (let start = 0; start < pixelCount; start += 1) {
if (!rawMask[start] || visited[start]) continue;
let head = 0;
let tail = 0;
let area = 0;
let minX = width;
let minY = height;
let maxX = 0;
let maxY = 0;
queue[tail] = start;
tail += 1;
visited[start] = 1;
while (head < tail) {
const current = queue[head];
head += 1;
area += 1;
const x = current % width;
const y = Math.floor(current / width);
if (x < minX) minX = x;
if (x > maxX) maxX = x;
if (y < minY) minY = y;
if (y > maxY) maxY = y;
const left = current - 1;
const right = current + 1;
const up = current - width;
const down = current + width;
if (x > 0 && rawMask[left] && !visited[left]) {
visited[left] = 1;
queue[tail] = left;
tail += 1;
}
if (x < width - 1 && rawMask[right] && !visited[right]) {
visited[right] = 1;
queue[tail] = right;
tail += 1;
}
if (y > 0 && rawMask[up] && !visited[up]) {
visited[up] = 1;
queue[tail] = up;
tail += 1;
}
if (y < height - 1 && rawMask[down] && !visited[down]) {
visited[down] = 1;
queue[tail] = down;
tail += 1;
}
}
components.push({ area, maxX, maxY, minX, minY });
}
const minArea = Math.max(900, Math.round(pixelCount * 0.0015));
const minBoxArea = Math.max(3200, Math.round(pixelCount * 0.008));
const candidates = components
.filter((component) => {
const boxArea =
(component.maxX - component.minX + 1) *
(component.maxY - component.minY + 1);
return component.area >= minArea && boxArea >= minBoxArea;
})
.sort((left, right) => right.area - left.area);
const primary = candidates[0];
if (!primary) {
return null;
}
const selected = candidates.filter((component, index) => {
if (index === 0) return true;
if (component.area >= primary.area * 0.18) return true;
const expandedPrimary = {
maxX: Math.min(width - 1, primary.maxX + Math.round(width * 0.05)),
maxY: Math.min(height - 1, primary.maxY + Math.round(height * 0.05)),
minX: Math.max(0, primary.minX - Math.round(width * 0.05)),
minY: Math.max(0, primary.minY - Math.round(height * 0.05)),
};
return !(
component.maxX < expandedPrimary.minX ||
component.minX > expandedPrimary.maxX ||
component.maxY < expandedPrimary.minY ||
component.minY > expandedPrimary.maxY
);
});
const selectedMask = new Uint8Array(pixelCount);
for (const component of selected) {
for (let y = component.minY; y <= component.maxY; y += 1) {
const rowOffset = y * width;
for (let x = component.minX; x <= component.maxX; x += 1) {
const index = rowOffset + x;
if (rawMask[index]) selectedMask[index] = 255;
}
}
}
return {
mask: selectedMask,
maskSource: "generated-diff",
selectedComponentCount: selected.length,
selectedPixelCount: selected.reduce((sum, component) => sum + component.area, 0),
};
}
function selectReferenceProductMask(params: {
height: number;
reference: Buffer;
width: number;
}) {
const { height, reference, width } = params;
const pixelCount = width * height;
const rawMask = new Uint8Array(pixelCount);
for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) {
const offset = pixelIndex * 3;
const red = reference[offset] || 0;
const green = reference[offset + 1] || 0;
const blue = reference[offset + 2] || 0;
const luminance = 0.2126 * red + 0.7152 * green + 0.0722 * blue;
const channelSpan = Math.max(red, green, blue) - Math.min(red, green, blue);
if (luminance <= 222 || channelSpan >= 38) {
rawMask[pixelIndex] = 1;
}
}
const visited = new Uint8Array(pixelCount);
const queue = new Int32Array(pixelCount);
const components: Array<{
area: number;
boxArea: number;
maxX: number;
maxY: number;
minX: number;
minY: number;
}> = [];
for (let start = 0; start < pixelCount; start += 1) {
if (!rawMask[start] || visited[start]) continue;
let head = 0;
let tail = 0;
let area = 0;
let minX = width;
let minY = height;
let maxX = 0;
let maxY = 0;
queue[tail] = start;
tail += 1;
visited[start] = 1;
while (head < tail) {
const current = queue[head];
head += 1;
area += 1;
const x = current % width;
const y = Math.floor(current / width);
if (x < minX) minX = x;
if (x > maxX) maxX = x;
if (y < minY) minY = y;
if (y > maxY) maxY = y;
const left = current - 1;
const right = current + 1;
const up = current - width;
const down = current + width;
if (x > 0 && rawMask[left] && !visited[left]) {
visited[left] = 1;
queue[tail] = left;
tail += 1;
}
if (x < width - 1 && rawMask[right] && !visited[right]) {
visited[right] = 1;
queue[tail] = right;
tail += 1;
}
if (y > 0 && rawMask[up] && !visited[up]) {
visited[up] = 1;
queue[tail] = up;
tail += 1;
}
if (y < height - 1 && rawMask[down] && !visited[down]) {
visited[down] = 1;
queue[tail] = down;
tail += 1;
}
}
const boxArea = (maxX - minX + 1) * (maxY - minY + 1);
components.push({ area, boxArea, maxX, maxY, minX, minY });
}
const minArea = Math.max(1400, Math.round(pixelCount * 0.002));
const minBoxArea = Math.max(6000, Math.round(pixelCount * 0.012));
const candidates = components
.filter((component) => {
const boxWidth = component.maxX - component.minX + 1;
const boxHeight = component.maxY - component.minY + 1;
const aspect = boxWidth / Math.max(1, boxHeight);
const centerY = (component.minY + component.maxY) / 2;
const looksLikeCtaButton = aspect >= 6 && boxHeight <= height * 0.08;
const looksLikeBottomUtility = centerY >= height * 0.78 && aspect >= 3.5;
return (
component.area >= minArea &&
component.boxArea >= minBoxArea &&
!looksLikeCtaButton &&
!looksLikeBottomUtility
);
})
.sort((left, right) => right.area - left.area);
const primary = candidates[0];
if (!primary) {
return null;
}
const selected = candidates.filter((component, index) => {
if (index === 0) return true;
if (component.area >= primary.area * 0.22) return true;
const expandedPrimary = {
maxX: Math.min(width - 1, primary.maxX + Math.round(width * 0.08)),
maxY: Math.min(height - 1, primary.maxY + Math.round(height * 0.08)),
minX: Math.max(0, primary.minX - Math.round(width * 0.08)),
minY: Math.max(0, primary.minY - Math.round(height * 0.08)),
};
return !(
component.maxX < expandedPrimary.minX ||
component.minX > expandedPrimary.maxX ||
component.maxY < expandedPrimary.minY ||
component.minY > expandedPrimary.maxY
);
});
const selectedMask = new Uint8Array(pixelCount);
for (const component of selected) {
for (let y = component.minY; y <= component.maxY; y += 1) {
const rowOffset = y * width;
for (let x = component.minX; x <= component.maxX; x += 1) {
const index = rowOffset + x;
if (rawMask[index]) selectedMask[index] = 255;
}
}
}
return {
mask: selectedMask,
maskSource: "reference-product",
selectedComponentCount: selected.length,
selectedPixelCount: selected.reduce((sum, component) => sum + component.area, 0),
};
}
async function buildLayoutLockedCompositeImage(params: {
generatedImageUrl: string;
referenceGeometry: { height: number; width: number };
referenceImageUrl: string;
}) {
const sharp = (await import("sharp")).default;
const [referenceSourceBuffer, generatedSourceBuffer] = await Promise.all([
fetchImageBufferFromUrl(params.referenceImageUrl),
fetchImageBufferFromUrl(params.generatedImageUrl),
]);
const targetWidth = params.referenceGeometry.width;
const targetHeight = params.referenceGeometry.height;
const referenceBuffer = await sharp(referenceSourceBuffer, { failOn: "none" })
.rotate()
.resize(targetWidth, targetHeight, {
fit: "fill",
withoutEnlargement: false,
})
.png()
.toBuffer();
const generatedBuffer = await sharp(generatedSourceBuffer, { failOn: "none" })
.rotate()
.resize(targetWidth, targetHeight, {
background: { r: 255, g: 255, b: 255, alpha: 1 },
fit: "contain",
position: "centre",
withoutEnlargement: false,
})
.png()
.toBuffer();
const [referenceRaw, generatedRaw] = await Promise.all([
sharp(referenceBuffer, { failOn: "none" })
.removeAlpha()
.raw()
.toBuffer({ resolveWithObject: true }),
sharp(generatedBuffer, { failOn: "none" })
.removeAlpha()
.raw()
.toBuffer({ resolveWithObject: true }),
]);
if (
referenceRaw.info.width !== generatedRaw.info.width ||
referenceRaw.info.height !== generatedRaw.info.height
) {
return null;
}
const selectedMask =
selectReferenceProductMask({
height: referenceRaw.info.height,
reference: referenceRaw.data,
width: referenceRaw.info.width,
}) ||
selectLayoutPatchMask({
generated: generatedRaw.data,
height: referenceRaw.info.height,
reference: referenceRaw.data,
width: referenceRaw.info.width,
});
if (!selectedMask) {
return null;
}
const usesReferenceProductMask = selectedMask.maskSource === "reference-product";
const firstBlurRadius = usesReferenceProductMask ? 14 : 10;
const maskThreshold = usesReferenceProductMask ? 12 : 2;
const pixelCount = targetWidth * targetHeight;
const alphaMask = await sharp(selectedMask.mask, {
raw: {
channels: 1,
height: targetHeight,
width: targetWidth,
},
})
.blur(firstBlurRadius)
.threshold(maskThreshold)
.blur(4)
.extractChannel(0)
.raw()
.toBuffer();
if (alphaMask.length !== pixelCount) {
throw new Error(
`排版替换蒙版通道异常: expected ${pixelCount}, got ${alphaMask.length}`,
);
}
const generatedPatchRaw = Buffer.alloc(pixelCount * 4);
const sourceChannels = generatedRaw.info.channels;
for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) {
const sourceOffset = pixelIndex * sourceChannels;
const targetOffset = pixelIndex * 4;
generatedPatchRaw[targetOffset] = generatedRaw.data[sourceOffset] || 0;
generatedPatchRaw[targetOffset + 1] =
generatedRaw.data[sourceOffset + 1] || 0;
generatedPatchRaw[targetOffset + 2] =
generatedRaw.data[sourceOffset + 2] || 0;
generatedPatchRaw[targetOffset + 3] = alphaMask[pixelIndex] || 0;
}
const generatedPatch = await sharp(generatedPatchRaw, {
raw: {
channels: 4,
height: targetHeight,
width: targetWidth,
},
})
.png()
.toBuffer();
const compositeBuffer = await sharp(referenceBuffer, { failOn: "none" })
.composite([{ input: generatedPatch }])
.png()
.toBuffer();
return {
buffer: compositeBuffer,
height: targetHeight,
maskSource: selectedMask.maskSource,
selectedComponentCount: selectedMask.selectedComponentCount,
selectedPixelCount: selectedMask.selectedPixelCount,
width: targetWidth,
};
}
export async function executeCanvasLayoutReplaceTask(params: {
request: CanvasLayoutReplaceTaskRequest;
userId: string;
}): Promise<CanvasGenerateImageResult> {
const sourceImageUrl = normalizeHttpUrl(params.request.sourceImageUrl);
const referenceImageUrl = normalizeHttpUrl(params.request.referenceImageUrl);
if (!sourceImageUrl) {
throw new Error("缺少当前产品图片");
}
if (!referenceImageUrl) {
throw new Error("缺少排版参考图");
}
const models = await getEnabledAiModelsWithProvider().catch((error) => {
console.warn("[Canvas Layout Replace] Failed to load models:", error);
return undefined;
});
const resolvedModelRef = await resolveDirectReplacementModelRef({
selectedChatModel: params.request.model,
finalModelId: params.request.model,
models,
});
if (!resolvedModelRef || resolvedModelRef === "auto") {
throw new Error("当前风格化未配置可用的生图模型");
}
const replacementStrategy: AgentDetailReplacementStrategy = "product_only";
const referenceLabel =
String(params.request.referenceLabel || "").trim() || "画布排版参考图";
const referenceGeometry =
await resolveReplacementReferenceGeometry(referenceImageUrl);
const planningRuntime = await resolveReplacementScenePlanningRuntime({ models });
const replacementExecutionPlan = await analyzeReplacementExecutionPlan({
modeType: "detail_replacement",
replacementStrategy,
referenceImageUrl,
referenceLabel,
productImageUrls: [sourceImageUrl],
modelImageUrl: null,
workflowPrompt: null,
planningRuntime,
models,
});
const basePrompt = "仅替换参考详情图中的产品主体";
const replacementPrompt = buildAgentReplacementPrompt({
basePrompt,
modeType: "detail_replacement",
replacementStrategy,
marketplaceProductDetailReference: false,
referenceLabel,
referenceIndex: 0,
totalReferences: 1,
hasModelImage: false,
productImageCount: 1,
originalProductImageCount: 1,
rerunReferenceCount: 0,
templateReferenceCount: 0,
scenePlan: replacementExecutionPlan.scenePlan,
referenceGeometry,
});
const supplement = normalizeLayoutReplacePromptSupplement(
params.request.supplementalPrompt,
);
const finalPrompt = [
replacementPrompt,
supplement
? `【画布排版替换补充】${supplement}。这段只用于理解新产品材质、颜色、结构或替换细节；不得改写、移动、删除、重排参考图中的任何文字、图标、按钮、标签、分割线、装饰、留白或页面结构。`
: "",
"【最终验收标准】输出是一张与最后一张参考图相同页面比例的单页结果；文字、图标、按钮、标签、分割线、品牌区、留白和信息层级必须视觉上与参考图一致，只允许旧产品主体/产品展示区被当前产品替换。",
]
.filter(Boolean)
.join("\n\n");
const generated = await generateCanvasImage({
prompt: finalPrompt,
displayPrompt: "排版替换结果",
model: resolvedModelRef,
ownerKey: params.userId,
aspectRatio: referenceGeometry?.aspectRatio || "auto",
imageSize: params.request.imageSize || "2K",
imageCount: 1,
promptExtend: false,
referenceImages: [sourceImageUrl, referenceImageUrl],
debugLabel: "canvas-layout-replace",
debugMeta: {
...(params.request.debugMeta || {}),
operation: "layout-replace",
layoutReplaceServerAlgorithm: "agent-detail-replacement/product_only",
sourceNodeId: params.request.sourceNodeId || null,
referenceWidth: referenceGeometry?.width,
referenceHeight: referenceGeometry?.height,
referenceAspectRatio: referenceGeometry?.aspectRatio,
pageKind: replacementExecutionPlan.scenePlan.pageKind,
generationRoute: replacementExecutionPlan.scenePlan.generationRoute,
promptPackPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy,
},
});
const firstImage = generated.images[0];
if (!firstImage?.originalUrl) {
throw new Error("排版替换未返回结果图片");
}
let normalizedImage = await normalizeReplacementResultImage({
generatedImageUrl: firstImage.originalUrl,
generatedThumbnailUrl: firstImage.thumbnailUrl,
referenceImageUrl,
referenceGeometry,
scenePlan: replacementExecutionPlan.scenePlan,
uploadPrefix: "canvas-layout-replace",
});
if (referenceGeometry) {
const lockedComposite = await buildLayoutLockedCompositeImage({
generatedImageUrl: firstImage.originalUrl,
referenceGeometry,
referenceImageUrl,
}).catch((error) => {
console.warn("[Canvas Layout Replace] Layout locked composite failed:", {
error: error instanceof Error ? error.message : String(error),
referenceImageUrl,
resultImageUrl: firstImage.originalUrl,
});
return null;
});
if (lockedComposite?.buffer) {
const thumbnailBuffer = await createAgentImageThumbnailBuffer(
lockedComposite.buffer,
);
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
const [originalUpload, thumbnailUpload] = await Promise.all([
uploadFile(
`canvas-layout-replace-locked-orig-${suffix}.png`,
bufferToArrayBuffer(lockedComposite.buffer),
),
uploadFile(
`canvas-layout-replace-locked-thumb-${suffix}.jpg`,
bufferToArrayBuffer(thumbnailBuffer),
),
]);
normalizedImage = {
originalUrl: originalUpload.url,
thumbnailUrl: thumbnailUpload.url || originalUpload.url,
imageSize: `${lockedComposite.width}x${lockedComposite.height}`,
};
console.info("[Canvas Layout Replace] Applied layout locked composite:", {
maskSource: lockedComposite.maskSource,
referenceWidth: lockedComposite.width,
referenceHeight: lockedComposite.height,
selectedComponentCount: lockedComposite.selectedComponentCount,
selectedPixelCount: lockedComposite.selectedPixelCount,
});
}
}
return {
images: [
{
originalUrl: normalizedImage.originalUrl,
thumbnailUrl: normalizedImage.thumbnailUrl || normalizedImage.originalUrl,
},
],
model: generated.model,
modelRef: generated.modelRef,
prompt: generated.prompt,
};
}

5
lib/db/migrate.ts
View File

@ -36,11 +36,12 @@ const providerBaseUrlDefaults = [
{ type: "bananaimage", baseUrl: "https://bananaimage.aigenmedia.art" },
{ type: "visionary", baseUrl: "https://api.visionary.beer" },
{ type: "atlascloud", baseUrl: "https://api.atlascloud.ai" },
{ type: "vip98k", baseUrl: "https://api.vip98k.com" },
{ type: "stepfun-image", baseUrl: "https://api.stepfun.com" },
{ type: "midjourney", baseUrl: "https://your-midjourney-api.example.com" },
{ type: "fal-video", baseUrl: "https://fal.run" },
{ type: "tripo-3d", baseUrl: "https://api.tripo3d.ai/v2/openapi" },
{ type: "worldlabs-3d", baseUrl: "https://api.worldlabs.ai" },
{ type: "worldlabs-3d", baseUrl: "https://api.vip98k.com" },
{ type: "kittycad-cad", baseUrl: "https://api.zoo.dev" },
{ type: "runway", baseUrl: "https://api.dev.runwayml.com" },
{ type: "krea", baseUrl: "https://api.krea.ai" },
@ -153,7 +154,7 @@ async function runMigrationStatements(db: ReturnType<typeof drizzle>) {
ADD CONSTRAINT "AiProvider_type_check"
CHECK (type IN (
'alibaba', 'openai', 'custom', 'local', 'gemini',
'grsai-gpt-image', 'grsai-video', 'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'bfl-image', 'ruxa-image', 'fal-image', 'snapgen-image', 'openai-image', 'bananaimage', 'visionary', 'atlascloud', 'stepfun-image', 'midjourney', 'fal-video', 'zhipu', 'tripo-3d', 'worldlabs-3d', 'kittycad-cad', 'runway', 'krea'
'grsai-gpt-image', 'grsai-video', 'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'bfl-image', 'ruxa-image', 'fal-image', 'snapgen-image', 'openai-image', 'bananaimage', 'visionary', 'atlascloud', 'vip98k', 'stepfun-image', 'midjourney', 'fal-video', 'zhipu', 'tripo-3d', 'worldlabs-3d', 'kittycad-cad', 'runway', 'krea'
));
`);
await db.execute(sql`

32
lib/db/migrations/0093_update_worldlabs_vip98k_base_url.sql Normal file
View File

@ -0,0 +1,32 @@
UPDATE "AiProvider"
SET
"baseUrl" = 'https://api.vip98k.com',
"updatedAt" = NOW()
WHERE
"type" = 'worldlabs-3d'
AND regexp_replace(BTRIM(COALESCE("baseUrl", '')), '/+$', '') IN (
'',
'https://api.worldlabs.ai',
'https://api.worldlabs.ai/marble/v1',
'https://api.vip98k.com/v1',
'https://api.vip98k.com/marble/v1'
);
UPDATE "SystemSettings"
SET
"value" = jsonb_set(
"value",
'{baseUrl}',
to_jsonb('https://api.vip98k.com'::text),
true
),
"updatedAt" = NOW()
WHERE
"key" = 'worldlabs_key_pool_config'
AND regexp_replace(BTRIM(COALESCE("value" ->> 'baseUrl', '')), '/+$', '') IN (
'',
'https://api.worldlabs.ai',
'https://api.worldlabs.ai/marble/v1',
'https://api.vip98k.com/v1',
'https://api.vip98k.com/marble/v1'
);

41
lib/db/migrations/0094_add_vip98k_provider_type.sql Normal file
View File

@ -0,0 +1,41 @@
ALTER TABLE "AiProvider"
DROP CONSTRAINT IF EXISTS "AiProvider_type_check";
ALTER TABLE "AiProvider"
ADD CONSTRAINT "AiProvider_type_check"
CHECK (
"type" IN (
'alibaba',
'openai',
'custom',
'local',
'gemini',
'grsai-gpt-image',
'grsai-video',
'huanke-gemini',
'qianwen-video',
'doubao-video',
'ruxa-video',
'volcano-vision',
'volcano-video',
'alibaba-image',
'bfl-image',
'ruxa-image',
'fal-image',
'snapgen-image',
'openai-image',
'bananaimage',
'visionary',
'atlascloud',
'vip98k',
'stepfun-image',
'midjourney',
'fal-video',
'zhipu',
'tripo-3d',
'worldlabs-3d',
'kittycad-cad',
'runway',
'krea'
)
);

14
lib/db/migrations/meta/_journal.json
View File

@ -540,6 +540,20 @@
"when": 1784592000000,
"tag": "0092_add_atlascloud_provider_type",
"breakpoints": true
},
{
"idx": 77,
"version": "7",
"when": 1785196800000,
"tag": "0093_update_worldlabs_vip98k_base_url",
"breakpoints": true
},
{
"idx": 78,
"version": "7",
"when": 1785369600000,
"tag": "0094_add_vip98k_provider_type",
"breakpoints": true
}
]
}

9
lib/db/queries.ts
View File

@ -3914,7 +3914,7 @@ export async function createAiProvider({
updatedBy,
}: {
name: string;
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "grsai-gpt-image" | "grsai-video" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "snapgen-image" | "openai-image" | "bananaimage" | "visionary" | "atlascloud" | "stepfun-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway" | "krea";
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "grsai-gpt-image" | "grsai-video" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "snapgen-image" | "openai-image" | "bananaimage" | "visionary" | "atlascloud" | "vip98k" | "stepfun-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway" | "krea";
baseUrl: string;
apiKey: string;
isEnabled?: boolean;
@ -3952,7 +3952,7 @@ export async function updateAiProvider({
}: {
id: string;
name?: string;
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "grsai-gpt-image" | "grsai-video" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "snapgen-image" | "openai-image" | "bananaimage" | "visionary" | "atlascloud" | "stepfun-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway" | "krea";
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "grsai-gpt-image" | "grsai-video" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "snapgen-image" | "openai-image" | "bananaimage" | "visionary" | "atlascloud" | "vip98k" | "stepfun-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway" | "krea";
baseUrl?: string;
apiKey?: string;
isEnabled?: boolean;
@ -3982,7 +3982,10 @@ export async function updateAiProvider({
export async function deleteAiProvider({ id }: { id: string }) {
try {
return await db.delete(aiProvider).where(eq(aiProvider.id, id)).returning();
return await db.transaction(async (tx) => {
await tx.delete(aiModel).where(eq(aiModel.providerId, id));
return tx.delete(aiProvider).where(eq(aiProvider.id, id)).returning();
});
} catch (_error) {
throw new ChatSDKError(
"bad_request:database",

2
lib/db/schema.ts
View File

@ -308,7 +308,7 @@ export type ToolConfig = InferSelectModel<typeof toolConfig>;
export const aiProvider = pgTable("AiProvider", {
id: uuid("id").primaryKey().notNull().defaultRandom(),
name: varchar("name", { length: 128 }).notNull(), // 提供商名称，如 "阿里云通义千问"
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "grsai-gpt-image", "grsai-video", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "stepfun-image", "midjourney", "fal-video", "zhipu", "tripo-3d", "worldlabs-3d", "kittycad-cad", "runway", "krea", "bfl-image"] })
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "grsai-gpt-image", "grsai-video", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "vip98k", "stepfun-image", "midjourney", "fal-video", "zhipu", "tripo-3d", "worldlabs-3d", "kittycad-cad", "runway", "krea", "bfl-image"] })
.notNull()
.default("custom"), // 提供商类型
baseUrl: text("baseUrl").notNull(), // API 基础 URL

1
lib/external-image-api/service.ts
View File

@ -348,6 +348,7 @@ function supportsImageEditForReferenceCount(
providerType === "bananaimage" ||
providerType === "visionary" ||
providerType === "atlascloud" ||
providerType === "vip98k" ||
providerType === "grsai-gpt-image" ||
providerType === "ruxa-image" ||
providerType === "snapgen-image";

8
lib/model3d/model-alias-config.ts
View File

@ -89,7 +89,7 @@ const DEFAULT_WORLD_MODEL_ALIASES: ThreeDModelAliasEntry[] = [
{
id: "world-standard",
displayName: "标准质量",
realModelId: "marble-1.1",
realModelId: "world-generation",
description: "默认世界生成质量，适合大多数空间创建。",
enabled: true,
isDefault: true,
@ -97,7 +97,7 @@ const DEFAULT_WORLD_MODEL_ALIASES: ThreeDModelAliasEntry[] = [
{
id: "world-fidelity",
displayName: "高保真",
realModelId: "marble-1.1-plus",
realModelId: "world-generation",
description: "更高保真与细节表现，适合高质量空间生成。",
enabled: true,
isDefault: false,
@ -105,7 +105,7 @@ const DEFAULT_WORLD_MODEL_ALIASES: ThreeDModelAliasEntry[] = [
{
id: "world-compatible",
displayName: "兼容质量",
realModelId: "marble-1.0",
realModelId: "world-generation",
description: "兼容旧版世界任务和历史工作流。",
enabled: true,
isDefault: false,
@ -113,7 +113,7 @@ const DEFAULT_WORLD_MODEL_ALIASES: ThreeDModelAliasEntry[] = [
{
id: "world-draft",
displayName: "快速草稿",
realModelId: "marble-1.0-draft",
realModelId: "world-generation",
description: "更适合快速预览与草稿生成。",
enabled: true,
isDefault: false,

2
lib/openclaw-chat/config.ts
View File

@ -1,6 +1,6 @@
export const OPENCLAW_CHAT_CONFIG_KEY = "openclaw_chat_config";
export const DEFAULT_OPENCLAW_CHAT_BASE_URL = "http://192.168.42.154:18790";
export const DEFAULT_OPENCLAW_CHAT_BASE_URL = "http://45.192.99.48:18790";
export type OpenClawChatConfig = {
enabled: boolean;

2
lib/scene-fusion/output-capabilities.ts
View File

@ -80,6 +80,7 @@ export function canProcessSceneFusionProviderType(providerType: string): boolean
case "bananaimage":
case "visionary":
case "atlascloud":
case "vip98k":
case "gemini":
case "volcano-vision":
return true;
@ -137,6 +138,7 @@ function getSupportedAspectRatiosByProvider(
case "bananaimage":
case "visionary":
case "atlascloud":
case "vip98k":
case "gemini":
default:
return [...ALL_ASPECT_RATIO_VALUES];

20
lib/worldlabs/client.ts
View File

@ -10,7 +10,8 @@ type WorldLabsRequestOptions = {
timeoutMs?: number;
};
const DEFAULT_WORLDLABS_BASE_URL = "https://api.worldlabs.ai";
export const DEFAULT_WORLDLABS_BASE_URL = "https://api.vip98k.com";
export const WORLDLABS_EXTERNAL_TASK_MODEL = "world-generation";
const DEFAULT_WORLDLABS_REQUEST_TIMEOUT_MS = 180_000;
function toIntMs(value: string | undefined) {
@ -66,6 +67,17 @@ function normalizeBaseUrl(baseUrl?: string) {
return resolved.replace(/\/+$/, "");
}
export function isWorldLabsExternalTaskApi(baseUrl?: string | null) {
const normalized = normalizeBaseUrl(baseUrl || "");
try {
const parsed = new URL(normalized);
const hostname = parsed.hostname.toLowerCase();
return hostname === "api.vip98k.com" || normalized.includes("/api/external/task");
} catch {
return normalized.includes("api.vip98k.com") || normalized.includes("/api/external/task");
}
}
function sanitizeApiKey(value: string): string {
return String(value || "")
.replace(/[\u2028\u2029]/g, "")
@ -122,7 +134,11 @@ export async function worldLabsRequest<T = unknown>(
});
}
headers.set("WLT-Api-Key", normalizedApiKey);
if (isWorldLabsExternalTaskApi(config.baseUrl)) {
headers.set("Authorization", `Bearer ${normalizedApiKey}`);
} else {
headers.set("WLT-Api-Key", normalizedApiKey);
}
headers.set("Accept", "application/json");
let body: BodyInit | undefined;

2
lib/worldlabs/key-pool-config-shared.ts
View File

@ -1,5 +1,5 @@
export const WORLDLABS_KEY_POOL_CONFIG_KEY = "worldlabs_key_pool_config";
export const DEFAULT_WORLDLABS_BASE_URL = "https://api.worldlabs.ai";
export const DEFAULT_WORLDLABS_BASE_URL = "https://api.vip98k.com";
export type WorldLabsKeyPoolConfig = {
importToken: string;

3
lib/worldlabs/server.ts
View File

@ -4,9 +4,8 @@ import {
getAiProviderById,
getAiProvidersByType,
} from "@/lib/db/queries";
import { DEFAULT_WORLDLABS_BASE_URL } from "./client";
import type { WorldLabsProviderConfig } from "./types";
const DEFAULT_WORLDLABS_BASE_URL = "https://api.worldlabs.ai";
let worldLabsProviderCursor = 0;
export function sanitizeWorldLabsApiKey(value: string): string {

42
lib/worldlabs/types.ts
View File

@ -1,4 +1,5 @@
export type WorldLabsModel =
| "world-generation"
| "marble-1.0-draft"
| "marble-1.0"
| "marble-1.1"
@ -19,8 +20,11 @@ export interface WorldLabsMediaAsset {
}
export interface WorldLabsUploadInfo {
upload_url: string;
upload_method: string;
upload_url?: string;
uploadUrl?: string;
url?: string;
upload_method?: string;
method?: string;
required_headers?: Record<string, string> | null;
curl_example?: string | null;
}
@ -75,3 +79,37 @@ export interface WorldLabsProviderConfig {
providerId?: string;
providerName?: string;
}
export interface WorldLabsExternalTaskCreateData {
taskId?: string;
task_id?: string;
id?: string;
status?: string;
done?: boolean;
resultPath?: string;
result_path?: string;
[key: string]: unknown;
}
export interface WorldLabsExternalTaskResultData {
taskId?: string;
task_id?: string;
id?: string;
status?: string;
done?: boolean;
error?: unknown;
errorMessage?: string;
message?: string;
result?: unknown;
output?: unknown;
[key: string]: unknown;
}
export interface WorldLabsExternalTaskResponse<T = unknown> {
data?: T;
requestId?: string;
request_id?: string;
message?: string;
error?: unknown;
[key: string]: unknown;
}

2
next-env.d.ts vendored
View File

@ -1,6 +1,6 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />
import "./.next/types/routes.d.ts";
import "./.next/dev/types/routes.d.ts";
// NOTE: This file should not be edited
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

45
services/canvas/index.ts
View File

@ -384,6 +384,9 @@ export const imageGenerationService = {
referenceImage?: string;
referenceImages?: string[];
billingFeature?: 'multi-view' | 'repaint';
displayPrompt?: string;
debugLabel?: string;
debugMeta?: Record<string, unknown>;
async?: boolean;
}): Promise<
ApiResponse<{
@ -419,6 +422,44 @@ export const imageGenerationService = {
}
},
async layoutReplace(params: {
sourceImageUrl: string;
referenceImageUrl: string;
supplementalPrompt?: string;
model?: string;
imageSize?: string;
sourceNodeId?: string;
referenceLabel?: string;
debugMeta?: Record<string, unknown>;
}): Promise<
ApiResponse<{
status?: string;
taskId: string;
}>
> {
try {
const response = await fetch('/api/canvas/layout-replace', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
const data = await response.json();
if (data.success && data.taskId) {
return {
success: true,
data: {
status: data.status,
taskId: data.taskId,
},
};
}
return { success: false, error: data.error || '排版替换提交失败' };
} catch (error) {
logCatch(error, "layout_replace_submit");
return { success: false, error: '排版替换提交失败' };
}
},
async localEdit(params: {
sourceImageUrl: string;
sourceNodeId?: string;
@ -1245,10 +1286,12 @@ export const worldLabsGenerationService = {
): Promise<
ApiResponse<{
mediaAssetId?: string;
sourceUrl?: string;
mediaAsset?: Record<string, any>;
uploadInfo?: Record<string, any>;
providerId?: string | null;
providerName?: string | null;
uploadFallback?: string | null;
}>
> {
try {
@ -1276,10 +1319,12 @@ export const worldLabsGenerationService = {
): Promise<
ApiResponse<{
mediaAssetId?: string;
sourceUrl?: string;
mediaAsset?: Record<string, any>;
uploadInfo?: Record<string, any>;
providerId?: string | null;
providerName?: string | null;
uploadFallback?: string | null;
}>
> {
try {

6
store/canvas/index.ts
View File

@ -73,7 +73,11 @@ export interface PickedObject {
};
}
export type ImageNodeActionType = 'box-cutout' | 'text-edit' | 'local-edit';
export type ImageNodeActionType =
| 'box-cutout'
| 'text-edit'
| 'local-edit'
| 'layout-replace';
export interface ImageNodeActionRequest {
nodeId: string;