chore: sync latest app updates
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-04-12 23:45:00 +08:00
parent 1b239eb2c7
commit dfc5f704b4
44 changed files with 3634 additions and 1089 deletions

View File

@ -345,6 +345,7 @@ type AiProvider = {
| "worldlabs-3d"
| "kittycad-cad"
| "runway"
| "krea"
| "volcano-multimodal";
baseUrl: string;
apiKey: string;
@ -720,6 +721,7 @@ const PROVIDER_TYPES = [
{ value: "worldlabs-3d", label: "World Labs 3D 世界生成" },
{ value: "kittycad-cad", label: "KittyCAD / Zoo CAD" },
{ value: "runway", label: "Runway API" },
{ value: "krea", label: "Krea API" },
{ value: "zhipu", label: "智普官方 4.7" },
{ value: "volcano-multimodal", label: "火山引擎多模态" },
{ value: "openai", label: "OpenAI 兼容" },
@ -749,6 +751,7 @@ const PROVIDER_DEFAULT_BASE_URL: Record<string, string> = {
"worldlabs-3d": "https://api.worldlabs.ai",
"kittycad-cad": "https://api.zoo.dev",
runway: "https://api.dev.runwayml.com",
krea: "https://api.krea.ai",
zhipu: "https://open.bigmodel.cn/api/paas/v4",
"volcano-multimodal": "https://ark.cn-beijing.volces.com/api/v3",
openai: "https://api.openai.com/v1",
@ -2377,6 +2380,7 @@ export function AdminSettingsClient() {
| "worldlabs-3d"
| "kittycad-cad"
| "runway"
| "krea"
| "volcano-multimodal",
baseUrl: "",
apiKey: "",
@ -2476,8 +2480,11 @@ export function AdminSettingsClient() {
modelForm.imageGenConfig
);
const modelKindHint =
selectedModelProviderType === "runway" && !modelForm.modelId.trim()
? "Runway 同时支持图片与视频模型,填写模型 ID 后系统会自动识别标签。"
(selectedModelProviderType === "runway" || selectedModelProviderType === "krea") &&
!modelForm.modelId.trim()
? `${
selectedModelProviderType === "krea" ? "Krea" : "Runway"
} 同时支持图片与视频模型,填写模型 ID 后系统会自动识别标签。`
: isVideoProviderForModelConfig
? "当前识别为视频模型,将使用视频能力、视频积分和视频路由。"
: isImageProviderForModelConfig
@ -4208,6 +4215,7 @@ export function AdminSettingsClient() {
const method = editingProvider ? "PUT" : "POST";
const isNewProvider = !editingProvider;
const isVolcanoVideo = providerForm.type === "volcano-video";
const isKreaProvider = providerForm.type === "krea";
const isLemonProvider =
providerForm.type === "openai" &&
isLemonApiBaseUrl(
@ -4280,6 +4288,28 @@ export function AdminSettingsClient() {
}
}
if (isNewProvider && isKreaProvider && providerId) {
toast.info("正在自动添加 Krea 官方预设模型...");
try {
const modelsResponse = await fetch("/api/admin/krea-models", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ providerId }),
});
const modelsData = await modelsResponse.json();
if (modelsResponse.ok && modelsData.addedCount > 0) {
toast.success(`已自动添加 ${modelsData.addedCount} 个 Krea 预设模型`);
mutateModels();
}
} catch (modelError) {
console.error("[Auto Add Krea Models] Error:", modelError);
toast.warning('Krea 预设模型添加失败,请手动点击"快速添加 Krea 预设模型"按钮');
}
}
if (isNewProvider && isLemonProvider && providerId) {
try {
await handleQuickAddLemonModels(providerId);
@ -4903,6 +4933,46 @@ export function AdminSettingsClient() {
}
};
const handleQuickAddKreaModels = async (providerId: string) => {
if (
!confirm(
"将为此服务商添加 Krea 官方 API 的全部图片、图片增强和视频模型预设。\n\n继续吗?"
)
) {
return;
}
try {
toast.info("正在添加 Krea 官方预设模型...");
const response = await fetch("/api/admin/krea-models", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ providerId }),
});
const data = await response.json();
if (!response.ok) {
throw new Error(data.error || "添加失败");
}
const { addedCount, errors } = data;
if (addedCount > 0) {
toast.success(`成功添加 ${addedCount} 个 Krea 预设模型`);
mutateModels();
}
if (errors && errors.length > 0) {
toast.warning(`${errors.length} 个模型添加失败(可能已存在)`);
}
} catch (error: any) {
console.error("[Quick Add Krea Models] Error:", error);
toast.error(error.message || "添加 Krea 预设模型失败");
}
};
const handleUseLemonProviderPreset = () => {
setProviderForm({
id: "",
@ -6948,6 +7018,7 @@ export function AdminSettingsClient() {
zhipu: "bg-lime-500",
"volcano-multimodal": "bg-fuchsia-500",
runway: "bg-rose-500",
krea: "bg-blue-500",
"qianwen-video": "bg-cyan-500",
custom: "bg-gray-500",
local: "bg-yellow-500",
@ -11104,6 +11175,8 @@ export function AdminSettingsClient() {
? "留空则不修改"
: providerForm.type === "runway"
? "key_ 后跟 128 位十六进制字符"
: providerForm.type === "krea"
? "Krea API Key"
: "sk-..."
}
/>
@ -11128,6 +11201,12 @@ export function AdminSettingsClient() {
Key:”或中文标点的内容。
</p>
)}
{providerForm.type === "krea" && (
<p className="text-xs text-muted-foreground">
Krea 使用 Bearer Token 认证。保存后会自动添加官方
OpenAPI 中的图片、图片增强和视频模型预设。
</p>
)}
</div>
<div className="flex items-center gap-2">
<Switch
@ -11192,6 +11271,17 @@ export function AdminSettingsClient() {
快速添加预设模型
</Button>
)}
{provider.type === "krea" && (
<Button
variant="secondary"
size="sm"
className="mt-2"
onClick={() => handleQuickAddKreaModels(provider.id)}
>
<Sparkles className="h-4 w-4 mr-1" />
快速添加 Krea 预设模型
</Button>
)}
{provider.type === "openai" &&
isLemonApiBaseUrl(provider.baseUrl) && (
<Button

File diff suppressed because it is too large Load Diff

View File

@ -33,6 +33,7 @@ const providerTypes = [
"worldlabs-3d",
"kittycad-cad",
"runway",
"krea",
"volcano-multimodal",
] as const;
type ProviderType = (typeof providerTypes)[number];
@ -67,6 +68,7 @@ const TRIPO_3D_BASE_URL = "https://api.tripo3d.ai/v2/openapi";
const WORLDLABS_3D_BASE_URL = "https://api.worldlabs.ai";
const KITTYCAD_CAD_BASE_URL = "https://api.zoo.dev";
const RUNWAY_BASE_URL = "https://api.dev.runwayml.com";
const KREA_BASE_URL = "https://api.krea.ai";
const RUNWAY_API_KEY_PATTERN = /^key_[0-9a-f]{128}$/;
function normalizeProviderTypeForStorage(
@ -218,6 +220,8 @@ export async function POST(request: Request) {
? data.baseUrl || KITTYCAD_CAD_BASE_URL
: data.type === "runway"
? data.baseUrl || RUNWAY_BASE_URL
: data.type === "krea"
? data.baseUrl || KREA_BASE_URL
: data.baseUrl,
updatedBy: session.user.id,
});

View File

@ -0,0 +1,118 @@
import { NextResponse } from "next/server";
import { and, eq } from "drizzle-orm";
import { requireAdmin } from "@/lib/middleware/admin";
import { db } from "@/lib/db";
import { aiModel } from "@/lib/db/schema";
import { getAiProviderById } from "@/lib/db/queries";
import { KREA_MODEL_PRESETS } from "@/lib/ai/krea-models";
export async function GET() {
try {
await requireAdmin();
return NextResponse.json({
presets: KREA_MODEL_PRESETS,
counts: {
image: KREA_MODEL_PRESETS.filter((preset) => preset.kind === "image").length,
imageEnhance: KREA_MODEL_PRESETS.filter((preset) => preset.kind === "image-enhance").length,
video: KREA_MODEL_PRESETS.filter((preset) => preset.kind === "video").length,
total: KREA_MODEL_PRESETS.length,
},
});
} catch (error: any) {
console.error("[Krea Models API] GET error:", error);
return NextResponse.json(
{ error: error.message || "Failed to fetch Krea model presets" },
{ status: 500 }
);
}
}
export async function POST(request: Request) {
try {
await requireAdmin();
const body = await request.json();
const providerId = String(body?.providerId || "").trim();
const modelIds = Array.isArray(body?.modelIds)
? body.modelIds.map((item: unknown) => String(item || "").trim()).filter(Boolean)
: null;
if (!providerId) {
return NextResponse.json({ error: "Provider ID is required" }, { status: 400 });
}
const provider = await getAiProviderById({ id: providerId });
if (!provider || provider.type !== "krea") {
return NextResponse.json(
{ error: "Provider must be a Krea provider" },
{ status: 400 }
);
}
const presetsToAdd = modelIds
? KREA_MODEL_PRESETS.filter((preset) => modelIds.includes(preset.modelId))
: KREA_MODEL_PRESETS;
const addedModels = [];
const errors = [];
for (const preset of presetsToAdd) {
try {
const existing = await db.query.aiModel.findFirst({
where: and(
eq(aiModel.providerId, providerId),
eq(aiModel.modelId, preset.modelId)
),
});
if (existing) {
errors.push({ modelId: preset.modelId, error: "Model already exists" });
continue;
}
const [newModel] = await db
.insert(aiModel)
.values({
providerId,
modelId: preset.modelId,
displayName: preset.displayName,
description: preset.description,
supportsVision: preset.supportsVision,
supportsImageGen: preset.supportsImageGen,
isEnabled: true,
isHidden: false,
isDefault: false,
sortOrder: preset.sortOrder,
capabilities: preset.capabilities || null,
defaultParams: preset.defaultParams || null,
imageGenConfig: preset.imageGenConfig || null,
isSubModel: false,
parentModelId: null,
})
.returning();
addedModels.push(newModel);
} catch (error: any) {
console.error(`[Krea Models API] Failed to add ${preset.modelId}:`, error);
errors.push({ modelId: preset.modelId, error: error.message || "Insert failed" });
}
}
return NextResponse.json({
success: true,
addedCount: addedModels.length,
addedModels,
errors: errors.length > 0 ? errors : undefined,
counts: {
requested: presetsToAdd.length,
total: KREA_MODEL_PRESETS.length,
},
});
} catch (error: any) {
console.error("[Krea Models API] POST error:", error);
return NextResponse.json(
{ error: error.message || "Failed to add Krea models" },
{ status: 500 }
);
}
}

View File

@ -1,4 +1,4 @@
import { and, desc, eq } from "drizzle-orm";
import { and, desc, eq, inArray } from "drizzle-orm";
import { randomUUID } from "node:crypto";
import { NextRequest, NextResponse } from "next/server";
import { auth } from "@/app/(auth)/auth";
@ -362,14 +362,30 @@ export async function PATCH(
}
const body = await request.json().catch(() => ({}));
const taskId = String((body as any)?.taskId || "").trim();
if (!taskId) {
const rawTaskIds = Array.isArray((body as any)?.taskIds)
? (body as any).taskIds
: [(body as any)?.taskId];
const taskIds = Array.from(
new Set<string>(
rawTaskIds
.map((value: unknown) => String(value || "").trim())
.filter(Boolean)
)
);
if (taskIds.length === 0) {
return NextResponse.json({ success: false, error: "缺少 taskId" }, { status: 400 });
}
if (taskIds.length > 30) {
return NextResponse.json(
{ success: false, error: "一次最多分享 30 个 3D 世界" },
{ status: 400 }
);
}
const [existingAsset] = await db
const existingAssets = await db
.select({
id: model3dAsset.id,
taskId: model3dAsset.taskId,
provider: model3dAsset.provider,
status: model3dAsset.status,
shareEnabled: model3dAsset.shareEnabled,
@ -380,23 +396,22 @@ export async function PATCH(
and(
eq(model3dAsset.projectId, projectId),
eq(model3dAsset.userId, session.user.id),
eq(model3dAsset.taskId, taskId)
inArray(model3dAsset.taskId, taskIds)
)
)
.limit(1);
);
if (!existingAsset) {
if (existingAssets.length !== taskIds.length) {
return NextResponse.json({ success: false, error: "资产不存在" }, { status: 404 });
}
if (existingAsset.provider !== "worldlabs-3d") {
if (existingAssets.some((asset) => asset.provider !== "worldlabs-3d")) {
return NextResponse.json(
{ success: false, error: "当前仅支持分享 3D 世界资产" },
{ status: 400 }
);
}
if (existingAsset.status !== "success") {
if (existingAssets.some((asset) => asset.status !== "success")) {
return NextResponse.json(
{ success: false, error: "仅成功生成的 3D 世界支持分享" },
{ status: 400 }
@ -404,10 +419,19 @@ export async function PATCH(
}
const shouldEnable = (body as any)?.shareEnabled !== false;
const existingShareToken = existingAssets.find(
(asset) => asset.shareEnabled && asset.shareToken
)?.shareToken;
const shareToken =
shouldEnable
? taskIds.length > 1
? existingShareToken || createShareToken()
: existingAssets[0]?.shareToken || createShareToken()
: null;
const nextValues = shouldEnable
? {
shareEnabled: true,
shareToken: existingAsset.shareToken || createShareToken(),
shareToken,
sharedAt: new Date(),
updatedAt: new Date(),
}
@ -418,13 +442,24 @@ export async function PATCH(
updatedAt: new Date(),
};
const [updatedAsset] = await db
const updatedAssets = await db
.update(model3dAsset)
.set(nextValues)
.where(eq(model3dAsset.id, existingAsset.id))
.where(inArray(model3dAsset.id, existingAssets.map((asset) => asset.id)))
.returning();
const updatedAssetByTaskId = new Map(
updatedAssets.map((asset) => [asset.taskId, asset])
);
const orderedUpdatedAssets = taskIds
.map((id) => updatedAssetByTaskId.get(id))
.filter((asset): asset is (typeof updatedAssets)[number] => Boolean(asset));
return NextResponse.json({ success: true, data: updatedAsset });
return NextResponse.json({
success: true,
data: orderedUpdatedAssets[0] || updatedAssets[0] || null,
items: orderedUpdatedAssets,
shareToken,
});
} catch (error) {
console.error("[Model3DAssets] PATCH failed:", error);
return NextResponse.json({ success: false, error: "更新 3D 世界分享状态失败" }, { status: 500 });

View File

@ -105,6 +105,7 @@ const MEMBERSHIP_CONFIG_KEY = "membership_config";
const MEMBERSHIP_USER_OVERRIDES_KEY = "membership_user_plan_overrides";
const STORAGE_FAILOVER_CONFIG_KEY = "storage_failover_config";
const REALTIME_CANVAS_CONFIG_KEY = "realtime_canvas_config";
const LAYER_SPLIT_CONFIG_KEY = "layer_split_config";
const AGENT_CATEGORY_TREE_CONFIG_KEY = "agent_category_tree_config";
const CANVAS_AGENT_MODE_ENABLED_KEY = "canvas_agent_mode_enabled";
const VIDEO_CANVAS_SIDEBAR_ENABLED_KEY = "video_canvas_sidebar_enabled";
@ -230,6 +231,13 @@ function normalizeVideoCanvasSidebarEnabled(value: unknown): boolean {
return value !== false;
}
function normalizeLayerSplitEnabled(value: unknown): boolean {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return false;
}
return (value as { enabled?: unknown }).enabled === true;
}
function normalizeAgentCategoryTreeConfig(value: unknown): AgentCategoryTreeConfig {
const raw =
value && typeof value === "object" && !Array.isArray(value)
@ -278,6 +286,7 @@ export async function GET() {
pointsPackagesSetting,
dulupayConfigSetting,
realtimeCanvasConfigSetting,
layerSplitConfigSetting,
canvasAgentModeEnabledSetting,
videoCanvasSidebarEnabledSetting,
storageFailoverConfigSetting,
@ -380,6 +389,12 @@ export async function GET() {
.where(eq(systemSettings.key, REALTIME_CANVAS_CONFIG_KEY))
.limit(1)
.then(rows => rows[0]),
db
.select()
.from(systemSettings)
.where(eq(systemSettings.key, LAYER_SPLIT_CONFIG_KEY))
.limit(1)
.then(rows => rows[0]),
db
.select()
.from(systemSettings)
@ -500,6 +515,9 @@ export async function GET() {
: normalizeDulupayForPublic(dulupayConfigSetting?.value);
const realtimeCanvasServiceEnabled =
(realtimeCanvasConfigSetting?.value as { enabled?: boolean } | undefined)?.enabled !== false;
const layerSplitEnabled = normalizeLayerSplitEnabled(
layerSplitConfigSetting?.value
);
const canvasAgentModeEnabled = normalizeCanvasAgentModeEnabled(
canvasAgentModeEnabledSetting?.value
);
@ -600,6 +618,7 @@ export async function GET() {
defaultAgentPsdTemplateCategoryTagIndex,
realtimeCanvasEnabled: true,
realtimeCanvasServiceEnabled: true,
layerSplitEnabled,
storageFailoverConfig,
ossImageMigrationConfig,
canvasDefaultModelId: null,
@ -658,6 +677,7 @@ export async function GET() {
pointsPackages,
dulupayConfig,
realtimeCanvasServiceEnabled,
layerSplitEnabled,
canvasAgentModeEnabled,
videoCanvasSidebarEnabled,
storageFailoverConfig,

View File

@ -7,6 +7,7 @@ import {
Bone,
Box,
Boxes,
Check,
ChevronDown,
ChevronUp,
Download,
@ -618,6 +619,7 @@ export default function CanvasModel3DPage() {
const [taskHistory, setTaskHistory] = useState<TaskRecord[]>([]);
const [currentTask, setCurrentTask] = useState<TaskRecord | null>(null);
const [selectedWorldShareTaskIds, setSelectedWorldShareTaskIds] = useState<string[]>([]);
const [autoDownloadTaskId, setAutoDownloadTaskId] = useState<string | null>(
null
);
@ -1032,6 +1034,25 @@ export default function CanvasModel3DPage() {
() => taskHistory.filter((task) => taskMatchesWorkspace(task, workspaceMode)),
[taskHistory, workspaceMode]
);
const shareableWorldTasks = useMemo(
() =>
visibleTaskHistory.filter(
(task) => task.provider === "worldlabs-3d" && task.status === "success"
),
[visibleTaskHistory]
);
const selectedWorldShareTasks = useMemo(() => {
const selectedIds = new Set(selectedWorldShareTaskIds);
return shareableWorldTasks.filter((task) => selectedIds.has(task.taskId));
}, [selectedWorldShareTaskIds, shareableWorldTasks]);
useEffect(() => {
const shareableIds = new Set(shareableWorldTasks.map((task) => task.taskId));
setSelectedWorldShareTaskIds((prev) => {
const next = prev.filter((taskId) => shareableIds.has(taskId));
return next.length === prev.length ? prev : next;
});
}, [shareableWorldTasks]);
const currentModelUrl = resolveTaskModelUrl(currentTask);
const currentPreviewImage =
@ -1920,6 +1941,75 @@ export default function CanvasModel3DPage() {
[updateWorldShareState]
);
const handleToggleWorldShareSelection = useCallback(
(task: TaskRecord, event?: MouseEvent<HTMLButtonElement>) => {
event?.stopPropagation();
if (task.provider !== "worldlabs-3d" || task.status !== "success") {
return;
}
setSelectedWorldShareTaskIds((prev) =>
prev.includes(task.taskId)
? prev.filter((taskId) => taskId !== task.taskId)
: [...prev, task.taskId]
);
},
[]
);
const handleSelectAllShareableWorlds = useCallback(() => {
setSelectedWorldShareTaskIds((prev) =>
prev.length === shareableWorldTasks.length
? []
: shareableWorldTasks.map((task) => task.taskId)
);
}, [shareableWorldTasks]);
const handleShareSelectedWorlds = useCallback(async () => {
if (selectedWorldShareTasks.length < 2) {
toast.error("请至少选择 2 个成功生成的 3D 世界");
return;
}
try {
const response = await fetch(`/api/canvas/${id}/model3d-assets`, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
taskIds: selectedWorldShareTasks.map((task) => task.taskId),
shareEnabled: true,
}),
});
const payload = (await response.json().catch(() => ({}))) as {
success?: boolean;
error?: string;
items?: TaskRecordApiEntity[];
shareToken?: string | null;
};
if (!response.ok || payload?.success === false) {
throw new Error(payload?.error || "批量分享世界失败");
}
const updatedTasks = Array.isArray(payload.items)
? payload.items
.map((item) => mapApiEntityToTaskRecord(item))
.filter((item): item is TaskRecord => Boolean(item))
: [];
for (const updatedTask of updatedTasks) {
upsertTaskHistory(updatedTask, { persist: false });
}
const shareToken = payload.shareToken || updatedTasks[0]?.shareToken;
const shareUrl = buildWorldShareUrl(shareToken);
if (!shareUrl) {
throw new Error("分享链接生成失败");
}
await navigator.clipboard.writeText(shareUrl);
toast.success(`已复制包含 ${selectedWorldShareTasks.length} 个世界的分享链接`);
} catch (error) {
toast.error(error instanceof Error ? error.message : "批量分享世界失败");
}
}, [id, selectedWorldShareTasks, upsertTaskHistory]);
const handleDisableCurrentShare = useCallback(async () => {
if (!currentTask || currentTask.provider !== "worldlabs-3d") {
toast.error("当前没有可关闭分享的世界资产");
@ -3570,14 +3660,52 @@ export default function CanvasModel3DPage() {
<X className="h-4 w-4" />
</Button>
</div>
{workspaceMode === "world" ? (
<div className="space-y-2 border-white/10 border-b px-3 py-3">
<div className="flex items-center justify-between gap-2 text-[11px] text-zinc-500">
<span>多世界分享</span>
<span>已选 {selectedWorldShareTasks.length}</span>
</div>
<div className="grid grid-cols-2 gap-2">
<Button
className="h-8 border-white/10 bg-zinc-950/60 text-xs text-zinc-200 hover:bg-zinc-800"
disabled={shareableWorldTasks.length === 0}
onClick={handleSelectAllShareableWorlds}
size="sm"
variant="outline"
>
{selectedWorldShareTaskIds.length === shareableWorldTasks.length &&
shareableWorldTasks.length > 0
? "取消全选"
: "全选世界"}
</Button>
<Button
className="h-8 border-cyan-400/30 bg-cyan-400/15 text-xs text-cyan-100 hover:bg-cyan-400/25"
disabled={selectedWorldShareTasks.length < 2}
onClick={handleShareSelectedWorlds}
size="sm"
variant="outline"
>
分享所选
</Button>
</div>
</div>
) : null}
<ScrollArea className="flex-1">
<div className="grid grid-cols-2 gap-2 p-3">
{visibleTaskHistory.map((task) => (
{visibleTaskHistory.map((task) => {
const canShareWorld =
task.provider === "worldlabs-3d" && task.status === "success";
const isSelectedForShare = selectedWorldShareTaskIds.includes(task.taskId);
return (
<button
className={cn(
"group relative aspect-square overflow-hidden rounded-lg border border-white/5 bg-zinc-950 transition-all hover:border-yellow-500/50",
currentTask?.taskId === task.taskId &&
"border-yellow-500 ring-2 ring-yellow-500/30 shadow-[0_18px_40px_rgba(0,0,0,0.28)]"
"border-yellow-500 ring-2 ring-yellow-500/30 shadow-[0_18px_40px_rgba(0,0,0,0.28)]",
isSelectedForShare &&
"border-cyan-300 ring-2 ring-cyan-300/40"
)}
key={task.taskId}
onClick={() => handleSelectTask(task)}
@ -3588,7 +3716,7 @@ export default function CanvasModel3DPage() {
当前
</Badge>
) : null}
{task.provider === "worldlabs-3d" && task.status === "success" ? (
{canShareWorld ? (
<Button
className={cn(
"absolute left-2 top-2 z-10 h-7 w-7 rounded-full border border-white/10 bg-black/60 text-zinc-100 opacity-0 shadow-lg backdrop-blur-sm transition-opacity hover:bg-black/80 group-hover:opacity-100",
@ -3603,6 +3731,24 @@ export default function CanvasModel3DPage() {
<Link2 className="h-3.5 w-3.5" />
</Button>
) : null}
{canShareWorld ? (
<Button
className={cn(
"absolute left-10 top-2 z-10 h-7 rounded-full border border-white/10 bg-black/60 px-2 text-[10px] text-zinc-100 opacity-0 shadow-lg backdrop-blur-sm transition-opacity hover:bg-black/80 group-hover:opacity-100",
isSelectedForShare &&
"border-cyan-300/60 bg-cyan-400/25 text-cyan-50 opacity-100"
)}
onClick={(event) => handleToggleWorldShareSelection(task, event)}
size="sm"
type="button"
variant="ghost"
>
{isSelectedForShare ? (
<Check className="mr-1 h-3 w-3" />
) : null}
{isSelectedForShare ? "已选" : "选择"}
</Button>
) : null}
{task.status === "success" && resolveTaskModelUrl(task) ? (
<Button
className="absolute right-2 top-2 z-10 h-7 w-7 rounded-full border border-white/10 bg-black/60 text-zinc-100 opacity-0 shadow-lg backdrop-blur-sm transition-opacity hover:bg-black/80 group-hover:opacity-100"
@ -3663,7 +3809,8 @@ export default function CanvasModel3DPage() {
) : null}
</div>
</button>
))}
);
})}
{visibleTaskHistory.length === 0 && (
<div className="col-span-2 py-10 text-center text-xs text-zinc-500">
{workspaceMode === "world" ? "暂无世界记录" : "暂无模型记录"}

View File

@ -11,7 +11,6 @@ import {
ChevronRight,
Clapperboard,
CheckCircle2,
Crosshair,
Download,
Eye,
FileImage,
@ -1402,10 +1401,6 @@ export default function CanvasEditorPage({ params }: PageProps) {
const updateNode = useCanvasStore((state) => state.updateNode);
const pushHistory = useCanvasStore((state) => state.pushHistory);
const setGenerating = useCanvasStore((state) => state.setGenerating);
const setPickerMode = useCanvasStore((state) => state.setPickerMode);
const requestImageNodeAction = useCanvasStore(
(state) => state.requestImageNodeAction
);
// Local state
const [isLoading, setIsLoading] = useState(true);
@ -1537,6 +1532,7 @@ export default function CanvasEditorPage({ params }: PageProps) {
agentEnabled?: boolean;
canvasAgentModeEnabled?: boolean;
videoCanvasSidebarEnabled?: boolean;
layerSplitEnabled?: boolean;
realtimeCanvasEnabled?: boolean;
realtimeCanvasServiceEnabled?: boolean;
agentWorkflowShowcaseConfig?: AgentWorkflowShowcaseConfig;
@ -8616,6 +8612,7 @@ export default function CanvasEditorPage({ params }: PageProps) {
showEmptyState={!isVideoMode}
workspaceLabel={workspaceLabel}
workspaceVariant={isVideoMode ? "video" : "canvas"}
layerSplitEnabled={siteSettingsData?.layerSplitEnabled === true}
/>
</div>
@ -9736,61 +9733,6 @@ export default function CanvasEditorPage({ params }: PageProps) {
</DropdownMenu>
) : null}
<Tooltip>
<TooltipTrigger asChild>
<Button
className={cn(
isVideoMode
? VIDEO_CANVAS_TOOLBAR_BUTTON_CLASS
: "h-9 w-9 rounded-xl"
)}
disabled={!selectedImageNode}
onClick={() => {
if (!selectedImageNode) {
toast.info("请先选中一张图片");
return;
}
setPickerMode(true, selectedImageNode.id);
toast.info("点击图片中的物品进行拾取");
}}
size="icon"
variant="ghost"
>
<Crosshair className="h-4 w-4" />
</Button>
</TooltipTrigger>
<TooltipContent side="top">
{selectedImageNode ? "识物" : "先选中图片后识物"}
</TooltipContent>
</Tooltip>
<Tooltip>
<TooltipTrigger asChild>
<Button
className={cn(
isVideoMode
? VIDEO_CANVAS_TOOLBAR_BUTTON_CLASS
: "h-9 w-9 rounded-xl"
)}
disabled={!selectedImageNode}
onClick={() => {
if (!selectedImageNode) {
toast.info("请先选中一张图片");
return;
}
requestImageNodeAction(selectedImageNode.id, "box-cutout");
}}
size="icon"
variant="ghost"
>
<Scan className="h-4 w-4" />
</Button>
</TooltipTrigger>
<TooltipContent side="top">
{selectedImageNode ? "框选抠图" : "先选中图片后框选抠图"}
</TooltipContent>
</Tooltip>
{siteSettingsData?.realtimeCanvasEnabled === true &&
siteSettingsData?.realtimeCanvasServiceEnabled !== false ? (
<Tooltip>

View File

@ -1,4 +1,4 @@
import { and, eq } from "drizzle-orm";
import { and, desc, eq } from "drizzle-orm";
import { notFound } from "next/navigation";
import { WorldSharePage } from "@/components/canvas/world3d/WorldSharePage";
import { db } from "@/lib/db";
@ -46,7 +46,7 @@ export default async function SharedWorldPage({
notFound();
}
const [asset] = await db
const assets = await db
.select({
taskId: model3dAsset.taskId,
prompt: model3dAsset.prompt,
@ -54,6 +54,8 @@ export default async function SharedWorldPage({
previewImageUrl: model3dAsset.previewImageUrl,
modelUrl: model3dAsset.modelUrl,
assetPayload: model3dAsset.assetPayload,
sharedAt: model3dAsset.sharedAt,
updatedAt: model3dAsset.updatedAt,
})
.from(model3dAsset)
.where(
@ -64,28 +66,47 @@ export default async function SharedWorldPage({
eq(model3dAsset.status, "success")
)
)
.limit(1);
.orderBy(desc(model3dAsset.sharedAt), desc(model3dAsset.updatedAt))
.limit(30);
if (!asset) {
if (assets.length === 0) {
notFound();
}
const assetPayload = readAssetPayload(asset.assetPayload);
const sharePath = `/worlds/share/${shareToken}`;
const worlds = assets.map((asset) => {
const assetPayload = readAssetPayload(asset.assetPayload);
return {
id: asset.taskId,
caption: assetPayload?.caption || null,
imageUrl: assetPayload?.panoUrl || assetPayload?.thumbnailUrl || asset.previewImageUrl || null,
meshUrl: asset.modelUrl || assetPayload?.meshUrl || null,
model: assetPayload?.model || asset.modelVersion || null,
previewImageUrl: asset.previewImageUrl || assetPayload?.thumbnailUrl || null,
prompt: asset.prompt || null,
semanticsMetadata: assetPayload?.semanticsMetadata || null,
splatUrl: pickBestWorldLabsSplatUrl(assetPayload?.spzUrls),
title: assetPayload?.displayName || asset.prompt || `World ${asset.taskId.slice(0, 8)}`,
worldUrl: assetPayload?.worldUrl || null,
};
});
const primaryWorld = worlds[0];
return (
<WorldSharePage
caption={assetPayload?.caption || null}
imageUrl={assetPayload?.panoUrl || assetPayload?.thumbnailUrl || asset.previewImageUrl || null}
meshUrl={asset.modelUrl || assetPayload?.meshUrl || null}
model={assetPayload?.model || asset.modelVersion || null}
previewImageUrl={asset.previewImageUrl || assetPayload?.thumbnailUrl || null}
prompt={asset.prompt || null}
semanticsMetadata={assetPayload?.semanticsMetadata || null}
caption={primaryWorld?.caption || null}
imageUrl={primaryWorld?.imageUrl || null}
meshUrl={primaryWorld?.meshUrl || null}
model={primaryWorld?.model || null}
previewImageUrl={primaryWorld?.previewImageUrl || null}
prompt={primaryWorld?.prompt || null}
semanticsMetadata={primaryWorld?.semanticsMetadata || null}
shareUrl={sharePath}
splatUrl={pickBestWorldLabsSplatUrl(assetPayload?.spzUrls)}
title={assetPayload?.displayName || asset.prompt || `World ${asset.taskId.slice(0, 8)}`}
worldUrl={assetPayload?.worldUrl || null}
splatUrl={primaryWorld?.splatUrl || null}
title={primaryWorld?.title || "3D 世界分享"}
worldUrl={primaryWorld?.worldUrl || null}
worlds={worlds}
/>
);
}

View File

@ -25,6 +25,10 @@ import {
} from "@/components/ui/select";
import { ScrollArea } from "@/components/ui/scroll-area";
import { cn, fetcher } from "@/lib/utils";
import {
isImageGenerationModelDescriptor,
isVideoModelDescriptor,
} from "@/lib/ai/model-classification";
import {
defaultAgentPresetModeConfig,
normalizeAgentPresetModeConfig,
@ -137,8 +141,10 @@ export function AgentPresetDialog({
models?: Array<{
id: string;
displayName: string;
modelId?: string;
supportsImageGen?: boolean;
providerType?: string;
capabilities?: unknown;
isMultimodal?: boolean;
}>;
}>("/api/models", fetcher);
@ -239,27 +245,46 @@ export function AgentPresetDialog({
[presetLevel3Options, selectedPresetLevel3Id]
);
const videoProviderTypes = useMemo(
() => new Set(["volcano-video", "doubao-video", "qianwen-video", "fal-video", "ruxa-video"]),
[]
);
const filteredModels = useMemo(() => {
const models = modelsData?.models || [];
if (type === "video") {
return models.filter((model) => videoProviderTypes.has(String(model.providerType || "")));
return models.filter((model) =>
isVideoModelDescriptor({
providerType: model.providerType,
modelId: model.modelId,
capabilities: model.capabilities,
supportsImageGen: model.supportsImageGen,
})
);
}
if (type === "text") {
return models.filter(
(model) =>
model.isMultimodal === true &&
!videoProviderTypes.has(String(model.providerType || ""))
!isVideoModelDescriptor({
providerType: model.providerType,
modelId: model.modelId,
capabilities: model.capabilities,
supportsImageGen: model.supportsImageGen,
})
);
}
return models.filter(
(model) =>
model.supportsImageGen && !videoProviderTypes.has(String(model.providerType || ""))
isImageGenerationModelDescriptor({
providerType: model.providerType,
modelId: model.modelId,
capabilities: model.capabilities,
supportsImageGen: model.supportsImageGen,
}) &&
!isVideoModelDescriptor({
providerType: model.providerType,
modelId: model.modelId,
capabilities: model.capabilities,
supportsImageGen: model.supportsImageGen,
})
);
}, [modelsData?.models, type, videoProviderTypes]);
}, [modelsData?.models, type]);
useEffect(() => {
if (!open) return;

View File

@ -182,6 +182,7 @@ interface CanvasWorkspaceProps {
workspaceLabel?: string;
emptyStateDescription?: string;
workspaceVariant?: "canvas" | "video";
layerSplitEnabled?: boolean;
showEmptyState?: boolean;
onBlankCanvasDoubleClick?: (payload: {
screenPosition: Position;
@ -194,6 +195,7 @@ export function CanvasWorkspace({
workspaceLabel = "画布",
emptyStateDescription = '点击顶部"添加"按钮开始创作',
workspaceVariant = "canvas",
layerSplitEnabled = false,
showEmptyState = true,
onBlankCanvasDoubleClick,
}: CanvasWorkspaceProps) {
@ -1488,7 +1490,11 @@ export function CanvasWorkspace({
) : null}
{isWorkflowCanvas ? <VideoCanvasNodeMarkers node={node} /> : null}
{node.type === 'image' && (
<ImageNode node={node} isSelected={isSelected} />
<ImageNode
isLayerSplitEnabled={layerSplitEnabled}
isSelected={isSelected}
node={node}
/>
)}
{node.type === 'text' && (
<TextNode node={node} isSelected={isSelected} />

View File

@ -847,6 +847,7 @@ export function RealtimeCanvasDialog({
const [refreshToken, setRefreshToken] = useState(0);
const [siteName, setSiteName] = useState("AI Chatbot");
const [siteLogo, setSiteLogo] = useState<string | null>(null);
const [isLayerSplitEnabled, setIsLayerSplitEnabled] = useState(false);
const statusBadge = useMemo(() => {
if (status === "live")
@ -908,6 +909,7 @@ export function RealtimeCanvasDialog({
) {
setSiteLogo(data.siteLogoDark.trim());
}
setIsLayerSplitEnabled(data.layerSplitEnabled === true);
})
.catch((error) => {
console.warn("[RealtimeCanvas] Failed to load site settings:", error);
@ -1731,6 +1733,9 @@ export function RealtimeCanvasDialog({
const importFilesAsSplitLayers = useCallback(
async (files: File[]) => {
if (!isLayerSplitEnabled) {
throw new Error("图层拆分功能未开启");
}
if (files.length === 0) return 0;
const invalidFile = files.find((file) => !file.type.startsWith("image/"));
if (invalidFile) {
@ -1778,7 +1783,7 @@ export function RealtimeCanvasDialog({
return appendLayerBatch(nextLayerBatch);
},
[appendLayerBatch, createPlacedLayer, fileToDataUrl]
[appendLayerBatch, createPlacedLayer, fileToDataUrl, isLayerSplitEnabled]
);
const triggerGenerate = useCallback(() => {
@ -3534,23 +3539,25 @@ export function RealtimeCanvasDialog({
)}
</div>
<div className="mt-2 flex flex-col gap-2">
<Button
className={cn(
"h-10 rounded-xl px-0 hover:bg-[#f5f5f5]",
controlSurfaceClass
)}
disabled={isLayerSplitImporting}
onClick={() => splitUploadInputRef.current?.click()}
title="图层拆分导入"
type="button"
variant="outline"
>
{isLayerSplitImporting ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : (
<Layers3 className="h-4 w-4" />
)}
</Button>
{isLayerSplitEnabled ? (
<Button
className={cn(
"h-10 rounded-xl px-0 hover:bg-[#f5f5f5]",
controlSurfaceClass
)}
disabled={isLayerSplitImporting}
onClick={() => splitUploadInputRef.current?.click()}
title="图层拆分导入"
type="button"
variant="outline"
>
{isLayerSplitImporting ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : (
<Layers3 className="h-4 w-4" />
)}
</Button>
) : null}
<Button
className={cn(
"h-10 rounded-xl px-0 hover:bg-[#f5f5f5]",

View File

@ -351,9 +351,14 @@ async function drawMarkerOnImage(
interface ImageNodeProps {
node: CanvasNode;
isSelected: boolean;
isLayerSplitEnabled?: boolean;
}
export function ImageNode({ node, isSelected }: ImageNodeProps) {
export function ImageNode({
node,
isSelected,
isLayerSplitEnabled = false,
}: ImageNodeProps) {
const data = node.data as ImageNodeData;
const contentScale = useMemo(
() =>
@ -1150,6 +1155,24 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
setPickerMode(false);
}, [setPickerMode]);
const handleStartPicking = useCallback(
(event?: React.MouseEvent) => {
event?.stopPropagation();
if (!data.src) {
toast.error("请先上传图片");
return;
}
if (isProcessing) {
toast.info("正在处理中,请稍候...");
return;
}
setPickerMode(true, node.id);
toast.info("点击图片中的物品进行拾取");
},
[data.src, isProcessing, node.id, setPickerMode]
);
const handleEnterBoxCutoutMode = useCallback(
(event?: React.MouseEvent | React.PointerEvent) => {
event?.stopPropagation();
@ -2248,6 +2271,10 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
return;
}
if (action === "layer-split") {
if (!isLayerSplitEnabled) {
toast.info("图层拆分功能未开启");
return;
}
setIsProcessing(true);
try {
const result = await imageGenerationService.splitImageLayers(data.src);
@ -3247,10 +3274,12 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
<StickyNote className="mr-2 h-4 w-4" />
备注
</DropdownMenuItem>
<DropdownMenuItem onClick={() => handleAction("layer-split")}>
<Layers className="mr-2 h-4 w-4" />
图层拆分
</DropdownMenuItem>
{isLayerSplitEnabled ? (
<DropdownMenuItem onClick={() => handleAction("layer-split")}>
<Layers className="mr-2 h-4 w-4" />
图层拆分
</DropdownMenuItem>
) : null}
<DropdownMenuItem onClick={() => handleAction("resize")}>
<Maximize2 className="mr-2 h-4 w-4" />
调尺
@ -3451,6 +3480,16 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
<div className="mx-1 h-6 w-px bg-slate-200 dark:bg-slate-700" />
<Button
className="h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
disabled={isProcessing || isPickerLoading || isThisNodePicking}
onClick={handleStartPicking}
variant="ghost"
>
<Crosshair className="mr-1.5 h-4 w-4" />
识物
</Button>
<Button
className="h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
disabled={isProcessing}
@ -3460,7 +3499,7 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
<span className="mr-1.5 rounded-md border border-slate-300 px-1.5 text-[11px] text-slate-600 leading-5 dark:border-slate-500 dark:text-slate-100">
HD
</span>
高清增强
高清
</Button>
<Button
@ -3473,6 +3512,16 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
抠图
</Button>
<Button
className="h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
disabled={isProcessing}
onClick={() => handleAction("box-cutout")}
variant="ghost"
>
<Scan className="mr-1.5 h-4 w-4" />
框选抠图
</Button>
<Button
className="h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
disabled={isProcessing}
@ -3490,7 +3539,7 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
variant="ghost"
>
<Maximize2 className="mr-1.5 h-4 w-4" />
尺寸调整
尺寸
</Button>
<Button
@ -3500,7 +3549,7 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
variant="ghost"
>
<Type className="mr-1.5 h-4 w-4" />
文字替换
文字
</Button>
<Button
@ -3510,18 +3559,20 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
variant="ghost"
>
<Box className="mr-1.5 h-4 w-4" />
视角生成
多视角
</Button>
<Button
className="h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
disabled={isProcessing}
onClick={() => handleAction("layer-split")}
variant="ghost"
>
<Layers className="mr-1.5 h-4 w-4" />
图层拆分
</Button>
{isLayerSplitEnabled ? (
<Button
className="h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
disabled={isProcessing}
onClick={() => handleAction("layer-split")}
variant="ghost"
>
<Layers className="mr-1.5 h-4 w-4" />
图层拆分
</Button>
) : null}
<div className="mx-1 h-6 w-px bg-slate-200 dark:bg-slate-700" />

View File

@ -8,7 +8,7 @@ import {
Sparkles,
Waypoints,
} from "lucide-react";
import { useMemo, useState } from "react";
import { useEffect, useMemo, useState } from "react";
import { World3DViewer } from "@/components/canvas/world3d/World3DViewer";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
@ -19,6 +19,7 @@ type SharedWorldPageProps = {
title: string;
prompt?: string | null;
caption?: string | null;
model?: string | null;
worldUrl?: string | null;
imageUrl?: string | null;
meshUrl?: string | null;
@ -26,6 +27,21 @@ type SharedWorldPageProps = {
previewImageUrl?: string | null;
semanticsMetadata?: Record<string, unknown> | null;
shareUrl: string;
worlds?: SharedWorldItem[];
};
type SharedWorldItem = {
id: string;
title: string;
prompt?: string | null;
caption?: string | null;
model?: string | null;
worldUrl?: string | null;
imageUrl?: string | null;
meshUrl?: string | null;
splatUrl?: string | null;
previewImageUrl?: string | null;
semanticsMetadata?: Record<string, unknown> | null;
};
const VIEW_OPTIONS: Array<{
@ -62,6 +78,7 @@ export function WorldSharePage({
title,
prompt,
caption,
model,
worldUrl,
imageUrl,
meshUrl,
@ -69,10 +86,55 @@ export function WorldSharePage({
previewImageUrl,
semanticsMetadata,
shareUrl,
worlds,
}: SharedWorldPageProps) {
const worldItems = useMemo<SharedWorldItem[]>(
() =>
worlds?.length
? worlds
: [
{
id: "shared-world",
title,
prompt,
caption,
model,
worldUrl,
imageUrl,
meshUrl,
splatUrl,
previewImageUrl,
semanticsMetadata,
},
],
[
caption,
imageUrl,
meshUrl,
model,
previewImageUrl,
prompt,
semanticsMetadata,
splatUrl,
title,
worldUrl,
worlds,
]
);
const [activeWorldId, setActiveWorldId] = useState(
() => worldItems[0]?.id || "shared-world"
);
const activeWorld =
worldItems.find((item) => item.id === activeWorldId) || worldItems[0];
const availableViews = useMemo(
() => resolveAvailableViews({ worldUrl, imageUrl, meshUrl, splatUrl }),
[imageUrl, meshUrl, splatUrl, worldUrl]
() =>
resolveAvailableViews({
worldUrl: activeWorld?.worldUrl,
imageUrl: activeWorld?.imageUrl,
meshUrl: activeWorld?.meshUrl,
splatUrl: activeWorld?.splatUrl,
}),
[activeWorld?.imageUrl, activeWorld?.meshUrl, activeWorld?.splatUrl, activeWorld?.worldUrl]
);
const [activeView, setActiveView] = useState<World3DViewMode>(
inferInitialView(availableViews)
@ -80,7 +142,24 @@ export function WorldSharePage({
const [firstPerson, setFirstPerson] = useState(false);
const [copied, setCopied] = useState(false);
const primaryDescription = caption || prompt || "这个 3D 世界已开启只读分享。";
useEffect(() => {
if (worldItems.some((item) => item.id === activeWorldId)) return;
setActiveWorldId(worldItems[0]?.id || "shared-world");
}, [activeWorldId, worldItems]);
useEffect(() => {
if (availableViews.includes(activeView)) return;
setActiveView(inferInitialView(availableViews));
}, [activeView, availableViews]);
useEffect(() => {
setFirstPerson(false);
}, [activeWorld?.id]);
const primaryDescription =
activeWorld?.caption || activeWorld?.prompt || "这个 3D 世界已开启只读分享。";
const activeTitle = activeWorld?.title || title;
const activeMeshUrl = activeWorld?.meshUrl || null;
const handleCopyShareUrl = async () => {
try {
@ -94,10 +173,10 @@ export function WorldSharePage({
};
const handleDownloadMesh = () => {
if (!meshUrl) return;
if (!activeMeshUrl) return;
const link = document.createElement("a");
link.href = meshUrl;
link.download = "shared-world.glb";
link.href = activeMeshUrl;
link.download = `shared-world-${activeWorld?.id?.slice(0, 8) || "asset"}.glb`;
link.click();
};
@ -108,7 +187,7 @@ export function WorldSharePage({
<div className="space-y-3">
<div className="flex flex-wrap items-center gap-2">
<Badge className="border-cyan-500/30 bg-cyan-500/10 text-cyan-200" variant="outline">
3D 世界分享
{worldItems.length > 1 ? `${worldItems.length} 个 3D 世界分享` : "3D 世界分享"}
</Badge>
{availableViews.map((view) => (
<Badge className="border-white/10 bg-white/5 text-zinc-300" key={view} variant="outline">
@ -118,7 +197,7 @@ export function WorldSharePage({
</div>
<div>
<h1 className="text-2xl font-semibold tracking-tight text-white md:text-3xl">
{title}
{activeTitle}
</h1>
<p className="mt-2 max-w-3xl text-sm leading-6 text-zinc-300">
{primaryDescription}
@ -135,7 +214,7 @@ export function WorldSharePage({
<Link2 className="mr-2 h-4 w-4" />
{copied ? "已复制链接" : "复制分享链接"}
</Button>
{meshUrl ? (
{activeMeshUrl ? (
<Button
className="border-white/10 bg-white/5 text-zinc-100 hover:bg-white/10"
onClick={handleDownloadMesh}
@ -176,7 +255,7 @@ export function WorldSharePage({
))}
</div>
{activeView === "splat" && splatUrl ? (
{activeView === "splat" && activeWorld?.splatUrl ? (
<Button
className={cn(
"rounded-full border border-white/10",
@ -196,21 +275,67 @@ export function WorldSharePage({
<div className="h-[62vh] min-h-[520px]">
<World3DViewer
key={`${activeWorld?.id || "shared-world"}-${activeView}`}
className="h-full w-full"
firstPerson={firstPerson}
imageUrl={imageUrl}
imageUrl={activeWorld?.imageUrl || null}
interactionMode="navigate"
meshUrl={meshUrl}
meshUrl={activeWorld?.meshUrl || null}
mode={activeView}
previewImageUrl={previewImageUrl}
semanticsMetadata={semanticsMetadata}
splatUrl={splatUrl}
worldUrl={worldUrl}
previewImageUrl={activeWorld?.previewImageUrl || null}
semanticsMetadata={activeWorld?.semanticsMetadata || null}
splatUrl={activeWorld?.splatUrl || null}
worldUrl={activeWorld?.worldUrl || null}
/>
</div>
</div>
<div className="space-y-4">
{worldItems.length > 1 ? (
<div className="rounded-3xl border border-white/10 bg-black/30 p-4 shadow-2xl backdrop-blur">
<h2 className="text-sm font-medium text-white">世界列表</h2>
<div className="mt-3 space-y-2">
{worldItems.map((item, index) => {
const thumbnail = item.previewImageUrl || item.imageUrl || null;
const isActive = item.id === activeWorld?.id;
return (
<button
className={cn(
"flex w-full items-center gap-3 rounded-2xl border border-white/10 bg-white/5 p-2 text-left transition hover:bg-white/10",
isActive && "border-cyan-300/60 bg-cyan-400/10"
)}
key={item.id}
onClick={() => setActiveWorldId(item.id)}
type="button"
>
<div className="flex h-14 w-16 shrink-0 items-center justify-center overflow-hidden rounded-xl bg-white/10">
{thumbnail ? (
<img
alt={item.title}
className="h-full w-full object-cover"
src={thumbnail}
/>
) : (
<Box className="h-5 w-5 text-zinc-500" />
)}
</div>
<div className="min-w-0 flex-1">
<div className="text-[11px] text-cyan-200">世界 {index + 1}</div>
<div className="truncate text-sm font-medium text-white">
{item.title}
</div>
<div className="mt-1 truncate text-xs text-zinc-400">
{item.caption || item.prompt || item.model || "暂无描述"}
</div>
</div>
</button>
);
})}
</div>
</div>
) : null}
<div className="rounded-3xl border border-white/10 bg-black/30 p-4 shadow-2xl backdrop-blur">
<h2 className="text-sm font-medium text-white">查看说明</h2>
<div className="mt-3 space-y-2 text-sm leading-6 text-zinc-300">
@ -224,15 +349,17 @@ export function WorldSharePage({
<div className="mt-3 grid grid-cols-1 gap-2 text-xs text-zinc-300">
<div className="rounded-2xl border border-white/10 bg-white/5 px-3 py-2">
<div className="text-zinc-500">World</div>
<div className="mt-1">{splatUrl ? "可查看" : "暂无"}</div>
<div className="mt-1">{activeWorld?.splatUrl ? "可查看" : "暂无"}</div>
</div>
<div className="rounded-2xl border border-white/10 bg-white/5 px-3 py-2">
<div className="text-zinc-500">图像</div>
<div className="mt-1">{imageUrl || previewImageUrl ? "可查看" : "暂无"}</div>
<div className="mt-1">
{activeWorld?.imageUrl || activeWorld?.previewImageUrl ? "可查看" : "暂无"}
</div>
</div>
<div className="rounded-2xl border border-white/10 bg-white/5 px-3 py-2">
<div className="text-zinc-500">Mesh</div>
<div className="mt-1">{meshUrl ? "可下载" : "暂无"}</div>
<div className="mt-1">{activeWorld?.meshUrl ? "可下载" : "暂无"}</div>
</div>
</div>
</div>

View File

@ -66,6 +66,11 @@ function normalizeImageParamsForProvider(
return normalized;
}
if (providerType === "krea") {
normalized.imageCount = Math.min(Math.max(currentCount, 1), 4);
return normalized;
}
if (providerType === "alibaba-image") {
normalized.imageCount = Math.min(Math.max(currentCount, 1), 6);
return normalized;
@ -1137,7 +1142,9 @@ export function Chat({
? "fal-video"
: model?.providerType === "ruxa-video"
? "ruxa-video"
: "volcano-video";
: model?.providerType === "krea"
? "krea-video"
: "volcano-video";
else if (
isImageGenerationModelDescriptor({
providerType: model?.providerType,
@ -1999,6 +2006,7 @@ export function Chat({
if (provider === "qianwen-video") return "qianwen-video";
if (provider === "fal-video") return "fal-video";
if (provider === "ruxa-video") return "ruxa-video";
if (provider === "krea") return "krea-video";
if (provider === "volcano-video" || provider === "doubao-video") return "volcano-video";
return "other";
};

View File

@ -12,6 +12,7 @@ import {
type ThemeColorMode,
Transforms,
type Viewport,
getHitElementsBySelection,
toHostPoint,
toViewBoxPoint,
} from "@plait/core";
@ -19,8 +20,9 @@ import { DrawTransforms, MemorizeKey } from "@plait/draw";
import {
canSetAbstract,
getSelectedMindElements,
type MindElement,
MindElement,
MindTransforms,
PlaitMind,
} from "@plait/mind";
import type { BoardChangeData } from "@plait-board/react-board";
import {
@ -441,6 +443,48 @@ function getExistingFlowBoardMindElements(
});
}
function uniqueFlowBoardMindElements(elements: MindElement[]) {
const seen = new Set<MindElement>();
return elements.filter((element) => {
if (seen.has(element)) {
return false;
}
seen.add(element);
return true;
});
}
function preferFlowBoardNodeElements(elements: MindElement[]) {
const nodeElements = elements.filter((element) => !PlaitMind.isMind(element));
return nodeElements.length > 0 ? nodeElements : elements;
}
function getFlowBoardMindElementsBySelection(
board: PlaitBoard,
selection: Selection | null | undefined
) {
if (!selection) {
return [] as MindElement[];
}
const hitElements = getHitElementsBySelection(board, selection, (element) =>
MindElement.isMindElement(board, element)
);
return getSelectedMindElements(board, hitElements);
}
function getFlowBoardSelectedMindElements(
board: PlaitBoard,
selection: Selection | null | undefined
) {
return preferFlowBoardNodeElements(
uniqueFlowBoardMindElements([
...getSelectedMindElements(board),
...getFlowBoardMindElementsBySelection(board, selection),
])
);
}
function getFlowBoardMarkerSelectionState(
board: PlaitBoard | null,
selectedElements?: MindElement[]
@ -599,6 +643,7 @@ export function DrawnixWorkspace({
const boardRef = useRef<PlaitBoard | null>(null);
const mindMapFileInputRef = useRef<HTMLInputElement | null>(null);
const selectedMindElementsRef = useRef<MindElement[]>([]);
const selectedMindSelectionRef = useRef<Selection | null>(null);
const [isMarkerPanelOpen, setIsMarkerPanelOpen] = useState(false);
const [isImportingMindMap, setIsImportingMindMap] = useState(false);
const [markerSelectionState, setMarkerSelectionState] =
@ -621,7 +666,10 @@ export function DrawnixWorkspace({
return [] as MindElement[];
}
const selectedElements = getSelectedMindElements(board);
const selectedElements = getFlowBoardSelectedMindElements(
board,
selectedMindSelectionRef.current || board.selection
);
if (selectedElements.length > 0) {
selectedMindElementsRef.current = selectedElements;
return selectedElements;
@ -636,7 +684,12 @@ export function DrawnixWorkspace({
}, []);
const refreshMarkerSelectionState = useCallback(
(options: { preserveFallback?: boolean } = {}) => {
(
options: {
preserveFallback?: boolean;
selection?: Selection | null;
} = {}
) => {
const board = boardRef.current;
if (!board) {
selectedMindElementsRef.current = [];
@ -644,7 +697,10 @@ export function DrawnixWorkspace({
return;
}
const selectedElements = getSelectedMindElements(board);
const selectedElements = getFlowBoardSelectedMindElements(
board,
options.selection ?? selectedMindSelectionRef.current ?? board.selection
);
if (selectedElements.length > 0) {
selectedMindElementsRef.current = selectedElements;
setMarkerSelectionState(
@ -674,7 +730,11 @@ export function DrawnixWorkspace({
const handleDrawnixInit = useCallback((board: PlaitBoard) => {
boardRef.current = board;
seedFlowBoardDefaultTextColor();
selectedMindElementsRef.current = getSelectedMindElements(board);
selectedMindSelectionRef.current = board.selection;
selectedMindElementsRef.current = getFlowBoardSelectedMindElements(
board,
board.selection
);
setMarkerSelectionState(
getFlowBoardMarkerSelectionState(board, selectedMindElementsRef.current)
);
@ -936,8 +996,9 @@ export function DrawnixWorkspace({
);
const handleDrawnixSelectionChange = useCallback(
(_selection: Selection | null) => {
refreshMarkerSelectionState();
(selection: Selection | null) => {
selectedMindSelectionRef.current = selection;
refreshMarkerSelectionState({ selection });
},
[refreshMarkerSelectionState]
);
@ -1471,6 +1532,12 @@ export function DrawnixWorkspace({
.drawnix .plait-text-container {
color: ${FLOW_BOARD_DEFAULT_TEXT_COLOR};
}
.drawnix:not(.drawnix--mobile) .app-toolbar {
top: auto;
bottom: 36px;
left: 36px;
}
`}
</style>
<input
@ -1502,10 +1569,10 @@ export function DrawnixWorkspace({
</FloatingSidebarShell>
<FloatingSidebarShell
className="absolute top-9 right-[180px] z-20"
className="absolute left-6 top-[92px] z-20"
flowBoardControl
>
<FloatingSidebarRail orientation="horizontal">
<FloatingSidebarRail className="max-h-[calc(100vh-172px)]">
<FloatingSidebarAction
disabled={isSavingNow}
icon={Save}
@ -1653,6 +1720,9 @@ export function DrawnixWorkspace({
onMouseDownCapture={(event) => {
event.stopPropagation();
}}
onPointerDownCapture={(event) => {
event.stopPropagation();
}}
>
<div className="flex items-center justify-between border-black/5 border-b px-5 py-4">
<div>
@ -1723,6 +1793,10 @@ export function DrawnixWorkspace({
event.preventDefault();
event.stopPropagation();
}}
onPointerDown={(event) => {
event.preventDefault();
event.stopPropagation();
}}
onClick={() =>
handleApplyFlowBoardMarker(group, marker)
}
@ -1747,6 +1821,10 @@ export function DrawnixWorkspace({
event.preventDefault();
event.stopPropagation();
}}
onPointerDown={(event) => {
event.preventDefault();
event.stopPropagation();
}}
onClick={handleClearFlowBoardMarkers}
type="button"
>

View File

@ -156,6 +156,10 @@ export function FloatingSidebarAction({
event.preventDefault();
event.stopPropagation();
}}
onPointerDown={(event) => {
event.preventDefault();
event.stopPropagation();
}}
onClick={onClick}
type="button"
>

View File

@ -24,6 +24,7 @@ const PROVIDER_MODEL_LABEL: Record<string, string> = {
midjourney: "Midjourney",
"volcano-vision": "即梦 4.5",
gemini: "Gemini 图片",
krea: "Krea 图片",
};
const PHASE_HINT: Record<NonNullable<ImageGenerationLoadingProps["phase"]>, string> = {

View File

@ -25,6 +25,8 @@ type DynamicModel = {
providerType: string;
modelId: string;
displayName: string;
supportsImageGen?: boolean;
capabilities?: unknown;
};
const fetcher = (url: string) => fetch(url).then((res) => res.json());
@ -114,7 +116,7 @@ function PureMessages({
const isQianwenVideoModel = modelsData?.models?.some(
(m: DynamicModel) =>
m.id === selectedModelId &&
(m.providerType === "qianwen-video" || m.providerType === "fal-video" || m.providerType === "ruxa-video")
(m.providerType === "qianwen-video" || m.providerType === "fal-video" || m.providerType === "ruxa-video" || m.providerType === "krea")
) ?? false;
const isVolcanoVisionModel = modelsData?.models?.some(
@ -615,6 +617,7 @@ function PureMessages({
modelType === "qianwen-video" ||
modelType === "fal-video" ||
modelType === "ruxa-video" ||
modelType === "krea-video" ||
(!modelType && (isQianwenVideoModel || isVideoModel));
const isAutoMode = selectedModelId === "auto";
@ -657,13 +660,15 @@ function PureMessages({
? "ruxa-video"
: modelType === "fal-video"
? "fal-video"
: modelType === "volcano-video"
? "volcano-video"
: modelType === "qianwen-video"
? "qianwen-video"
: currentModelProviderType === "doubao-video"
? "volcano-video"
: currentModelProviderType;
: modelType === "krea-video"
? "krea"
: modelType === "volcano-video"
? "volcano-video"
: modelType === "qianwen-video"
? "qianwen-video"
: currentModelProviderType === "doubao-video"
? "volcano-video"
: currentModelProviderType;
const videoPhase =
imageGenerationPhase === "idle" ? "processing" : imageGenerationPhase;

View File

@ -1554,7 +1554,7 @@ function PureModelSelectorCompact({
const modeFilteredModels = useMemo(() => {
if (modeFilter === "video") {
return dynamicModels.filter((model) => isVideoProviderType(model.providerType));
return dynamicModels.filter((model) => isVideoProviderType(model));
}
if (modeFilter === "image") {
return dynamicModels.filter((model) => isImageGenerationModel(model));

View File

@ -19,6 +19,7 @@ const PROVIDER_MODEL_LABEL: Record<string, string> = {
"ruxa-video": "RUXA 视频",
"volcano-video": "火山视频",
runway: "Runway 视频",
krea: "Krea 视频",
};
const PHASE_HINT: Record<VideoGenerationPhase, string> = {
@ -35,6 +36,7 @@ const PROVIDER_GRADIENT_CLASS: Record<string, string> = {
"ruxa-video": "from-violet-400 via-fuchsia-400 to-pink-400",
"volcano-video": "from-orange-400 via-amber-400 to-yellow-400",
runway: "from-rose-400 via-orange-400 to-amber-400",
krea: "from-blue-400 via-cyan-400 to-emerald-400",
};
/**

View File

@ -8,7 +8,7 @@ import { useEffect, useState, useCallback } from "react";
export interface PendingGeneration {
chatId: string;
modelId: string;
modelType: "gemini" | "volcano-vision" | "volcano-video" | "qianwen-video" | "fal-video" | "ruxa-video" | "alibaba-image" | "other";
modelType: "gemini" | "volcano-vision" | "volcano-video" | "qianwen-video" | "fal-video" | "ruxa-video" | "krea-video" | "alibaba-image" | "other";
timestamp: number;
userMessageId?: string;
}

View File

@ -26,6 +26,7 @@ import { createMidjourney } from "./providers/midjourney";
import { createFalVideo } from "./providers/fal-video";
import { createRuxaVideo } from "./providers/ruxa-video";
import { createRunway } from "./providers/runway";
import { createKrea } from "./providers/krea";
import { createVolcanoMultimodal } from "./providers/volcano-multimodal";
import { getProviderDefaultBaseUrl } from "./provider-defaults";
@ -247,6 +248,15 @@ function getOrCreateProvider(provider: Pick<AiProvider, "id" | "type" | "baseUrl
return runwayProvider as any;
}
case "krea": {
const kreaProvider = createKrea({
baseURL: resolvedBaseUrl,
apiKey: provider.apiKey,
});
console.log(`[Provider] Created Krea provider: ${resolvedBaseUrl}`);
return kreaProvider as any;
}
case "custom":
default: {
if (isVolcanoMultimodalBaseUrl(resolvedBaseUrl)) {
@ -381,6 +391,10 @@ export async function getDynamicModel(
console.log("[getDynamicModel] Getting Runway chat model..."); // 调试日志
baseModel = providerInstance.chat(resolvedModelId);
console.log("[getDynamicModel] Runway model retrieved:", baseModel ? "success" : "failed"); // 调试日志
} else if (provider.type === "krea") {
console.log("[getDynamicModel] Getting Krea chat model...");
baseModel = providerInstance.chat(resolvedModelId);
console.log("[getDynamicModel] Krea model retrieved:", baseModel ? "success" : "failed");
} else if (provider.type === "huanke-gemini") {
// 环科 Gemini 原生格式使用 chat() 方法
console.log("[getDynamicModel] Getting Huanke Gemini chat model..."); // 调试日志

294
lib/ai/krea-models.ts Normal file
View File

@ -0,0 +1,294 @@
export type KreaModelKind = "image" | "image-enhance" | "video";
export interface KreaModelPreset {
modelId: string;
displayName: string;
description: string;
kind: KreaModelKind;
supportsVision: boolean;
supportsImageGen: boolean;
requestFields: string[];
requiredFields: string[];
capabilities?: Record<string, any>;
defaultParams: Record<string, any>;
imageGenConfig?: Record<string, any>;
sortOrder: number;
}
const KREA_OPENAPI_URL = "https://api.krea.ai/openapi.json";
const REFERENCE_INPUT_FIELDS = new Set([
"imageUrl",
"imageUrls",
"image_url",
"styleImages",
"imageStyleRefs",
"characterReferenceImages",
"backgroundImage",
"referenceImages",
"startImage",
"endImage",
"init_image_url",
"references",
"subjects",
"styleReferences",
"elementReferences",
"videoReference",
"startVideo",
"init_video",
"referenceVideos",
"referenceAudios",
]);
function hasReferenceInput(fields: string[]) {
return fields.some((field) => REFERENCE_INPUT_FIELDS.has(field));
}
function createDefaultParams(
kind: KreaModelKind,
modelId: string,
requestFields: string[],
requiredFields: string[]
) {
return {
provider: "krea",
kind,
endpoint: modelId,
requestFields,
requiredFields,
officialApi: KREA_OPENAPI_URL,
};
}
function createImageConfig(fields: string[]) {
return {
maxImages: fields.includes("batchSize") ? 4 : 1,
supportsSize: fields.includes("width") && fields.includes("height"),
supports1K: true,
supports2K: true,
supports4K: fields.includes("resolution") || fields.includes("width"),
supportsPromptExtend: fields.includes("skipPromptExpansion"),
supportsNegativePrompt: fields.includes("negative_prompt"),
supportsWatermark: false,
supportsSeed: fields.includes("seed"),
creditCostPerImage: 0,
enableLoadingPlaceholder: true,
loadingColor: "blue",
};
}
function createVideoDurationOptions(modelId: string) {
if (modelId.includes("/minimax/hailuo-02") || modelId.includes("/minimax/hailuo-2.3")) {
return [6, 10];
}
if (modelId.includes("/google/veo")) {
return [4, 8];
}
if (modelId.includes("/openai/sora-2")) {
return [4, 8, 12, 20];
}
return [5, 10];
}
function createVideoCapabilities(modelId: string, fields: string[]) {
const supportedModes = new Set<string>();
if (fields.includes("prompt")) supportedModes.add("text-to-video");
if (
fields.includes("startImage") ||
fields.includes("image_url") ||
fields.includes("init_image_url")
) {
supportedModes.add("first-frame");
}
if (fields.includes("endImage")) supportedModes.add("first-last-frame");
if (
fields.includes("referenceImages") ||
fields.includes("styleReferences") ||
fields.includes("elementReferences") ||
fields.includes("subjects") ||
fields.includes("references")
) {
supportedModes.add("reference-images");
}
if (supportedModes.size === 0) supportedModes.add("text-to-video");
const durationOptions = fields.includes("duration") ? createVideoDurationOptions(modelId) : [5];
const resolutionOptions = fields.includes("resolution")
? (modelId.includes("/minimax/") ? ["720p", "1080p"] : ["480p", "720p", "1080p"])
: ["720p"];
return {
supportedModes: Array.from(supportedModes),
supportedResolutions: resolutionOptions,
supportedRatios: ["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"],
durationRange: {
min: Math.min(...durationOptions),
max: Math.max(...durationOptions),
},
durationOptions,
supportWatermark: false,
supportCameraFixed: fields.includes("cameraControl"),
supportSeed: fields.includes("seed"),
supportReturnLastFrame: fields.includes("endImage"),
maxReferenceImages: fields.includes("referenceImages")
? modelId.includes("/seedance-2")
? 9
: 4
: fields.includes("styleReferences") || fields.includes("elementReferences")
? 3
: undefined,
};
}
function imagePreset(
sortOrder: number,
modelId: string,
displayName: string,
fields: string[],
requiredFields: string[]
): KreaModelPreset {
return {
modelId,
displayName,
description: `Krea official image endpoint: ${modelId}`,
kind: "image",
supportsVision: hasReferenceInput(fields),
supportsImageGen: true,
requestFields: fields,
requiredFields,
imageGenConfig: createImageConfig(fields),
defaultParams: createDefaultParams("image", modelId, fields, requiredFields),
sortOrder,
};
}
function enhancePreset(
sortOrder: number,
modelId: string,
displayName: string,
fields: string[],
requiredFields: string[]
): KreaModelPreset {
return {
modelId,
displayName,
description: `Krea official image enhance endpoint: ${modelId}`,
kind: "image-enhance",
supportsVision: true,
supportsImageGen: true,
requestFields: fields,
requiredFields,
imageGenConfig: createImageConfig(fields),
defaultParams: createDefaultParams("image-enhance", modelId, fields, requiredFields),
sortOrder,
};
}
function videoPreset(
sortOrder: number,
modelId: string,
displayName: string,
fields: string[],
requiredFields: string[]
): KreaModelPreset {
const durationOptions = fields.includes("duration") ? createVideoDurationOptions(modelId) : [];
return {
modelId,
displayName,
description: `Krea official video endpoint: ${modelId}`,
kind: "video",
supportsVision: hasReferenceInput(fields),
supportsImageGen: false,
requestFields: fields,
requiredFields,
capabilities: createVideoCapabilities(modelId, fields),
defaultParams: createDefaultParams("video", modelId, fields, requiredFields),
imageGenConfig: {
creditCostPerVideo: 0,
videoDurationOptions: durationOptions,
creditCostPerVideoByDuration: {},
enableLoadingPlaceholder: true,
loadingColor: "blue",
},
sortOrder,
};
}
export const KREA_MODEL_PRESETS: KreaModelPreset[] = [
imagePreset(10, "/generate/image/bfl/flux-1-dev", "Flux", ["strength", "seed", "steps", "styleImages", "imageStyleRefs", "prompt", "width", "height", "guidance_scale_flux", "imageUrl", "relaxedModeAccess", "styles"], ["prompt"]),
imagePreset(20, "/generate/image/bfl/flux-1-kontext-dev", "Flux Kontext", ["strength", "seed", "steps", "styleImages", "imageStyleRefs", "prompt", "width", "height", "guidance_scale_flux", "imageUrl", "relaxedModeAccess", "styles"], ["prompt"]),
imagePreset(30, "/generate/image/google/nano-banana-pro", "Nano Banana Pro", ["batchSize", "prompt", "width", "height", "imageUrls", "aspectRatio", "styleImages", "resolution"], ["prompt"]),
imagePreset(40, "/generate/image/google/nano-banana-2", "Nano Banana 2", ["batchSize", "prompt", "width", "height", "imageUrls", "aspectRatio", "styleImages", "resolution"], ["prompt"]),
imagePreset(50, "/generate/image/google/nano-banana", "Nano Banana", ["batchSize", "prompt", "width", "height", "imageUrls", "styleImages", "aspectRatio"], ["prompt"]),
imagePreset(60, "/generate/image/bfl/flux-1.1-pro", "Flux 1.1 Pro", ["batchSize", "prompt", "width", "height", "seed"], ["prompt", "width", "height"]),
imagePreset(70, "/generate/image/bfl/flux-1.1-pro-ultra", "Flux 1.1 Pro Ultra", ["batchSize", "prompt", "width", "height", "seed", "raw"], ["prompt"]),
imagePreset(80, "/generate/image/ideogram/ideogram-2-turbo", "Ideogram 2.0A Turbo", ["batchSize", "prompt", "width", "height", "seed"], ["prompt"]),
imagePreset(90, "/generate/image/ideogram/ideogram-3", "Ideogram 3.0", ["batchSize", "prompt", "width", "height", "seed", "styleImages", "characterReferenceImages"], ["prompt"]),
imagePreset(100, "/generate/image/google/imagen-3", "Imagen 3", ["batchSize", "prompt", "width", "height", "seed"], ["prompt"]),
imagePreset(110, "/generate/image/google/imagen-4", "Imagen 4", ["batchSize", "prompt", "width", "height", "seed"], ["prompt"]),
imagePreset(120, "/generate/image/google/imagen-4-fast", "Imagen 4 Fast", ["batchSize", "prompt", "width", "height", "seed"], ["prompt"]),
imagePreset(130, "/generate/image/google/imagen-4-ultra", "Imagen 4 Ultra", ["batchSize", "prompt", "width", "height", "seed"], ["prompt"]),
imagePreset(140, "/generate/image/runway/gen-4-image", "Runway Gen-4", ["batchSize", "prompt", "width", "height", "seed", "referenceImages"], ["prompt", "referenceImages"]),
imagePreset(150, "/generate/image/openai/gpt-image", "ChatGPT Image", ["batchSize", "prompt", "width", "height", "quality", "backgroundImage", "imageUrls", "styleImages", "styles"], ["prompt"]),
imagePreset(160, "/generate/image/bytedance/seedream-3", "Seedream 3", ["batchSize", "prompt", "width", "height", "seed", "model"], ["prompt", "model"]),
imagePreset(170, "/generate/image/bytedance/seedream-4", "Seedream 4", ["batchSize", "prompt", "seed", "width", "height", "styleImages"], ["prompt", "width", "height"]),
imagePreset(180, "/generate/image/bytedance/seedream-5-lite", "Seedream 5 Lite", ["batchSize", "prompt", "seed", "width", "height", "styleImages"], ["prompt", "width", "height"]),
imagePreset(190, "/generate/image/qwen/2512", "Qwen 2512", ["prompt", "negative_prompt", "width", "height", "num_inference_steps", "cfg_scale", "seed", "batchSize", "skipPromptExpansion", "styles"], ["prompt"]),
imagePreset(200, "/generate/image/z-image/z-image", "Z Image", ["prompt", "width", "height", "batchSize", "seed", "imageUrl", "denoising_strength", "styleImages", "skipPromptExpansion", "styles"], ["prompt", "width", "height"]),
imagePreset(210, "/generate/image/bytedance/seededit", "SeedEdit", ["batchSize", "prompt", "seed", "imageUrl"], ["prompt", "imageUrl"]),
enhancePreset(220, "/generate/enhance/topaz/generative-enhance", "Topaz Generative", ["batchSize", "width", "height", "seed", "prompt", "output_format", "subject_detection", "face_enhancement", "face_enhancement_creativity", "face_enhancement_strength", "crop_to_fill", "image_url", "model", "upscaling_activated", "image_scaling_factor", "creativity", "texture", "sharpen", "denoise", "detail"], ["width", "height", "image_url"]),
enhancePreset(230, "/generate/enhance/topaz/standard-enhance", "Topaz", ["batchSize", "width", "height", "seed", "prompt", "output_format", "subject_detection", "face_enhancement", "face_enhancement_creativity", "face_enhancement_strength", "crop_to_fill", "image_url", "model", "upscaling_activated", "image_scaling_factor", "sharpen", "denoise", "fix_compression", "strength"], ["width", "height", "image_url", "model"]),
enhancePreset(240, "/generate/enhance/topaz/bloom-enhance", "Topaz Bloom", ["batchSize", "width", "height", "seed", "prompt", "output_format", "crop_to_fill", "image_url", "model", "creativity", "face_preservation", "color_preservation", "upscaling_activated", "image_scaling_factor"], ["width", "height", "image_url", "model"]),
videoPreset(250, "/generate/video/kling/kling-1", "Kling 1.0", ["startImage", "prompt", "aspectRatio", "duration", "cameraControl", "endImage", "mode"], ["prompt"]),
videoPreset(260, "/generate/video/kling/kling-1.5", "Kling 1.5", ["startImage", "prompt", "aspectRatio", "duration", "cameraControl", "endImage"], ["prompt"]),
videoPreset(270, "/generate/video/kling/kling-1.6", "Kling 1.6", ["startImage", "prompt", "aspectRatio", "duration", "cameraControl", "endImage", "mode"], ["prompt"]),
videoPreset(280, "/generate/video/kling/kling-2", "Kling 2.0", ["startImage", "prompt", "aspectRatio", "endImage", "mode"], ["prompt"]),
videoPreset(290, "/generate/video/kling/kling-2.1", "Kling 2.1", ["startImage", "prompt", "aspectRatio", "duration", "endImage", "mode"], ["prompt"]),
videoPreset(300, "/generate/video/kling/kling-2.5", "Kling 2.5", ["startImage", "prompt", "aspectRatio", "duration", "endImage"], ["prompt"]),
videoPreset(310, "/generate/video/kling/kling-2.6", "Kling 2.6", ["startImage", "prompt", "aspectRatio", "duration", "endImage", "generateAudio"], ["prompt"]),
videoPreset(320, "/generate/video/kling/kling-o1", "Kling o1", ["startImage", "prompt", "aspectRatio", "duration", "endImage", "styleReferences", "elementReferences", "videoReference"], ["prompt"]),
videoPreset(330, "/generate/video/minimax/hailuo", "Hailuo", ["startImage", "prompt", "expandPrompt", "cameraControl", "model", "subjects"], ["prompt"]),
videoPreset(340, "/generate/video/minimax/hailuo-02", "Hailuo 02", ["startImage", "prompt", "expandPrompt", "endImage", "resolution", "duration"], ["prompt"]),
videoPreset(350, "/generate/video/minimax/hailuo-2.3", "Hailuo 2.3", ["startImage", "prompt", "expandPrompt", "endImage", "resolution", "duration"], ["prompt"]),
videoPreset(360, "/generate/video/minimax/hailuo-2.3-fast", "Hailuo 2.3 Fast", ["startImage", "prompt", "expandPrompt", "endImage", "resolution", "duration"], ["prompt"]),
videoPreset(370, "/generate/video/alibaba/wan-2.1", "Wan 2.1", ["startImage", "prompt", "width", "height", "endImage", "aspectRatio", "frame_num", "styles"], ["prompt", "width", "height"]),
videoPreset(380, "/generate/video/alibaba/wan-2.2", "Wan 2.2", ["startImage", "guide_scale", "prompt", "width", "height", "aspectRatio", "frame_num", "styles"], ["prompt", "width", "height"]),
videoPreset(390, "/generate/video/alibaba/wan-2.5", "Wan 2.5", ["startImage", "prompt", "aspectRatio", "resolution", "enablePromptExpansion", "duration", "seed", "audio_url", "generateAudio"], ["prompt"]),
videoPreset(400, "/generate/video/google/veo-2", "Veo 2", ["startImage", "prompt", "aspectRatio", "duration", "image_url"], ["prompt"]),
videoPreset(410, "/generate/video/google/veo-3", "Veo 3", ["startImage", "prompt", "aspectRatio", "duration", "generateAudio", "resolution"], ["prompt"]),
videoPreset(420, "/generate/video/google/veo-3-fast", "Veo 3 Fast", ["startImage", "prompt", "aspectRatio", "duration", "generateAudio", "resolution"], ["prompt"]),
videoPreset(430, "/generate/video/google/veo-3.1", "Veo 3.1", ["startImage", "prompt", "aspectRatio", "duration", "generateAudio", "resolution", "endImage", "referenceImages"], ["prompt"]),
videoPreset(440, "/generate/video/google/veo-3.1-fast", "Veo 3.1 Fast", ["startImage", "prompt", "aspectRatio", "duration", "generateAudio", "resolution", "endImage", "referenceImages"], ["prompt"]),
videoPreset(450, "/generate/video/google/veo-3.1-lite", "Veo 3.1 Lite", ["startImage", "prompt", "aspectRatio", "duration", "resolution", "endImage"], ["prompt"]),
videoPreset(460, "/generate/video/bytedance/seedance-1.0-lite", "Seedance Lite", ["startImage", "prompt", "aspectRatio", "endImage", "referenceImages", "duration", "resolution"], ["prompt"]),
videoPreset(470, "/generate/video/bytedance/seedance-1.0-pro", "Seedance Pro", ["startImage", "prompt", "aspectRatio", "endImage", "referenceImages", "duration", "resolution"], ["prompt"]),
videoPreset(480, "/generate/video/bytedance/seedance-1.0-pro-fast", "Seedance Pro Fast", ["startImage", "prompt", "aspectRatio", "duration", "resolution"], ["prompt"]),
videoPreset(490, "/generate/video/bytedance/seedance-2", "Seedance 2.0", ["startImage", "prompt", "aspectRatio", "endImage", "referenceImages", "referenceVideos", "referenceAudios", "duration", "generateAudio", "resolution", "seed"], ["prompt"]),
videoPreset(500, "/generate/video/bytedance/seedance-2-fast", "Seedance 2.0 Fast", ["startImage", "prompt", "aspectRatio", "endImage", "referenceImages", "referenceVideos", "referenceAudios", "duration", "generateAudio", "resolution", "seed"], ["prompt"]),
videoPreset(510, "/generate/video/openai/sora-2", "Sora 2", ["startImage", "prompt", "duration", "aspectRatio"], ["prompt"]),
videoPreset(520, "/generate/video/luma/ray-2", "Ray 2", ["startImage", "prompt", "aspectRatio", "loop", "width", "height"], ["prompt", "width", "height"]),
videoPreset(530, "/generate/video/runway/gen-3", "Runway Gen-3", ["startImage", "prompt", "aspectRatio", "duration", "seed"], ["prompt"]),
videoPreset(540, "/generate/video/runway/gen-4-video", "Runway Gen-4", ["startImage", "prompt", "aspectRatio", "duration", "seed"], ["prompt"]),
videoPreset(550, "/generate/video/runway/gen-4.5", "Runway Gen-4.5", ["startImage", "prompt", "aspectRatio", "duration", "seed"], ["prompt"]),
videoPreset(560, "/generate/video/runway/aleph", "Runway Aleph", ["startImage", "startVideo", "startVideoTrim", "endVideoTrim", "prompt", "init_video", "init_image_url", "references", "ratio", "seed", "contentModeration"], ["prompt", "init_video"]),
];
export function getKreaModelPresetById(modelId: string) {
const normalized = String(modelId || "").trim();
return KREA_MODEL_PRESETS.find((preset) => preset.modelId === normalized);
}
export function inferKreaModelKind(modelId?: string | null): KreaModelKind | null {
const normalized = String(modelId || "").trim().toLowerCase();
if (!normalized) return null;
if (normalized.startsWith("/generate/video/") || normalized.includes("/generate/video/")) {
return "video";
}
if (normalized.startsWith("/generate/enhance/") || normalized.includes("/generate/enhance/")) {
return "image-enhance";
}
if (normalized.startsWith("/generate/image/") || normalized.includes("/generate/image/")) {
return "image";
}
return null;
}

View File

@ -1,3 +1,5 @@
import { inferKreaModelKind } from "@/lib/ai/krea-models";
const STATIC_VIDEO_PROVIDER_TYPES = new Set([
"doubao-video",
"volcano-video",
@ -88,6 +90,10 @@ export function isVideoModelDescriptor(params: {
return inferRunwayModelKind(params.modelId) === "video";
}
if (providerType === "krea") {
return inferKreaModelKind(params.modelId) === "video";
}
return false;
}
@ -105,6 +111,12 @@ export function isImageGenerationModelDescriptor(params: {
if (runwayModelKind === "video") return false;
}
if (providerType === "krea") {
const kreaModelKind = inferKreaModelKind(params.modelId);
if (kreaModelKind === "image" || kreaModelKind === "image-enhance") return true;
if (kreaModelKind === "video") return false;
}
if (isVideoModelDescriptor(params)) {
return false;
}

View File

@ -155,7 +155,7 @@ const INTENT_MODEL_MAP: Record<
> = {
// 图片编辑:优先 alibaba-image(灵魂4.5,支持提示词扩展)
image_edit: {
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "huanke-gemini"],
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini", "huanke-gemini"],
requireVision: true,
requireImageGen: true,
description: "图片编辑",
@ -170,14 +170,14 @@ const INTENT_MODEL_MAP: Record<
// 图片生成:优先灵魂4.5
image_gen: {
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "huanke-gemini", "volcano-vision"],
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini", "huanke-gemini", "volcano-vision"],
requireImageGen: true,
description: "图片生成",
},
// 视频生成
video_gen: {
providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video", "runway"],
providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video", "runway", "krea"],
description: "视频生成",
},
@ -198,25 +198,25 @@ const SUB_INTENT_OVERRIDES: Record<
> = {
// 抠图:alibaba-image 效果最好
remove_background: {
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini"],
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini"],
reason: "抠图首选灵魂4.5",
},
// 换背景:alibaba-image 支持场景生成
replace_background: {
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini"],
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini"],
reason: "换背景首选灵魂4.5",
},
// 图片增强
enhance: {
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "volcano-vision"],
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini", "volcano-vision"],
reason: "增强首选灵魂4.5",
},
// 风格迁移:gemini 可能更有创意
style_transfer: {
providerTypes: ["gemini", "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway"],
providerTypes: ["gemini", "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea"],
reason: "风格迁移优选Gemini",
},
};

View File

@ -21,6 +21,7 @@ const PROVIDER_DEFAULT_BASE_URL: Partial<Record<AiProvider["type"], string>> = {
"worldlabs-3d": "https://api.worldlabs.ai",
"kittycad-cad": "https://api.zoo.dev",
runway: "https://api.dev.runwayml.com",
krea: "https://api.krea.ai",
};
export function getProviderDefaultBaseUrl(type: AiProvider["type"]): string | undefined {

832
lib/ai/providers/krea.ts Normal file
View File

@ -0,0 +1,832 @@
import "server-only";
import sharp from "sharp";
import {
getKreaModelPresetById,
inferKreaModelKind,
type KreaModelKind,
} from "@/lib/ai/krea-models";
import { uploadFile } from "@/lib/storage/service";
interface KreaConfig {
apiKey: string;
baseURL: string;
}
interface KreaImageParams {
aspectRatio?: string;
imageSize?: string;
imageCount?: number;
seed?: number;
negativePrompt?: string;
promptExtend?: boolean;
}
interface KreaVideoParams {
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
ratio?: string;
duration?: number;
resolution?: string;
seed?: number;
generateAudio?: boolean;
}
interface UploadedImage {
thumbnail: string;
original: string;
}
const DEFAULT_BASE_URL = "https://api.krea.ai";
const IMAGE_POLL_INTERVAL_MS = 2000;
const VIDEO_POLL_INTERVAL_MS = 5000;
const IMAGE_TIMEOUT_MS = 5 * 60 * 1000;
const VIDEO_TIMEOUT_MS = 20 * 60 * 1000;
function normalizeText(value?: string | null) {
return String(value || "").replace(/\u00a0/g, " ").trim().normalize("NFKC");
}
function normalizeBaseURL(baseURL: string) {
const normalized = normalizeText(baseURL).replace(/\/+$/, "");
return normalized || DEFAULT_BASE_URL;
}
function normalizeKreaApiKey(apiKey: string) {
const stripped = normalizeText(apiKey)
.replace(/^bearer\s+/i, "")
.replace(/^key\s+/i, "")
.replace(/^(?:krea[_-]?api[_-]?key|api[_-]?key|token)\s*[::]\s*/i, "");
return stripped.trim().replace(/[^\x20-\x7E]/g, "");
}
function buildHeaders(apiKey: string) {
return {
Authorization: `Bearer ${normalizeKreaApiKey(apiKey)}`,
"Content-Type": "application/json",
Accept: "application/json",
};
}
function joinEndpoint(baseURL: string, path: string) {
const cleanPath = path.startsWith("/") ? path : `/${path}`;
return `${normalizeBaseURL(baseURL)}${cleanPath}`;
}
function resolveKreaEndpoint(modelId: string) {
const normalized = normalizeText(modelId);
if (!normalized) {
throw new Error("Krea 模型 ID 不能为空");
}
if (normalized.startsWith("/generate/")) return normalized;
if (normalized.startsWith("generate/")) return `/${normalized}`;
if (/^(image|video|enhance)\//i.test(normalized)) return `/generate/${normalized}`;
throw new Error("Krea 模型 ID 需要使用官方 /generate/... endpoint");
}
function detectExtensionFromContentType(contentType?: string | null) {
const normalized = String(contentType || "").toLowerCase();
if (normalized.includes("jpeg") || normalized.includes("jpg")) return "jpg";
if (normalized.includes("webp")) return "webp";
if (normalized.includes("gif")) return "gif";
if (normalized.includes("png")) return "png";
if (normalized.includes("mp4")) return "mp4";
if (normalized.includes("quicktime")) return "mov";
if (normalized.includes("webm")) return "webm";
return "bin";
}
function generateFilename(prefix: string, extension: string) {
const timestamp = Date.now();
const random = Math.random().toString(36).slice(2, 8);
return `${prefix}-${timestamp}-${random}.${extension}`;
}
function extractDataUriParts(dataUri: string) {
const match = dataUri.match(/^data:([^;]+);base64,(.+)$/);
if (!match) {
throw new Error("不支持的 Data URI 格式");
}
return {
mimeType: match[1],
buffer: Buffer.from(match[2], "base64"),
};
}
async function uploadDataUriAsset(dataUri: string, prefix: "krea-input" | "krea-video") {
const { mimeType, buffer } = extractDataUriParts(dataUri);
const extension = detectExtensionFromContentType(mimeType);
const arrayBuffer = buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength
) as ArrayBuffer;
const uploaded = await uploadFile(generateFilename(prefix, extension), arrayBuffer);
return uploaded.url;
}
async function normalizeAssetUrl(url?: string | null) {
const normalized = normalizeText(url);
if (!normalized) return "";
if (/^https?:\/\//i.test(normalized)) return normalized;
if (/^data:(image|video)\//i.test(normalized)) {
return uploadDataUriAsset(
normalized,
/^data:video\//i.test(normalized) ? "krea-video" : "krea-input"
);
}
return normalized;
}
function isVideoUrl(url: string, mediaType?: string | null) {
const normalizedMediaType = String(mediaType || "").toLowerCase();
if (normalizedMediaType.startsWith("video/")) return true;
return /\.(mp4|mov|webm|m4v)(\?|#|$)/i.test(url) || /^data:video\//i.test(url);
}
function extractPromptAndAssets(messages: Array<{ role: string; content: any }>) {
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
if (!lastUserMessage) {
throw new Error("未找到用户输入内容");
}
let prompt = "";
const imageUrls: string[] = [];
const videoUrls: string[] = [];
const pushAssetUrl = (value?: string, mediaType?: string | null) => {
if (!value) return;
const normalized = String(value || "").trim();
if (!normalized) return;
const target = isVideoUrl(normalized, mediaType) ? videoUrls : imageUrls;
if (!target.includes(normalized)) {
target.push(normalized);
}
};
const readContentPart = (part: any) => {
if (!part) return;
if (part.type === "text" && typeof part.text === "string") {
prompt = `${prompt} ${part.text}`.trim();
return;
}
if (part.type === "image_url" && part.image_url?.url) {
pushAssetUrl(part.image_url.url, "image/jpeg");
return;
}
if (part.type === "file") {
pushAssetUrl(part.url || part.data, part.mediaType || part.mimeType);
return;
}
if (part.type === "image") {
pushAssetUrl(part.image || part.url || part.data, part.mediaType || "image/jpeg");
return;
}
if (part.type === "video") {
pushAssetUrl(part.video || part.url || part.data, part.mediaType || "video/mp4");
}
};
if (typeof lastUserMessage.content === "string") {
prompt = normalizeText(lastUserMessage.content);
} else if (Array.isArray(lastUserMessage.content)) {
for (const part of lastUserMessage.content) {
readContentPart(part);
}
}
return {
prompt: normalizeText(prompt) || "生成内容",
imageUrls,
videoUrls,
};
}
function normalizeAspectRatio(value?: string, fallback = "16:9") {
const normalized = normalizeText(value || fallback);
if (
[
"21:9",
"16:9",
"9:16",
"4:3",
"3:4",
"1:1",
"9:21",
"auto",
].includes(normalized)
) {
return normalized;
}
return fallback;
}
function clamp(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, value));
}
function roundToMultiple(value: number, multiple = 16) {
return Math.max(multiple, Math.round(value / multiple) * multiple);
}
function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" {
const normalized = normalizeText(imageSize).toUpperCase();
if (normalized === "1K" || normalized === "2K" || normalized === "4K") {
return normalized;
}
return "2K";
}
function resolveImageDimensions(aspectRatio?: string, imageSize?: string) {
const size = normalizeImageSizeLabel(imageSize);
const base = size === "4K" ? 2048 : size === "2K" ? 1536 : 1024;
const ratio = normalizeAspectRatio(aspectRatio, "1:1");
if (ratio === "auto" || !ratio.includes(":")) {
return { width: base, height: base, imageSize: size };
}
const [rawW, rawH] = ratio.split(":").map((item) => Number(item));
const ratioW = Number.isFinite(rawW) && rawW > 0 ? rawW : 1;
const ratioH = Number.isFinite(rawH) && rawH > 0 ? rawH : 1;
const landscape = ratioW >= ratioH;
const width = landscape ? base : Math.round((base * ratioW) / ratioH);
const height = landscape ? Math.round((base * ratioH) / ratioW) : base;
return {
width: clamp(roundToMultiple(width), 256, 2048),
height: clamp(roundToMultiple(height), 256, 2048),
imageSize: size,
};
}
function resolveVideoDimensions(ratio?: string, resolution?: string) {
const normalizedResolution = normalizeText(resolution).toLowerCase();
const baseHeight = normalizedResolution === "1080p" ? 1080 : 720;
const aspectRatio = normalizeAspectRatio(ratio, "16:9");
if (aspectRatio === "9:16") {
return { width: baseHeight, height: Math.round((baseHeight * 16) / 9) };
}
if (aspectRatio === "1:1") {
return { width: baseHeight, height: baseHeight };
}
if (aspectRatio === "4:3") {
return { width: Math.round((baseHeight * 4) / 3), height: baseHeight };
}
if (aspectRatio === "3:4") {
return { width: baseHeight, height: Math.round((baseHeight * 4) / 3) };
}
if (aspectRatio === "21:9") {
return { width: Math.round((baseHeight * 21) / 9), height: baseHeight };
}
return { width: Math.round((baseHeight * 16) / 9), height: baseHeight };
}
function normalizeSeed(seed?: number) {
if (!Number.isFinite(Number(seed))) return undefined;
return clamp(Math.round(Number(seed)), 0, 4294967295);
}
function normalizeBatchSize(value?: number) {
if (!Number.isFinite(Number(value))) return 1;
return clamp(Math.round(Number(value)), 1, 4);
}
function normalizeVideoDuration(duration: unknown, endpoint: string) {
const value = Number.isFinite(Number(duration)) ? Math.round(Number(duration)) : 5;
if (endpoint.includes("/minimax/hailuo-02") || endpoint.includes("/minimax/hailuo-2.3")) {
return clamp(value || 6, 6, 10);
}
if (endpoint.includes("/google/veo")) {
return clamp(value || 8, 4, 8);
}
if (endpoint.includes("/openai/sora-2")) {
return clamp(value || 4, 4, 20);
}
if (endpoint.includes("/kling/")) {
return value >= 8 ? 10 : 5;
}
if (endpoint.includes("/alibaba/wan-2.5")) {
return clamp(value || 5, 5, 10);
}
return value >= 8 ? 10 : 5;
}
function normalizeVideoResolution(resolution: unknown, endpoint: string) {
const normalized = normalizeText(String(resolution || "")).toLowerCase();
if (endpoint.includes("/minimax/")) {
if (normalized === "1080p") return "1080p";
if (normalized === "512p" || normalized === "480p") return "512p";
return "768p";
}
if (normalized === "1080p" || normalized === "480p") return normalized;
return "720p";
}
function normalizeVideoAspectRatio(ratio: unknown, endpoint: string) {
const normalized = normalizeAspectRatio(String(ratio || ""), "16:9");
if (endpoint.includes("/openai/sora-2")) {
return normalized === "9:16" ? "9:16" : "16:9";
}
if (endpoint.includes("/google/veo") || endpoint.includes("/kling/")) {
return ["16:9", "9:16", "1:1"].includes(normalized) ? normalized : "16:9";
}
return normalized === "auto" ? "16:9" : normalized;
}
function buildStyleImages(imageUrls: string[]) {
return imageUrls.map((url) => ({ url, strength: 1 }));
}
function buildTaggedReferences(imageUrls: string[], tagPrefix = "ref", max = 3) {
return imageUrls.slice(0, max).map((url, index) => ({
url,
tag: `${tagPrefix}_${index + 1}`.replace(/[^a-z0-9_]/gi, "_").toLowerCase(),
}));
}
function collectHttpUrls(payload: unknown): string[] {
const urls = new Set<string>();
const visit = (value: unknown) => {
if (typeof value === "string") {
const normalized = value.trim();
if (/^https?:\/\//i.test(normalized)) {
urls.add(normalized);
}
return;
}
if (Array.isArray(value)) {
value.forEach(visit);
return;
}
if (value && typeof value === "object") {
Object.values(value as Record<string, unknown>).forEach(visit);
}
};
visit(payload);
return Array.from(urls);
}
async function downloadRemoteAsset(url: string) {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 120000);
try {
const response = await fetch(url, {
cache: "no-store",
signal: controller.signal,
});
if (!response.ok) {
throw new Error(`下载 Krea 输出失败: ${response.status}`);
}
return {
buffer: Buffer.from(await response.arrayBuffer()),
contentType: response.headers.get("content-type"),
};
} finally {
clearTimeout(timeoutId);
}
}
async function uploadGeneratedBuffer(buffer: Buffer, extension: string): Promise<UploadedImage> {
const thumbnailBuffer = await sharp(buffer)
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
.jpeg({ quality: 80 })
.toBuffer();
const originalArrayBuffer = buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength
) as ArrayBuffer;
const originalUploaded = await uploadFile(
generateFilename("krea-image-orig", extension === "bin" ? "png" : extension),
originalArrayBuffer
);
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
thumbnailBuffer.byteOffset,
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
) as ArrayBuffer;
const thumbnailUploaded = await uploadFile(
generateFilename("krea-image-thumb", "jpg"),
thumbnailArrayBuffer
);
return {
thumbnail: thumbnailUploaded.url,
original: originalUploaded.url,
};
}
async function uploadKreaImages(payload: unknown) {
const uploadedImages: UploadedImage[] = [];
for (const url of collectHttpUrls(payload)) {
try {
const { buffer, contentType } = await downloadRemoteAsset(url);
if (!String(contentType || "").startsWith("image/")) continue;
uploadedImages.push(await uploadGeneratedBuffer(buffer, detectExtensionFromContentType(contentType)));
} catch (error) {
console.warn("[Krea] Failed to archive image output:", error);
}
}
if (uploadedImages.length === 0) {
throw new Error("Krea 未返回可用图片");
}
return uploadedImages;
}
async function uploadVideoToStorage(remoteVideoUrl: string) {
try {
const { buffer, contentType } = await downloadRemoteAsset(remoteVideoUrl);
const extension = detectExtensionFromContentType(contentType || "video/mp4");
const arrayBuffer = buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength
) as ArrayBuffer;
const uploaded = await uploadFile(
generateFilename("krea-video", extension === "bin" ? "mp4" : extension),
arrayBuffer
);
return uploaded.url;
} catch (error) {
console.warn("[Krea] Failed to archive video output, using temporary URL:", error);
return remoteVideoUrl;
}
}
async function resolveKreaVideoUrl(payload: unknown) {
const outputUrls = collectHttpUrls(payload);
if (outputUrls.length === 0) {
throw new Error("Krea 未返回可用视频地址");
}
const videoUrl =
outputUrls.find((url) => /\.(mp4|mov|webm)(\?|#|$)/i.test(url) || /video/i.test(url)) ||
outputUrls[0]!;
return uploadVideoToStorage(videoUrl);
}
function formatImageMarkdown(images: UploadedImage[], imageSize: string) {
return images
.map((image) => `![thumb|||${image.original}|||${imageSize}](${image.thumbnail})`)
.join("\n\n");
}
function formatVideoMarkdown(videoUrl: string) {
return `<video src="${videoUrl}" controls style="max-width: 100%; border-radius: 8px; margin-bottom: 12px;"></video>\n\n[下载视频](${videoUrl})`;
}
async function requestKreaJob(
baseURL: string,
apiKey: string,
endpoint: string,
body: Record<string, any>
) {
const response = await fetch(joinEndpoint(baseURL, endpoint), {
method: "POST",
headers: buildHeaders(apiKey),
body: JSON.stringify(body),
});
const rawText = await response.text();
let payload: any = null;
try {
payload = rawText ? JSON.parse(rawText) : null;
} catch {
payload = rawText;
}
if (!response.ok) {
console.error("[Krea] Job request failed:", {
endpoint,
status: response.status,
requestFields: Object.keys(body),
response: payload,
});
const message =
typeof payload?.error === "string"
? payload.error
: typeof payload?.message === "string"
? payload.message
: rawText || "Krea 请求失败";
throw new Error(`${message} (HTTP ${response.status})`);
}
const jobId = normalizeText(payload?.job_id || payload?.id);
if (!jobId && Array.isArray(payload?.result?.urls) && payload.result.urls.length > 0) {
return payload;
}
if (!jobId) {
throw new Error("Krea 未返回任务 ID");
}
return { ...payload, job_id: jobId };
}
async function waitForKreaJob(
baseURL: string,
apiKey: string,
jobId: string,
kind: KreaModelKind
) {
const isVideo = kind === "video";
const deadline = Date.now() + (isVideo ? VIDEO_TIMEOUT_MS : IMAGE_TIMEOUT_MS);
const pollInterval = isVideo ? VIDEO_POLL_INTERVAL_MS : IMAGE_POLL_INTERVAL_MS;
while (Date.now() < deadline) {
const response = await fetch(joinEndpoint(baseURL, `/jobs/${jobId}`), {
method: "GET",
headers: buildHeaders(apiKey),
cache: "no-store",
});
const rawText = await response.text();
let payload: any = null;
try {
payload = rawText ? JSON.parse(rawText) : null;
} catch {
payload = rawText;
}
if (!response.ok) {
const message =
typeof payload?.error === "string"
? payload.error
: typeof payload?.message === "string"
? payload.message
: rawText || "Krea 轮询失败";
throw new Error(`${message} (HTTP ${response.status})`);
}
const status = normalizeText(payload?.status).toLowerCase();
if (status === "completed") {
return payload;
}
if (status === "failed" || status === "cancelled") {
throw new Error(String(payload?.error || payload?.message || "Krea 任务失败"));
}
await new Promise((resolve) => setTimeout(resolve, pollInterval));
}
throw new Error("Krea 任务超时,请稍后重试");
}
function getRequiredTopazModel(endpoint: string) {
if (endpoint.includes("/standard-enhance")) return "Standard V2";
if (endpoint.includes("/bloom-enhance")) return "Reimagine";
if (endpoint.includes("/generative-enhance")) return "Redefine";
return undefined;
}
function getRequiredModelValue(endpoint: string) {
if (endpoint.includes("/seedream-3")) return "seedream-3-0-t2i-250415";
if (endpoint.includes("/minimax/hailuo")) return "video-01";
return getRequiredTopazModel(endpoint);
}
function buildKreaBody(params: {
endpoint: string;
kind: KreaModelKind;
requestFields: string[];
requiredFields: string[];
prompt: string;
imageUrls: string[];
videoUrls: string[];
imageParams?: KreaImageParams;
videoParams?: KreaVideoParams;
}) {
const {
endpoint,
kind,
requestFields,
requiredFields,
prompt,
imageUrls,
videoUrls,
imageParams,
videoParams,
} = params;
const fields = new Set(requestFields);
const required = new Set(requiredFields);
const body: Record<string, any> = {};
const isVideo = kind === "video";
const ratio = isVideo
? normalizeVideoAspectRatio(videoParams?.ratio, endpoint)
: normalizeAspectRatio(imageParams?.aspectRatio, "1:1");
const videoResolution = normalizeVideoResolution(videoParams?.resolution, endpoint);
const imageDimensions = resolveImageDimensions(imageParams?.aspectRatio, imageParams?.imageSize);
const videoDimensions = resolveVideoDimensions(ratio, videoResolution);
if (fields.has("prompt")) body.prompt = prompt;
if (fields.has("batchSize")) body.batchSize = normalizeBatchSize(imageParams?.imageCount);
if (fields.has("width")) body.width = isVideo ? videoDimensions.width : imageDimensions.width;
if (fields.has("height")) body.height = isVideo ? videoDimensions.height : imageDimensions.height;
if (fields.has("aspectRatio")) body.aspectRatio = ratio === "auto" ? "1:1" : ratio;
if (fields.has("ratio")) body.ratio = ratio === "auto" ? "16:9" : ratio;
if (fields.has("resolution")) {
body.resolution = isVideo ? videoResolution : normalizeImageSizeLabel(imageParams?.imageSize);
}
const seed = normalizeSeed(isVideo ? videoParams?.seed : imageParams?.seed);
if (fields.has("seed") && seed !== undefined) body.seed = seed;
if (fields.has("negative_prompt") && imageParams?.negativePrompt) {
body.negative_prompt = imageParams.negativePrompt;
}
if (fields.has("skipPromptExpansion") && typeof imageParams?.promptExtend === "boolean") {
body.skipPromptExpansion = !imageParams.promptExtend;
}
if (fields.has("enablePromptExpansion")) {
body.enablePromptExpansion = true;
}
if (fields.has("expandPrompt")) {
body.expandPrompt = true;
}
if (fields.has("duration")) {
body.duration = normalizeVideoDuration(videoParams?.duration, endpoint);
}
if (fields.has("generateAudio") && typeof videoParams?.generateAudio === "boolean") {
body.generateAudio = videoParams.generateAudio;
}
if (fields.has("loop")) {
body.loop = false;
}
if (fields.has("mode")) {
body.mode = endpoint.includes("/kling/kling-2") ? "master" : "std";
}
if (fields.has("model")) {
const modelValue = getRequiredModelValue(endpoint);
if (modelValue) body.model = modelValue;
}
if (fields.has("output_format")) body.output_format = "jpg";
if (fields.has("backgroundImage")) body.backgroundImage = "auto";
if (fields.has("imageUrls") && imageUrls.length > 0) {
body.imageUrls = imageUrls.slice(0, 8);
}
if (fields.has("imageUrl") && imageUrls[0]) {
body.imageUrl = imageUrls[0];
}
if (fields.has("image_url") && imageUrls[0]) {
body.image_url = imageUrls[0];
}
if (fields.has("styleImages") && imageUrls.length > 0 && !body.imageUrls && !body.imageUrl) {
body.styleImages = buildStyleImages(imageUrls.slice(0, 4));
}
if (fields.has("imageStyleRefs") && imageUrls.length > 1) {
body.imageStyleRefs = buildStyleImages(imageUrls.slice(1, 5));
}
if (fields.has("characterReferenceImages") && imageUrls.length > 0) {
body.characterReferenceImages = imageUrls.slice(0, 4);
}
if (fields.has("referenceImages") && imageUrls.length > 0) {
body.referenceImages = endpoint.includes("/image/runway/gen-4-image")
? buildTaggedReferences(imageUrls, "ref", 3)
: imageUrls.slice(0, endpoint.includes("/seedance-2") ? 9 : 4);
}
if (fields.has("startImage") && imageUrls[0]) body.startImage = imageUrls[0];
if (fields.has("endImage") && imageUrls[1]) body.endImage = imageUrls[1];
if (fields.has("init_image_url") && imageUrls[0]) body.init_image_url = imageUrls[0];
if (fields.has("references") && imageUrls[0]) {
body.references = [{ type: "image", uri: imageUrls[0] }];
}
if (fields.has("subjects") && imageUrls[0] && videoParams?.mode === "reference-images") {
body.subjects = [{ type: "character", image: [imageUrls[0]] }];
}
if (fields.has("styleReferences") && imageUrls.length > 0) {
body.styleReferences = buildTaggedReferences(imageUrls, "style", 3);
}
if (fields.has("elementReferences") && imageUrls.length > 0) {
body.elementReferences = [
{
tag: "element_1",
images: imageUrls.slice(0, 3),
},
];
}
if (fields.has("videoReference") && videoUrls[0]) {
body.videoReference = { url: videoUrls[0], tag: "video_1" };
}
if (fields.has("startVideo") && videoUrls[0]) body.startVideo = videoUrls[0];
if (fields.has("init_video") && videoUrls[0]) body.init_video = videoUrls[0];
if (fields.has("referenceVideos") && videoUrls.length > 0) {
body.referenceVideos = videoUrls.slice(0, 3);
}
if (required.has("referenceImages") && !body.referenceImages) {
throw new Error("该 Krea 模型需要至少 1 张参考图");
}
if ((required.has("imageUrl") || required.has("image_url")) && !body.imageUrl && !body.image_url) {
throw new Error("该 Krea 模型需要上传图片");
}
if (required.has("init_video") && !body.init_video) {
throw new Error("该 Krea 模型需要上传视频");
}
return { body, imageSize: imageDimensions.imageSize };
}
export function createKrea({ apiKey, baseURL }: KreaConfig) {
return {
provider: "krea",
chat: (modelId: string) => ({
provider: "krea",
doGenerate: async (options: {
messages: Array<{ role: string; content: any }>;
imageParams?: KreaImageParams;
videoParams?: KreaVideoParams;
overrideModelId?: string;
}) => {
try {
const activeModelId = normalizeText(options.overrideModelId || modelId);
const endpoint = resolveKreaEndpoint(activeModelId);
const preset = getKreaModelPresetById(endpoint);
const kind =
preset?.kind ||
inferKreaModelKind(endpoint) ||
(options.videoParams ? "video" : "image");
const { prompt, imageUrls, videoUrls } = extractPromptAndAssets(options.messages);
const normalizedImageUrls = (
await Promise.all(imageUrls.map((url) => normalizeAssetUrl(url)))
).filter(Boolean);
const normalizedVideoUrls = (
await Promise.all(videoUrls.map((url) => normalizeAssetUrl(url)))
).filter(Boolean);
const requestFields =
preset?.requestFields ||
(kind === "video"
? ["prompt", "aspectRatio", "duration", "startImage", "endImage", "referenceImages"]
: ["prompt", "width", "height", "imageUrls", "imageUrl", "batchSize"]);
const requiredFields = preset?.requiredFields || ["prompt"];
const { body, imageSize } = buildKreaBody({
endpoint,
kind,
requestFields,
requiredFields,
prompt,
imageUrls: normalizedImageUrls,
videoUrls: normalizedVideoUrls,
imageParams: options.imageParams,
videoParams: options.videoParams,
});
console.log("[Krea] Submitting job:", {
endpoint,
kind,
requestFields: Object.keys(body),
imageCount: normalizedImageUrls.length,
videoCount: normalizedVideoUrls.length,
});
const job = await requestKreaJob(baseURL, apiKey, endpoint, body);
const output = job?.result?.urls?.length
? job
: await waitForKreaJob(baseURL, apiKey, job.job_id, kind);
const text =
kind === "video"
? formatVideoMarkdown(await resolveKreaVideoUrl(output?.result ?? output))
: formatImageMarkdown(await uploadKreaImages(output?.result ?? output), imageSize);
return {
text,
finishReason: "stop" as const,
usage: {
promptTokens: 0,
completionTokens: text.length,
},
};
} catch (error) {
console.error("[Krea] Generation failed:", error);
const kind =
inferKreaModelKind(options.overrideModelId || modelId) ||
(options.videoParams ? "video" : "image");
return {
text:
error instanceof Error
? `${kind === "video" ? "视频" : "图片"}生成失败: ${error.message}`
: "Krea 生成失败,请稍后重试",
finishReason: "error" as const,
usage: {
promptTokens: 0,
completionTokens: 0,
},
};
}
},
doStream: async function* (options: {
messages: Array<{ role: string; content: any }>;
imageParams?: KreaImageParams;
videoParams?: KreaVideoParams;
overrideModelId?: string;
}) {
const result = await this.doGenerate(options);
yield {
type: "text-delta",
textDelta: result.text,
};
},
modelId,
specificationVersion: "v1" as const,
defaultObjectGenerationMode: "json" as const,
}),
};
}

View File

@ -0,0 +1,53 @@
export interface AgentToolImageMarkdownItem {
originalUrl: string;
thumbnailUrl?: string | null;
imageSize?: string;
}
export function prependImageModelBadge(
text: string,
modelDisplayName?: string | null,
): string {
const normalizedText = String(text || "").trim();
if (!normalizedText) return normalizedText;
const normalizedModelName = String(modelDisplayName || "").trim();
if (!normalizedModelName) return normalizedText;
if (
/^\s*[◈◇◆🔹⬡✦•·]\s*[^\n]+/im.test(normalizedText) ||
/^\s*(?:Routed to|Using model|使用模型|模型)\s*[::]/im.test(normalizedText)
) {
return normalizedText;
}
return `◈ ${normalizedModelName}\n\n${normalizedText}`;
}
export function buildAgentToolImageMarkdown(params: AgentToolImageMarkdownItem): string {
const originalUrl = String(params.originalUrl || "").trim();
const thumbnailUrl = String(params.thumbnailUrl || originalUrl).trim() || originalUrl;
const imageSize = String(params.imageSize || "2K").trim() || "2K";
return `![thumb|||${originalUrl}|||${imageSize}](${thumbnailUrl})`;
}
export function buildAgentToolImagesMarkdown(items: AgentToolImageMarkdownItem[]) {
return items
.map((item) =>
buildAgentToolImageMarkdown({
originalUrl: item.originalUrl,
thumbnailUrl: item.thumbnailUrl,
imageSize: item.imageSize,
}),
)
.join("\n");
}
export function buildAgentToolVideoMarkdown(videoUrl: string) {
const normalizedUrl = String(videoUrl || "").trim();
if (!normalizedUrl) return "";
return [
`<video src="${normalizedUrl}" controls style="max-width: 100%; border-radius: 8px;"></video>`,
`[下载视频](${normalizedUrl})`,
].join("\n\n");
}

View File

@ -0,0 +1,30 @@
import { saveMessages } from "@/lib/db/queries";
export async function saveAssistantTextOnlyMessage(params: {
chatId: string;
messageId: string;
text: string;
videoStorageStatus?: "temporary" | "permanent" | "failed" | null;
temporaryVideoUrl?: string | null;
permanentVideoUrl?: string | null;
archivedAt?: Date | null;
videoMetadata?: Record<string, any> | null;
}) {
await saveMessages({
messages: [
{
id: params.messageId,
chatId: params.chatId,
role: "assistant",
parts: [{ type: "text", text: params.text }],
attachments: [],
createdAt: new Date(),
videoStorageStatus: params.videoStorageStatus ?? null,
temporaryVideoUrl: params.temporaryVideoUrl ?? null,
permanentVideoUrl: params.permanentVideoUrl ?? null,
archivedAt: params.archivedAt ?? null,
videoMetadata: params.videoMetadata ?? null,
},
],
});
}

View File

@ -48,6 +48,12 @@ export interface ResolvedDirectAgentToolExecution {
publishToCanvas: boolean;
}
export interface ResolvedDirectAgentToolContext {
execution: ResolvedDirectAgentToolExecution;
targetImageUrls: string[];
targetImageUrl: string | null;
}
export interface DirectAgentWorkingStateSnapshot {
lastResolvedTargetImageUrls?: string[] | null;
lastGeneratedImageUrls?: string[] | null;
@ -162,8 +168,9 @@ export function resolveDirectAgentToolExecution(
plan?: AgentExecutionPlan,
): ResolvedDirectAgentToolExecution | null {
const mergedToolCalls = collectAgentToolCalls(execution, plan);
const toolNames = mergedToolCalls.map((tool) => tool.name);
const hasToolCall = (toolName: string) => toolNames.includes(toolName);
const toolNames = mergedToolCalls.map((tool) => String(tool.name || ""));
const toolNameSet = new Set(toolNames);
const hasToolCall = (toolName: string) => toolNameSet.has(toolName);
const sourceTools = toolNames.filter((toolName): toolName is DirectAgentSourceToolName =>
toolName === "useSelectedCanvasImage" ||
toolName === "useLatestResultImage" ||
@ -288,6 +295,52 @@ export function resolveDirectAgentToolExecution(
};
}
export function resolveDirectAgentToolContext(params: {
execution?: AgentPreparedExecution;
plan?: AgentExecutionPlan;
requestResolvedMediaInput?: ResolvedChatMediaInput | null;
requestTargetResolution?: ResolvedTargetAssets | null;
requestWorkingState?: DirectAgentWorkingStateSnapshot | null;
}): ResolvedDirectAgentToolContext | null {
const execution = resolveDirectAgentToolExecution(params.execution, params.plan);
if (!execution) return null;
const targetImageUrls = pickAgentSourceImageUrls({
sourcePreferences: execution.sourcePreferences,
requestResolvedMediaInput: params.requestResolvedMediaInput,
requestTargetResolution: params.requestTargetResolution,
requestWorkingState: params.requestWorkingState,
});
return {
execution,
targetImageUrls,
targetImageUrl: targetImageUrls[0] || null,
};
}
export function isDirectAgentImageEditToolName(
toolName?: DirectAgentToolName | null,
): toolName is
| "editImage"
| "enhanceImage"
| "removeBackground"
| "resizeImage"
| "extendImage"
| "boxCutout" {
switch (toolName) {
case "editImage":
case "enhanceImage":
case "removeBackground":
case "resizeImage":
case "extendImage":
case "boxCutout":
return true;
default:
return false;
}
}
export function getDirectAgentToolLabel(capability: DirectAgentToolCapability): string {
switch (capability) {
case "analyze":

View File

@ -0,0 +1,444 @@
import {
BoxCutoutServiceError,
executeBoxCutoutJob,
} from "@/lib/ai/box-cutout/service";
import {
executeExtendJob,
ExtendServiceError,
inferExtendOptionsFromText,
} from "@/lib/ai/extend/service";
import {
executeResizeJob,
inferResizeOptionsFromText,
ResizeServiceError,
} from "@/lib/ai/resize/service";
import {
executeRmbgJob,
fetchImageBufferFromUrl,
RmbgServiceError,
} from "@/lib/ai/rmbg/service";
import { generateCanvasImage } from "@/lib/canvas/generate-image";
import { requestLongTaskResult } from "@/lib/client/long-task";
import { getMediaAssetsByUserAndUrls } from "@/lib/db/queries";
import type {
AgentExecutionPlan,
AgentPreparedExecution,
} from "@/lib/ai/auto-agent-graph";
import {
findAgentToolCallArgs,
type DirectAgentToolCapability,
} from "@/lib/ai/runtime/direct-agent-execution";
import { buildDirectAgentExecutionPrompt } from "@/lib/ai/runtime/optimized-prompt";
import { buildAgentToolImagesMarkdown } from "@/lib/ai/runtime/agent-media-markdown";
interface DirectAgentImageParamsSnapshot {
negativePrompt?: string;
aspectRatio?: string;
imageSize?: string;
}
type SaveGeneratedImages = (
imageMarkdown: string,
userId: string,
chatId: string,
messageId: string,
prompt: string,
modelId: string,
options?: {
sourceAssetIds?: string[];
sourceImageUrls?: string[];
relationType?: "generated_from" | "edited_from";
sourceType?: string;
projectId?: string | null;
userPrompt?: string;
optimizedPrompt?: string;
requestedImageCount?: number;
aspectRatio?: string;
imageSize?: string;
},
) => Promise<void>;
export interface DirectAgentImageEditExecutionResult {
originalUrl: string;
thumbnailUrl: string | null;
imageSize: string;
mediaAssetId: string | null;
}
export interface ExecuteDirectAgentImageEditToolParams {
capability: DirectAgentToolCapability;
requestUrl: string;
requestHeaders: Headers;
userId: string;
chatId: string;
messageId: string;
projectId?: string | null;
targetAssetIds?: string[] | null;
sourceImageUrl: string;
sourceImageUrls: string[];
requestTextContent: string;
optimizedPrompt?: string;
hiddenPromptPrefix?: string;
finalModelId: string;
imageParams?: DirectAgentImageParamsSnapshot | null;
defaultImageQualityParams?: DirectAgentImageParamsSnapshot | null;
smartAutoExecution?: AgentPreparedExecution;
smartAutoPlan?: AgentExecutionPlan;
saveGeneratedImages: SaveGeneratedImages;
}
function toFiniteNumber(value: unknown): number | undefined {
const normalized = Number(value);
return Number.isFinite(normalized) ? normalized : undefined;
}
async function executeDirectAgentEnhanceJob(params: {
requestUrl: string;
requestHeaders: Headers;
imageUrl: string;
prompt: string;
chatId: string;
messageId: string;
}) {
const endpoint = new URL("/api/enhance", params.requestUrl).toString();
const cookieHeader = params.requestHeaders.get("cookie") || "";
const result = await requestLongTaskResult({
endpoint,
payload: {
imageUrl: params.imageUrl,
prompt: params.prompt,
chatId: params.chatId,
messageId: params.messageId,
},
pollingParams: {
prompt: params.prompt,
chatId: params.chatId,
messageId: params.messageId,
sourceImageUrl: params.imageUrl,
},
timeoutMs: 8 * 60 * 1000,
headers: cookieHeader
? {
cookie: cookieHeader,
}
: undefined,
});
if (!result?.success || !result?.original_url) {
throw new Error(String(result?.error || result?.detail || "图像增强失败"));
}
return {
originalUrl: String(result.original_url),
thumbnailUrl: String(result.thumbnail_url || result.original_url),
imageSize: "2K",
};
}
export function isDirectAgentImageEditServiceError(error: unknown) {
return (
error instanceof RmbgServiceError ||
error instanceof ResizeServiceError ||
error instanceof ExtendServiceError ||
error instanceof BoxCutoutServiceError
);
}
async function resolveDirectAgentResultAssetId(params: {
userId: string;
originalUrl: string;
mediaAssetId?: string | null;
capability: DirectAgentToolCapability;
}) {
if (params.mediaAssetId || !params.originalUrl) {
return params.mediaAssetId || null;
}
try {
const [resultAsset] = await getMediaAssetsByUserAndUrls({
userId: params.userId,
urls: [params.originalUrl],
});
return resultAsset?.id || null;
} catch (assetLookupError) {
console.warn(
`[Direct Agent Image Edit] Failed to lookup result asset after ${params.capability}:`,
assetLookupError,
);
return null;
}
}
async function finalizeDirectAgentImageEditResult(params: {
userId: string;
capability: DirectAgentToolCapability;
originalUrl: string;
thumbnailUrl?: string | null;
imageSize: string;
mediaAssetId?: string | null;
}): Promise<DirectAgentImageEditExecutionResult> {
return {
originalUrl: params.originalUrl,
thumbnailUrl: params.thumbnailUrl || null,
imageSize: params.imageSize,
mediaAssetId: await resolveDirectAgentResultAssetId({
userId: params.userId,
originalUrl: params.originalUrl,
mediaAssetId: params.mediaAssetId,
capability: params.capability,
}),
};
}
export async function executeDirectAgentImageEditTool(
params: ExecuteDirectAgentImageEditToolParams,
): Promise<DirectAgentImageEditExecutionResult> {
if (params.capability === "generic_edit") {
const directEditPrompt =
buildDirectAgentExecutionPrompt({
originalText: params.requestTextContent || "",
optimizedPrompt: params.optimizedPrompt,
hiddenPromptPrefix: params.hiddenPromptPrefix,
fallback: "在保持主体和整体构图稳定的前提下完成本轮图片编辑。",
}) || "在保持主体和整体构图稳定的前提下完成本轮图片编辑。";
const aspectRatio =
params.imageParams?.aspectRatio || params.defaultImageQualityParams?.aspectRatio;
const imageSize =
params.imageParams?.imageSize || params.defaultImageQualityParams?.imageSize || "2K";
const result = await generateCanvasImage({
prompt: directEditPrompt,
negativePrompt: params.imageParams?.negativePrompt,
model: params.finalModelId,
aspectRatio,
imageSize,
imageCount: 1,
referenceImages: params.sourceImageUrls,
debugLabel: "agent-direct-edit",
debugMeta: {
chatId: params.chatId,
projectId: params.projectId || null,
subIntent: params.smartAutoExecution?.routeSubIntent || null,
},
});
const firstImage = result.images[0];
if (!firstImage?.originalUrl) {
throw new Error("图片编辑完成但未返回结果图");
}
const originalUrl = firstImage.originalUrl;
const thumbnailUrl = firstImage.thumbnailUrl || null;
const editResultMarkdown = buildAgentToolImagesMarkdown([
{
originalUrl,
thumbnailUrl,
imageSize,
},
]);
await params.saveGeneratedImages(
editResultMarkdown,
params.userId,
params.chatId,
params.messageId,
directEditPrompt,
result.modelRef || params.finalModelId,
{
sourceAssetIds: params.targetAssetIds || undefined,
sourceImageUrls: params.sourceImageUrls,
relationType: "edited_from",
sourceType: "edited",
projectId: params.projectId || null,
userPrompt: params.requestTextContent,
optimizedPrompt: directEditPrompt,
requestedImageCount: 1,
aspectRatio,
imageSize,
},
);
return finalizeDirectAgentImageEditResult({
userId: params.userId,
capability: params.capability,
originalUrl,
thumbnailUrl,
imageSize,
mediaAssetId: null,
});
}
if (params.capability === "enhance") {
const result = await executeDirectAgentEnhanceJob({
requestUrl: params.requestUrl,
requestHeaders: params.requestHeaders,
imageUrl: params.sourceImageUrl,
prompt: params.requestTextContent || "图像增强处理",
chatId: params.chatId,
messageId: params.messageId,
});
return finalizeDirectAgentImageEditResult({
userId: params.userId,
capability: params.capability,
originalUrl: result.originalUrl,
thumbnailUrl: result.thumbnailUrl,
imageSize: result.imageSize,
mediaAssetId: null,
});
}
if (params.capability === "remove_background") {
const sourceBuffer = await fetchImageBufferFromUrl(params.sourceImageUrl);
const result = await executeRmbgJob({
userId: params.userId,
imageBuffer: sourceBuffer,
prompt: params.requestTextContent || "抠图处理",
chatId: params.chatId,
messageId: params.messageId,
sourceImageUrl: params.sourceImageUrl,
});
return finalizeDirectAgentImageEditResult({
userId: params.userId,
capability: params.capability,
originalUrl: result.original_url,
thumbnailUrl: result.thumbnail_url,
imageSize: "2K",
mediaAssetId: result.mediaAssetId || null,
});
}
if (params.capability === "resize") {
const plannedArgs = findAgentToolCallArgs(
"resizeImage",
params.smartAutoExecution,
params.smartAutoPlan,
);
const inferredOptions = inferResizeOptionsFromText(params.requestTextContent);
const canvasWidth = toFiniteNumber(
plannedArgs?.canvasWidth ?? plannedArgs?.width ?? inferredOptions.canvasWidth,
);
const canvasHeight = toFiniteNumber(
plannedArgs?.canvasHeight ?? plannedArgs?.height ?? inferredOptions.canvasHeight,
);
if (!canvasWidth || !canvasHeight) {
throw new ResizeServiceError(
"请告诉我目标尺寸或比例,例如 1024x1024、4:5、16:9。",
400,
);
}
const transparentBackground =
plannedArgs?.transparentBackground === true ||
plannedArgs?.transparentBackground === "true" ||
inferredOptions.transparentBackground === true;
const backgroundColor =
typeof plannedArgs?.backgroundColor === "string"
? plannedArgs.backgroundColor
: inferredOptions.backgroundColor;
const mimeType =
typeof plannedArgs?.mimeType === "string" ? plannedArgs.mimeType : undefined;
const result = await executeResizeJob({
userId: params.userId,
imageUrl: params.sourceImageUrl,
prompt: params.requestTextContent || "尺寸调整处理",
chatId: params.chatId,
messageId: params.messageId,
sourceImageUrl: params.sourceImageUrl,
canvasWidth,
canvasHeight,
transparentBackground,
backgroundColor,
mimeType,
});
return finalizeDirectAgentImageEditResult({
userId: params.userId,
capability: params.capability,
originalUrl: result.original_url,
thumbnailUrl: result.thumbnail_url,
imageSize: `${result.canvas_width}x${result.canvas_height}`,
mediaAssetId: result.mediaAssetId || null,
});
}
if (params.capability === "extend") {
const plannedArgs = findAgentToolCallArgs(
"extendImage",
params.smartAutoExecution,
params.smartAutoPlan,
);
const inferredOptions = inferExtendOptionsFromText(params.requestTextContent);
const xScale = toFiniteNumber(plannedArgs?.xScale ?? inferredOptions.xScale);
const yScale = toFiniteNumber(plannedArgs?.yScale ?? inferredOptions.yScale);
const result = await executeExtendJob({
userId: params.userId,
imageUrl: params.sourceImageUrl,
prompt: params.requestTextContent || "图像拓展处理",
chatId: params.chatId,
messageId: params.messageId,
sourceImageUrl: params.sourceImageUrl,
...(xScale ? { xScale } : {}),
...(yScale ? { yScale } : {}),
});
return finalizeDirectAgentImageEditResult({
userId: params.userId,
capability: params.capability,
originalUrl: result.original_url,
thumbnailUrl: result.thumbnail_url,
imageSize: "2K",
mediaAssetId: result.mediaAssetId || null,
});
}
if (params.capability === "box_cutout") {
const plannedArgs = findAgentToolCallArgs(
"boxCutout",
params.smartAutoExecution,
params.smartAutoPlan,
);
const cropRect =
plannedArgs?.cropRect && typeof plannedArgs.cropRect === "object"
? (plannedArgs.cropRect as Record<string, unknown>)
: undefined;
if (!cropRect) {
throw new BoxCutoutServiceError(
"缺少框选区域,暂时无法执行框选抠图。",
400,
);
}
const result = await executeBoxCutoutJob({
userId: params.userId,
imageUrl: params.sourceImageUrl,
cropRect: {
x: toFiniteNumber(cropRect.x),
y: toFiniteNumber(cropRect.y),
width: toFiniteNumber(cropRect.width),
height: toFiniteNumber(cropRect.height),
},
paddingRatio: toFiniteNumber(plannedArgs?.paddingRatio),
prompt: params.requestTextContent || "框选抠图处理",
chatId: params.chatId,
messageId: params.messageId,
sourceNodeId:
typeof plannedArgs?.sourceNodeId === "string"
? plannedArgs.sourceNodeId
: undefined,
});
return finalizeDirectAgentImageEditResult({
userId: params.userId,
capability: params.capability,
originalUrl: result.original_url,
thumbnailUrl: result.thumbnail_url || null,
imageSize: "2K",
mediaAssetId: result.mediaAssetId || null,
});
}
throw new Error(`不支持的图片编辑工具能力:${params.capability}`);
}

View File

@ -0,0 +1,238 @@
import { generateCanvasImage } from "@/lib/canvas/generate-image";
import {
executeDirectVideoGeneration,
type ExecuteDirectVideoGenerationParams,
type ExecuteDirectVideoGenerationResult,
} from "@/lib/ai/tools/generate-video";
import type {
AgentExecutionPlan,
AgentPreparedExecution,
} from "@/lib/ai/auto-agent-graph";
import { findAgentToolCallArgs } from "@/lib/ai/runtime/direct-agent-execution";
import { buildDirectAgentImageAnalysisAnswer } from "@/lib/ai/runtime/direct-agent-analysis-answer";
import { buildAgentToolImagesMarkdown } from "@/lib/ai/runtime/agent-media-markdown";
interface DirectAgentImageParamsSnapshot {
negativePrompt?: string;
aspectRatio?: string;
imageSize?: string;
imageCount?: number;
}
type DirectAgentVideoMode = ExecuteDirectVideoGenerationParams["mode"];
type DirectAgentVideoRatio = NonNullable<ExecuteDirectVideoGenerationParams["ratio"]>;
export interface DirectAgentImageAnalysisResult {
summary: string;
answer: string;
}
export interface ExecuteDirectAgentImageAnalysisParams {
requestTextContent: string;
imageUrls: string[];
context?: string;
preferredModelId?: string;
answerModelId?: string;
assetIds?: string[];
}
export interface DirectAgentImageGenerationResult {
prompt: string;
imageMarkdown: string;
images: Array<{
originalUrl: string;
thumbnailUrl?: string | null;
}>;
model: string;
modelRef?: string | null;
imageSize?: string;
imageCount: number;
}
export interface ExecuteDirectAgentImageGenerationParams {
requestTextContent: string;
optimizedPrompt?: string;
finalModelId: string;
imageParams?: DirectAgentImageParamsSnapshot | null;
defaultImageQualityParams?: DirectAgentImageParamsSnapshot | null;
referenceImageUrls: string[];
}
export interface DirectAgentVideoGenerationResult {
prompt: string;
mode: DirectAgentVideoMode;
images: string[];
videoUrl: string;
result: ExecuteDirectVideoGenerationResult;
}
export interface ExecuteDirectAgentVideoGenerationParams {
requestTextContent: string;
optimizedPrompt?: string;
finalModelId: string;
sourceImageUrls: string[];
videoParams?: {
mode?: string;
resolution?: ExecuteDirectVideoGenerationParams["resolution"];
ratio?: string;
duration?: number;
} | null;
smartAutoExecution?: AgentPreparedExecution;
smartAutoPlan?: AgentExecutionPlan;
}
function normalizeDirectAgentVideoMode(value: unknown): DirectAgentVideoMode | undefined {
const normalized = String(value || "").trim();
switch (normalized) {
case "text-to-video":
case "first-frame":
case "first-last-frame":
case "reference-images":
return normalized;
default:
return undefined;
}
}
function normalizeDirectAgentVideoRatio(value: unknown): DirectAgentVideoRatio | undefined {
const normalized = String(value || "").trim();
switch (normalized) {
case "16:9":
case "4:3":
case "1:1":
case "3:4":
case "9:16":
case "21:9":
case "adaptive":
return normalized;
default:
return undefined;
}
}
function inferDirectAgentVideoMode(imageUrls: string[]): DirectAgentVideoMode {
if (imageUrls.length >= 2) return "first-last-frame";
if (imageUrls.length >= 1) return "first-frame";
return "text-to-video";
}
function pickDirectAgentVideoImages(mode: DirectAgentVideoMode, imageUrls: string[]): string[] {
switch (mode) {
case "first-last-frame":
return imageUrls.slice(0, 2);
case "first-frame":
return imageUrls.slice(0, 1);
case "reference-images":
return imageUrls.slice(0, 4);
case "text-to-video":
default:
return [];
}
}
export async function executeDirectAgentImageAnalysis(
params: ExecuteDirectAgentImageAnalysisParams,
): Promise<DirectAgentImageAnalysisResult> {
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
const summary = await summarizeImageForExecutionReport({
text: params.requestTextContent,
imageUrls: params.imageUrls,
context: params.context,
preferredModelId: params.preferredModelId,
assetIds: params.assetIds,
});
if (!summary?.trim()) {
throw new Error("当前图片暂时无法提取稳定的视觉摘要,请稍后再试。");
}
const normalizedSummary = summary.trim();
const answer = await buildDirectAgentImageAnalysisAnswer({
question: params.requestTextContent,
summary: normalizedSummary,
modelId: params.answerModelId,
});
return {
summary: normalizedSummary,
answer,
};
}
export async function executeDirectAgentImageGeneration(
params: ExecuteDirectAgentImageGenerationParams,
): Promise<DirectAgentImageGenerationResult> {
const prompt =
String(params.optimizedPrompt || params.requestTextContent || "").trim() || "生成图片";
const imageSize =
params.imageParams?.imageSize || params.defaultImageQualityParams?.imageSize;
const imageCount =
params.imageParams?.imageCount || params.defaultImageQualityParams?.imageCount;
const result = await generateCanvasImage({
prompt,
negativePrompt: params.imageParams?.negativePrompt,
model: params.finalModelId,
aspectRatio:
params.imageParams?.aspectRatio || params.defaultImageQualityParams?.aspectRatio,
imageSize,
imageCount,
referenceImages: params.referenceImageUrls,
});
const imageMarkdown = buildAgentToolImagesMarkdown(
result.images.map((image) => ({
originalUrl: image.originalUrl,
thumbnailUrl: image.thumbnailUrl,
imageSize,
})),
);
return {
prompt,
imageMarkdown,
images: result.images,
model: result.model,
modelRef: result.modelRef || null,
imageSize,
imageCount: result.images.length,
};
}
export async function executeDirectAgentVideoGeneration(
params: ExecuteDirectAgentVideoGenerationParams,
): Promise<DirectAgentVideoGenerationResult> {
const prompt =
String(params.optimizedPrompt || params.requestTextContent || "").trim() || "生成视频";
const plannedArgs = findAgentToolCallArgs(
"generateVideo",
params.smartAutoExecution,
params.smartAutoPlan,
);
const mode =
normalizeDirectAgentVideoMode(plannedArgs?.mode) ||
normalizeDirectAgentVideoMode(params.videoParams?.mode) ||
inferDirectAgentVideoMode(params.sourceImageUrls);
const images = pickDirectAgentVideoImages(mode, params.sourceImageUrls);
const result = await executeDirectVideoGeneration({
prompt,
mode,
images,
resolution: params.videoParams?.resolution,
ratio: normalizeDirectAgentVideoRatio(params.videoParams?.ratio),
duration: params.videoParams?.duration,
preferredModelRef: params.finalModelId,
});
const videoUrl = String(result.videoUrl || "").trim();
if (!videoUrl) {
throw new Error("视频生成完成但未返回视频地址");
}
return {
prompt,
mode,
images,
videoUrl,
result,
};
}

View File

@ -0,0 +1,213 @@
import type {
DirectAgentToolCapability,
DirectAgentToolName,
} from "@/lib/ai/runtime/direct-agent-execution";
type DirectAgentProgressStage =
| "planning"
| "clarify"
| "executing"
| "completed"
| "failed";
type DirectAgentProgressStep =
| "lock_target"
| "analyze"
| "select_tool"
| "execute"
| "finalize";
export interface DirectAgentProgressPayload {
stage: DirectAgentProgressStage;
currentStep: DirectAgentProgressStep;
title?: string;
message?: string;
toolName?: DirectAgentToolName;
capability?: DirectAgentToolCapability;
}
function buildDirectAgentProgressPayload(params: DirectAgentProgressPayload) {
return params;
}
export function buildDirectAgentAnalyzePlanningProgress() {
return buildDirectAgentProgressPayload({
stage: "planning",
currentStep: "analyze",
title: "正在分析目标图片",
message: "我会先提取稳定的视觉摘要,再整理成更直接的分析结论。",
toolName: "analyzeImage",
capability: "analyze",
});
}
export function buildDirectAgentAnalyzeCompletedProgress() {
return buildDirectAgentProgressPayload({
stage: "completed",
currentStep: "finalize",
title: "图片分析完成",
message: "我已经整理好可直接阅读的分析结论。",
toolName: "analyzeImage",
capability: "analyze",
});
}
export function buildDirectAgentAnalyzeFailedProgress(errorMessage: string) {
return buildDirectAgentProgressPayload({
stage: "failed",
currentStep: "analyze",
title: "图片分析失败",
message: errorMessage,
toolName: "analyzeImage",
capability: "analyze",
});
}
export function buildDirectAgentImageGenerateSelectProgress() {
return buildDirectAgentProgressPayload({
stage: "planning",
currentStep: "select_tool",
title: "已选中图片生成工具",
message: "我正在整理参考图和生图参数,准备开始生成。",
toolName: "generateImage",
capability: "generate_image",
});
}
export function buildDirectAgentImageGenerateExecutingProgress() {
return buildDirectAgentProgressPayload({
stage: "executing",
currentStep: "execute",
title: "正在生成图片",
message: "我会结合当前提示词和参考图生成新的画面。",
toolName: "generateImage",
capability: "generate_image",
});
}
export function buildDirectAgentImageGenerateCompletedProgress() {
return buildDirectAgentProgressPayload({
stage: "completed",
currentStep: "finalize",
title: "图片生成完成",
message: "结果已经回写到对话和画布。",
toolName: "generateImage",
capability: "generate_image",
});
}
export function buildDirectAgentImageGenerateFailedProgress(errorMessage: string) {
return buildDirectAgentProgressPayload({
stage: "failed",
currentStep: "execute",
title: "图片生成失败",
message: errorMessage,
toolName: "generateImage",
capability: "generate_image",
});
}
export function buildDirectAgentVideoGenerateSelectProgress() {
return buildDirectAgentProgressPayload({
stage: "planning",
currentStep: "select_tool",
title: "已选中视频生成工具",
message: "我正在整理模式、参考图和视频参数。",
toolName: "generateVideo",
capability: "generate_video",
});
}
export function buildDirectAgentVideoGenerateExecutingProgress() {
return buildDirectAgentProgressPayload({
stage: "executing",
currentStep: "execute",
title: "正在生成视频",
message: "我正在按当前模式执行视频生成。",
toolName: "generateVideo",
capability: "generate_video",
});
}
export function buildDirectAgentVideoGenerateCompletedProgress() {
return buildDirectAgentProgressPayload({
stage: "completed",
currentStep: "finalize",
title: "视频生成完成",
message: "结果已经写回对话和画布。",
toolName: "generateVideo",
capability: "generate_video",
});
}
export function buildDirectAgentVideoGenerateFailedProgress(errorMessage: string) {
return buildDirectAgentProgressPayload({
stage: "failed",
currentStep: "execute",
title: "视频生成失败",
message: errorMessage,
toolName: "generateVideo",
capability: "generate_video",
});
}
export function buildDirectAgentImageEditSelectProgress(params: {
toolName: DirectAgentToolName;
capability: DirectAgentToolCapability;
label: string;
}) {
return buildDirectAgentProgressPayload({
stage: "planning",
currentStep: "select_tool",
title: `已选中${params.label}`,
message: "我正在整理目标图和当前参数,准备开始执行。",
toolName: params.toolName,
capability: params.capability,
});
}
export function buildDirectAgentImageEditExecutingProgress(params: {
toolName: DirectAgentToolName;
capability: DirectAgentToolCapability;
label: string;
}) {
return buildDirectAgentProgressPayload({
stage: "executing",
currentStep: "execute",
title: `正在执行${params.label}`,
message: "执行完成后我会把结果直接回写到当前对话和画布。",
toolName: params.toolName,
capability: params.capability,
});
}
export function buildDirectAgentImageEditCompletedProgress(params: {
toolName: DirectAgentToolName;
capability: DirectAgentToolCapability;
label: string;
}) {
return buildDirectAgentProgressPayload({
stage: "completed",
currentStep: "finalize",
title: `${params.label}完成`,
message: "结果已经回写到当前对话和画布。",
toolName: params.toolName,
capability: params.capability,
});
}
export function buildDirectAgentImageEditFailedProgress(params: {
toolName: DirectAgentToolName;
capability: DirectAgentToolCapability;
label: string;
errorMessage: string;
}) {
return buildDirectAgentProgressPayload({
stage: "failed",
currentStep: "execute",
title: `${params.label}失败`,
message: params.errorMessage,
toolName: params.toolName,
capability: params.capability,
});
}

View File

@ -0,0 +1,78 @@
import {
buildAgentToolImageMarkdown,
buildAgentToolVideoMarkdown,
prependImageModelBadge,
} from "@/lib/ai/runtime/agent-media-markdown";
import {
getDirectAgentToolLabel,
getDirectAgentToolSuccessMessage,
type DirectAgentToolCapability,
} from "@/lib/ai/runtime/direct-agent-execution";
export function buildDirectAgentAnalysisAssistantText(params: {
summary: string;
answer?: string | null;
}) {
const answer = String(params.answer || "").trim();
return prependImageModelBadge(
[
getDirectAgentToolSuccessMessage("analyze"),
`图像观察:\n${String(params.summary || "").trim()}`,
answer ? `分析结论:\n${answer}` : "",
]
.filter(Boolean)
.join("\n\n"),
getDirectAgentToolLabel("analyze"),
);
}
export function buildDirectAgentImageGenerationAssistantText(params: {
imageMarkdown: string;
modelDisplayName?: string | null;
}) {
return prependImageModelBadge(
[
getDirectAgentToolSuccessMessage("generate_image"),
params.imageMarkdown,
]
.filter(Boolean)
.join("\n\n"),
params.modelDisplayName,
);
}
export function buildDirectAgentVideoGenerationAssistantText(params: {
videoUrl: string;
modelDisplayName?: string | null;
}) {
return prependImageModelBadge(
[
getDirectAgentToolSuccessMessage("generate_video"),
buildAgentToolVideoMarkdown(params.videoUrl),
]
.filter(Boolean)
.join("\n\n"),
params.modelDisplayName,
);
}
export function buildDirectAgentImageEditAssistantText(params: {
capability: DirectAgentToolCapability;
originalUrl: string;
thumbnailUrl?: string | null;
imageSize?: string;
}) {
return prependImageModelBadge(
[
getDirectAgentToolSuccessMessage(params.capability),
buildAgentToolImageMarkdown({
originalUrl: params.originalUrl,
thumbnailUrl: params.thumbnailUrl,
imageSize: params.imageSize,
}),
]
.filter(Boolean)
.join("\n\n"),
getDirectAgentToolLabel(params.capability),
);
}

View File

@ -0,0 +1,144 @@
import {
persistImageWorkflowTurn,
persistVideoGenerationCompletedTurn,
persistVideoGenerationFailedTurn,
} from "@/lib/ai/conversation-state-protocol";
import type { DirectAgentToolCapability } from "@/lib/ai/runtime/direct-agent-execution";
export async function safePersistDirectAgentAnalyzeTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
currentSubIntent?: string | null;
targetAssetIds?: string[] | null;
targetImageUrls: string[];
messageId: string;
requestTextContent: string;
}) {
try {
const targetImageUrls = params.targetImageUrls.length > 0 ? params.targetImageUrls : null;
await persistImageWorkflowTurn({
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId || null,
currentIntent: "vision",
currentSubIntent: params.currentSubIntent || "analyze",
currentFocusAssetIds: params.targetAssetIds || null,
currentFocusImageUrls: targetImageUrls,
lastResolvedTargetAssetIds: params.targetAssetIds || null,
lastResolvedTargetImageUrls: targetImageUrls,
lastMessageId: params.messageId,
lastUserPrompt: params.requestTextContent,
lastSourceImageUrls: targetImageUrls,
});
} catch (memoryError) {
console.warn("[Direct Agent Working State] Failed to persist analyze turn:", memoryError);
}
}
export async function safePersistDirectAgentImageEditTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
capability: DirectAgentToolCapability;
currentSubIntent?: string | null;
resultAssetId?: string | null;
resultOriginalUrl?: string | null;
resultImageSize?: string | null;
sourceImageUrls: string[];
messageId: string;
requestTextContent: string;
optimizedPrompt?: string | null;
}) {
try {
const resultAssetIds = params.resultAssetId ? [params.resultAssetId] : null;
const resultImageUrls = params.resultOriginalUrl ? [params.resultOriginalUrl] : null;
await persistImageWorkflowTurn({
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId || null,
currentIntent: "image_edit",
currentSubIntent: params.currentSubIntent || params.capability,
currentFocusAssetIds: resultAssetIds,
currentFocusImageUrls: resultImageUrls,
lastGeneratedAssetIds: resultAssetIds,
lastGeneratedImageUrls: resultImageUrls,
lastResolvedTargetAssetIds: resultAssetIds,
lastResolvedTargetImageUrls: resultImageUrls,
lastMessageId: params.messageId,
lastUserPrompt: params.requestTextContent,
lastOptimizedPrompt: params.optimizedPrompt || undefined,
lastRequestedImageCount: resultImageUrls?.length || 1,
lastImageSize: params.resultImageSize || undefined,
lastSourceImageUrls: params.sourceImageUrls,
lastResultImageUrls: resultImageUrls,
});
} catch (memoryError) {
console.warn(
`[Direct Agent Working State] Failed to persist image edit turn after ${params.capability}:`,
memoryError,
);
}
}
export async function safePersistDirectAgentVideoCompletedTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
messageId: string;
requestTextContent: string;
modelRef: string;
mode: string;
temporaryVideoUrl: string;
}) {
try {
await persistVideoGenerationCompletedTurn({
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId || null,
messageId: params.messageId,
text: params.requestTextContent,
source: "agent_direct_generate_video",
providerType: "direct-video",
modelRef: params.modelRef,
mode: params.mode as any,
status: "completed",
temporaryVideoUrl: params.temporaryVideoUrl,
});
} catch (memoryError) {
console.warn(
"[Direct Agent Working State] Failed to persist direct video generation:",
memoryError,
);
}
}
export async function safePersistDirectAgentVideoFailedTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
messageId: string;
requestTextContent: string;
modelRef: string;
error: unknown;
}) {
try {
await persistVideoGenerationFailedTurn({
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId || null,
messageId: params.messageId,
text: params.requestTextContent,
source: "agent_direct_generate_video",
providerType: "direct-video",
modelRef: params.modelRef,
status: "failed",
error: params.error instanceof Error ? params.error.message : String(params.error),
});
} catch (videoMemoryError) {
console.warn(
"[Direct Agent Working State] Failed to persist direct video failure:",
videoMemoryError,
);
}
}

View File

@ -8,6 +8,7 @@ import { createFalImage } from "@/lib/ai/providers/fal-image";
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
import { createMidjourney } from "@/lib/ai/providers/midjourney";
import { createRunway } from "@/lib/ai/providers/runway";
import { createKrea } from "@/lib/ai/providers/krea";
import { filterReachableRemoteUrls } from "@/lib/net/url-access";
export interface CanvasGenerateImageRequest {
@ -43,6 +44,7 @@ const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
"openai-image",
"midjourney",
"runway",
"krea",
]);
function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
@ -82,6 +84,8 @@ function getProviderBatchLimit(providerType: string) {
case "midjourney":
case "runway":
return 1;
case "krea":
return 4;
case "gemini":
return 2;
case "alibaba-image":
@ -162,7 +166,7 @@ export async function generateCanvasImage(
}
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(fallbackProvider.type)) {
throw new Error(
"当前图片模型类型暂不支持画布生成,请在后台选择 Gemini/阿里/RUXA/FAL/OpenAI/Runway 图片模型。"
"当前图片模型类型暂不支持画布生成,请在后台选择 Gemini/阿里/RUXA/FAL/OpenAI/Runway/Krea 图片模型。"
);
}
if (fallbackModel.model.id !== selectedModel.model.id) {
@ -349,9 +353,25 @@ export async function generateCanvasImage(
});
return result.text || "";
};
} else if (provider.type === "krea") {
const kreaProvider = createKrea({
apiKey: provider.apiKey,
baseURL: provider.baseUrl,
});
const modelInstance = kreaProvider.chat(selectedModel.model.modelId);
executeGenerateBatch = async (batchImageCount) => {
const result = await modelInstance.doGenerate({
messages,
imageParams: {
...imageParams,
imageCount: batchImageCount,
},
});
return result.text || "";
};
} else {
throw new Error(
`不支持的图片生成模型类型: ${provider.type || "unknown"},请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney/Runway 图片模型`
`不支持的图片生成模型类型: ${provider.type || "unknown"},请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney/Runway/Krea 图片模型`
);
}

View File

@ -36,6 +36,7 @@ const providerBaseUrlDefaults = [
{ type: "worldlabs-3d", baseUrl: "https://api.worldlabs.ai" },
{ type: "kittycad-cad", baseUrl: "https://api.zoo.dev" },
{ type: "runway", baseUrl: "https://api.dev.runwayml.com" },
{ type: "krea", baseUrl: "https://api.krea.ai" },
] as const;
function assertMigrationJournalIsConsistent() {
@ -95,7 +96,7 @@ const runMigrate = async () => {
ADD CONSTRAINT "AiProvider_type_check"
CHECK (type IN (
'alibaba', 'openai', 'custom', 'local', 'gemini',
'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'bfl-image', 'ruxa-image', 'fal-image', 'openai-image', 'midjourney', 'fal-video', 'zhipu', 'tripo-3d', 'worldlabs-3d', 'kittycad-cad', 'runway'
'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'bfl-image', 'ruxa-image', 'fal-image', 'openai-image', 'midjourney', 'fal-video', 'zhipu', 'tripo-3d', 'worldlabs-3d', 'kittycad-cad', 'runway', 'krea'
));
`);
await db.execute(sql`

View File

@ -0,0 +1,5 @@
DROP INDEX IF EXISTS "Model3DAsset_shareToken_idx";
CREATE INDEX IF NOT EXISTS "Model3DAsset_shareToken_lookup_idx"
ON "Model3DAsset" ("shareToken")
WHERE "shareToken" IS NOT NULL;

View File

@ -0,0 +1,33 @@
ALTER TABLE "AiProvider"
DROP CONSTRAINT IF EXISTS "AiProvider_type_check";
ALTER TABLE "AiProvider"
ADD CONSTRAINT "AiProvider_type_check"
CHECK (
"type" IN (
'alibaba',
'openai',
'custom',
'local',
'gemini',
'huanke-gemini',
'qianwen-video',
'doubao-video',
'ruxa-video',
'volcano-vision',
'volcano-video',
'alibaba-image',
'bfl-image',
'ruxa-image',
'fal-image',
'openai-image',
'midjourney',
'fal-video',
'zhipu',
'tripo-3d',
'worldlabs-3d',
'kittycad-cad',
'runway',
'krea'
)
);

View File

@ -400,6 +400,20 @@
"when": 1775908800000,
"tag": "0070_flow_board_cloud_files",
"breakpoints": true
},
{
"idx": 57,
"version": "7",
"when": 1775955600000,
"tag": "0071_allow_group_world_shares",
"breakpoints": true
},
{
"idx": 58,
"version": "7",
"when": 1776042000000,
"tag": "0072_add_krea_provider_type",
"breakpoints": true
}
]
}

View File

@ -3750,7 +3750,7 @@ export async function createAiProvider({
updatedBy,
}: {
name: string;
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway";
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway" | "krea";
baseUrl: string;
apiKey: string;
isEnabled?: boolean;
@ -3788,7 +3788,7 @@ export async function updateAiProvider({
}: {
id: string;
name?: string;
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway";
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway" | "krea";
baseUrl?: string;
apiKey?: string;
isEnabled?: boolean;

View File

@ -264,7 +264,7 @@ export type ToolConfig = InferSelectModel<typeof toolConfig>;
export const aiProvider = pgTable("AiProvider", {
id: uuid("id").primaryKey().notNull().defaultRandom(),
name: varchar("name", { length: 128 }).notNull(), // 提供商名称,如 "阿里云通义千问"
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "fal-video", "zhipu", "tripo-3d", "worldlabs-3d", "kittycad-cad", "runway", "bfl-image"] })
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "fal-video", "zhipu", "tripo-3d", "worldlabs-3d", "kittycad-cad", "runway", "krea", "bfl-image"] })
.notNull()
.default("custom"), // 提供商类型
baseUrl: text("baseUrl").notNull(), // API 基础 URL

File diff suppressed because one or more lines are too long