chore: sync latest app updates
This commit is contained in:
parent
1b239eb2c7
commit
dfc5f704b4
@ -345,6 +345,7 @@ type AiProvider = {
|
||||
| "worldlabs-3d"
|
||||
| "kittycad-cad"
|
||||
| "runway"
|
||||
| "krea"
|
||||
| "volcano-multimodal";
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
@ -720,6 +721,7 @@ const PROVIDER_TYPES = [
|
||||
{ value: "worldlabs-3d", label: "World Labs 3D 世界生成" },
|
||||
{ value: "kittycad-cad", label: "KittyCAD / Zoo CAD" },
|
||||
{ value: "runway", label: "Runway API" },
|
||||
{ value: "krea", label: "Krea API" },
|
||||
{ value: "zhipu", label: "智普官方 4.7" },
|
||||
{ value: "volcano-multimodal", label: "火山引擎多模态" },
|
||||
{ value: "openai", label: "OpenAI 兼容" },
|
||||
@ -749,6 +751,7 @@ const PROVIDER_DEFAULT_BASE_URL: Record<string, string> = {
|
||||
"worldlabs-3d": "https://api.worldlabs.ai",
|
||||
"kittycad-cad": "https://api.zoo.dev",
|
||||
runway: "https://api.dev.runwayml.com",
|
||||
krea: "https://api.krea.ai",
|
||||
zhipu: "https://open.bigmodel.cn/api/paas/v4",
|
||||
"volcano-multimodal": "https://ark.cn-beijing.volces.com/api/v3",
|
||||
openai: "https://api.openai.com/v1",
|
||||
@ -2377,6 +2380,7 @@ export function AdminSettingsClient() {
|
||||
| "worldlabs-3d"
|
||||
| "kittycad-cad"
|
||||
| "runway"
|
||||
| "krea"
|
||||
| "volcano-multimodal",
|
||||
baseUrl: "",
|
||||
apiKey: "",
|
||||
@ -2476,8 +2480,11 @@ export function AdminSettingsClient() {
|
||||
modelForm.imageGenConfig
|
||||
);
|
||||
const modelKindHint =
|
||||
selectedModelProviderType === "runway" && !modelForm.modelId.trim()
|
||||
? "Runway 同时支持图片与视频模型,填写模型 ID 后系统会自动识别标签。"
|
||||
(selectedModelProviderType === "runway" || selectedModelProviderType === "krea") &&
|
||||
!modelForm.modelId.trim()
|
||||
? `${
|
||||
selectedModelProviderType === "krea" ? "Krea" : "Runway"
|
||||
} 同时支持图片与视频模型,填写模型 ID 后系统会自动识别标签。`
|
||||
: isVideoProviderForModelConfig
|
||||
? "当前识别为视频模型,将使用视频能力、视频积分和视频路由。"
|
||||
: isImageProviderForModelConfig
|
||||
@ -4208,6 +4215,7 @@ export function AdminSettingsClient() {
|
||||
const method = editingProvider ? "PUT" : "POST";
|
||||
const isNewProvider = !editingProvider;
|
||||
const isVolcanoVideo = providerForm.type === "volcano-video";
|
||||
const isKreaProvider = providerForm.type === "krea";
|
||||
const isLemonProvider =
|
||||
providerForm.type === "openai" &&
|
||||
isLemonApiBaseUrl(
|
||||
@ -4280,6 +4288,28 @@ export function AdminSettingsClient() {
|
||||
}
|
||||
}
|
||||
|
||||
if (isNewProvider && isKreaProvider && providerId) {
|
||||
toast.info("正在自动添加 Krea 官方预设模型...");
|
||||
|
||||
try {
|
||||
const modelsResponse = await fetch("/api/admin/krea-models", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ providerId }),
|
||||
});
|
||||
|
||||
const modelsData = await modelsResponse.json();
|
||||
|
||||
if (modelsResponse.ok && modelsData.addedCount > 0) {
|
||||
toast.success(`已自动添加 ${modelsData.addedCount} 个 Krea 预设模型`);
|
||||
mutateModels();
|
||||
}
|
||||
} catch (modelError) {
|
||||
console.error("[Auto Add Krea Models] Error:", modelError);
|
||||
toast.warning('Krea 预设模型添加失败,请手动点击"快速添加 Krea 预设模型"按钮');
|
||||
}
|
||||
}
|
||||
|
||||
if (isNewProvider && isLemonProvider && providerId) {
|
||||
try {
|
||||
await handleQuickAddLemonModels(providerId);
|
||||
@ -4903,6 +4933,46 @@ export function AdminSettingsClient() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleQuickAddKreaModels = async (providerId: string) => {
|
||||
if (
|
||||
!confirm(
|
||||
"将为此服务商添加 Krea 官方 API 的全部图片、图片增强和视频模型预设。\n\n继续吗?"
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
toast.info("正在添加 Krea 官方预设模型...");
|
||||
|
||||
const response = await fetch("/api/admin/krea-models", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ providerId }),
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(data.error || "添加失败");
|
||||
}
|
||||
|
||||
const { addedCount, errors } = data;
|
||||
|
||||
if (addedCount > 0) {
|
||||
toast.success(`成功添加 ${addedCount} 个 Krea 预设模型`);
|
||||
mutateModels();
|
||||
}
|
||||
|
||||
if (errors && errors.length > 0) {
|
||||
toast.warning(`${errors.length} 个模型添加失败(可能已存在)`);
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error("[Quick Add Krea Models] Error:", error);
|
||||
toast.error(error.message || "添加 Krea 预设模型失败");
|
||||
}
|
||||
};
|
||||
|
||||
const handleUseLemonProviderPreset = () => {
|
||||
setProviderForm({
|
||||
id: "",
|
||||
@ -6948,6 +7018,7 @@ export function AdminSettingsClient() {
|
||||
zhipu: "bg-lime-500",
|
||||
"volcano-multimodal": "bg-fuchsia-500",
|
||||
runway: "bg-rose-500",
|
||||
krea: "bg-blue-500",
|
||||
"qianwen-video": "bg-cyan-500",
|
||||
custom: "bg-gray-500",
|
||||
local: "bg-yellow-500",
|
||||
@ -11104,6 +11175,8 @@ export function AdminSettingsClient() {
|
||||
? "留空则不修改"
|
||||
: providerForm.type === "runway"
|
||||
? "key_ 后跟 128 位十六进制字符"
|
||||
: providerForm.type === "krea"
|
||||
? "Krea API Key"
|
||||
: "sk-..."
|
||||
}
|
||||
/>
|
||||
@ -11128,6 +11201,12 @@ export function AdminSettingsClient() {
|
||||
Key:”或中文标点的内容。
|
||||
</p>
|
||||
)}
|
||||
{providerForm.type === "krea" && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Krea 使用 Bearer Token 认证。保存后会自动添加官方
|
||||
OpenAPI 中的图片、图片增强和视频模型预设。
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Switch
|
||||
@ -11192,6 +11271,17 @@ export function AdminSettingsClient() {
|
||||
快速添加预设模型
|
||||
</Button>
|
||||
)}
|
||||
{provider.type === "krea" && (
|
||||
<Button
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
className="mt-2"
|
||||
onClick={() => handleQuickAddKreaModels(provider.id)}
|
||||
>
|
||||
<Sparkles className="h-4 w-4 mr-1" />
|
||||
快速添加 Krea 预设模型
|
||||
</Button>
|
||||
)}
|
||||
{provider.type === "openai" &&
|
||||
isLemonApiBaseUrl(provider.baseUrl) && (
|
||||
<Button
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -33,6 +33,7 @@ const providerTypes = [
|
||||
"worldlabs-3d",
|
||||
"kittycad-cad",
|
||||
"runway",
|
||||
"krea",
|
||||
"volcano-multimodal",
|
||||
] as const;
|
||||
type ProviderType = (typeof providerTypes)[number];
|
||||
@ -67,6 +68,7 @@ const TRIPO_3D_BASE_URL = "https://api.tripo3d.ai/v2/openapi";
|
||||
const WORLDLABS_3D_BASE_URL = "https://api.worldlabs.ai";
|
||||
const KITTYCAD_CAD_BASE_URL = "https://api.zoo.dev";
|
||||
const RUNWAY_BASE_URL = "https://api.dev.runwayml.com";
|
||||
const KREA_BASE_URL = "https://api.krea.ai";
|
||||
const RUNWAY_API_KEY_PATTERN = /^key_[0-9a-f]{128}$/;
|
||||
|
||||
function normalizeProviderTypeForStorage(
|
||||
@ -218,6 +220,8 @@ export async function POST(request: Request) {
|
||||
? data.baseUrl || KITTYCAD_CAD_BASE_URL
|
||||
: data.type === "runway"
|
||||
? data.baseUrl || RUNWAY_BASE_URL
|
||||
: data.type === "krea"
|
||||
? data.baseUrl || KREA_BASE_URL
|
||||
: data.baseUrl,
|
||||
updatedBy: session.user.id,
|
||||
});
|
||||
|
||||
118
app/api/admin/krea-models/route.ts
Normal file
118
app/api/admin/krea-models/route.ts
Normal file
@ -0,0 +1,118 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { requireAdmin } from "@/lib/middleware/admin";
|
||||
import { db } from "@/lib/db";
|
||||
import { aiModel } from "@/lib/db/schema";
|
||||
import { getAiProviderById } from "@/lib/db/queries";
|
||||
import { KREA_MODEL_PRESETS } from "@/lib/ai/krea-models";
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
await requireAdmin();
|
||||
return NextResponse.json({
|
||||
presets: KREA_MODEL_PRESETS,
|
||||
counts: {
|
||||
image: KREA_MODEL_PRESETS.filter((preset) => preset.kind === "image").length,
|
||||
imageEnhance: KREA_MODEL_PRESETS.filter((preset) => preset.kind === "image-enhance").length,
|
||||
video: KREA_MODEL_PRESETS.filter((preset) => preset.kind === "video").length,
|
||||
total: KREA_MODEL_PRESETS.length,
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error("[Krea Models API] GET error:", error);
|
||||
return NextResponse.json(
|
||||
{ error: error.message || "Failed to fetch Krea model presets" },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
await requireAdmin();
|
||||
|
||||
const body = await request.json();
|
||||
const providerId = String(body?.providerId || "").trim();
|
||||
const modelIds = Array.isArray(body?.modelIds)
|
||||
? body.modelIds.map((item: unknown) => String(item || "").trim()).filter(Boolean)
|
||||
: null;
|
||||
|
||||
if (!providerId) {
|
||||
return NextResponse.json({ error: "Provider ID is required" }, { status: 400 });
|
||||
}
|
||||
|
||||
const provider = await getAiProviderById({ id: providerId });
|
||||
if (!provider || provider.type !== "krea") {
|
||||
return NextResponse.json(
|
||||
{ error: "Provider must be a Krea provider" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const presetsToAdd = modelIds
|
||||
? KREA_MODEL_PRESETS.filter((preset) => modelIds.includes(preset.modelId))
|
||||
: KREA_MODEL_PRESETS;
|
||||
|
||||
const addedModels = [];
|
||||
const errors = [];
|
||||
|
||||
for (const preset of presetsToAdd) {
|
||||
try {
|
||||
const existing = await db.query.aiModel.findFirst({
|
||||
where: and(
|
||||
eq(aiModel.providerId, providerId),
|
||||
eq(aiModel.modelId, preset.modelId)
|
||||
),
|
||||
});
|
||||
|
||||
if (existing) {
|
||||
errors.push({ modelId: preset.modelId, error: "Model already exists" });
|
||||
continue;
|
||||
}
|
||||
|
||||
const [newModel] = await db
|
||||
.insert(aiModel)
|
||||
.values({
|
||||
providerId,
|
||||
modelId: preset.modelId,
|
||||
displayName: preset.displayName,
|
||||
description: preset.description,
|
||||
supportsVision: preset.supportsVision,
|
||||
supportsImageGen: preset.supportsImageGen,
|
||||
isEnabled: true,
|
||||
isHidden: false,
|
||||
isDefault: false,
|
||||
sortOrder: preset.sortOrder,
|
||||
capabilities: preset.capabilities || null,
|
||||
defaultParams: preset.defaultParams || null,
|
||||
imageGenConfig: preset.imageGenConfig || null,
|
||||
isSubModel: false,
|
||||
parentModelId: null,
|
||||
})
|
||||
.returning();
|
||||
|
||||
addedModels.push(newModel);
|
||||
} catch (error: any) {
|
||||
console.error(`[Krea Models API] Failed to add ${preset.modelId}:`, error);
|
||||
errors.push({ modelId: preset.modelId, error: error.message || "Insert failed" });
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
addedCount: addedModels.length,
|
||||
addedModels,
|
||||
errors: errors.length > 0 ? errors : undefined,
|
||||
counts: {
|
||||
requested: presetsToAdd.length,
|
||||
total: KREA_MODEL_PRESETS.length,
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error("[Krea Models API] POST error:", error);
|
||||
return NextResponse.json(
|
||||
{ error: error.message || "Failed to add Krea models" },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
import { and, desc, eq } from "drizzle-orm";
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
@ -362,14 +362,30 @@ export async function PATCH(
|
||||
}
|
||||
|
||||
const body = await request.json().catch(() => ({}));
|
||||
const taskId = String((body as any)?.taskId || "").trim();
|
||||
if (!taskId) {
|
||||
const rawTaskIds = Array.isArray((body as any)?.taskIds)
|
||||
? (body as any).taskIds
|
||||
: [(body as any)?.taskId];
|
||||
const taskIds = Array.from(
|
||||
new Set<string>(
|
||||
rawTaskIds
|
||||
.map((value: unknown) => String(value || "").trim())
|
||||
.filter(Boolean)
|
||||
)
|
||||
);
|
||||
if (taskIds.length === 0) {
|
||||
return NextResponse.json({ success: false, error: "缺少 taskId" }, { status: 400 });
|
||||
}
|
||||
if (taskIds.length > 30) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "一次最多分享 30 个 3D 世界" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const [existingAsset] = await db
|
||||
const existingAssets = await db
|
||||
.select({
|
||||
id: model3dAsset.id,
|
||||
taskId: model3dAsset.taskId,
|
||||
provider: model3dAsset.provider,
|
||||
status: model3dAsset.status,
|
||||
shareEnabled: model3dAsset.shareEnabled,
|
||||
@ -380,23 +396,22 @@ export async function PATCH(
|
||||
and(
|
||||
eq(model3dAsset.projectId, projectId),
|
||||
eq(model3dAsset.userId, session.user.id),
|
||||
eq(model3dAsset.taskId, taskId)
|
||||
inArray(model3dAsset.taskId, taskIds)
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
);
|
||||
|
||||
if (!existingAsset) {
|
||||
if (existingAssets.length !== taskIds.length) {
|
||||
return NextResponse.json({ success: false, error: "资产不存在" }, { status: 404 });
|
||||
}
|
||||
|
||||
if (existingAsset.provider !== "worldlabs-3d") {
|
||||
if (existingAssets.some((asset) => asset.provider !== "worldlabs-3d")) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "当前仅支持分享 3D 世界资产" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
if (existingAsset.status !== "success") {
|
||||
if (existingAssets.some((asset) => asset.status !== "success")) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "仅成功生成的 3D 世界支持分享" },
|
||||
{ status: 400 }
|
||||
@ -404,10 +419,19 @@ export async function PATCH(
|
||||
}
|
||||
|
||||
const shouldEnable = (body as any)?.shareEnabled !== false;
|
||||
const existingShareToken = existingAssets.find(
|
||||
(asset) => asset.shareEnabled && asset.shareToken
|
||||
)?.shareToken;
|
||||
const shareToken =
|
||||
shouldEnable
|
||||
? taskIds.length > 1
|
||||
? existingShareToken || createShareToken()
|
||||
: existingAssets[0]?.shareToken || createShareToken()
|
||||
: null;
|
||||
const nextValues = shouldEnable
|
||||
? {
|
||||
shareEnabled: true,
|
||||
shareToken: existingAsset.shareToken || createShareToken(),
|
||||
shareToken,
|
||||
sharedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
}
|
||||
@ -418,13 +442,24 @@ export async function PATCH(
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
|
||||
const [updatedAsset] = await db
|
||||
const updatedAssets = await db
|
||||
.update(model3dAsset)
|
||||
.set(nextValues)
|
||||
.where(eq(model3dAsset.id, existingAsset.id))
|
||||
.where(inArray(model3dAsset.id, existingAssets.map((asset) => asset.id)))
|
||||
.returning();
|
||||
const updatedAssetByTaskId = new Map(
|
||||
updatedAssets.map((asset) => [asset.taskId, asset])
|
||||
);
|
||||
const orderedUpdatedAssets = taskIds
|
||||
.map((id) => updatedAssetByTaskId.get(id))
|
||||
.filter((asset): asset is (typeof updatedAssets)[number] => Boolean(asset));
|
||||
|
||||
return NextResponse.json({ success: true, data: updatedAsset });
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: orderedUpdatedAssets[0] || updatedAssets[0] || null,
|
||||
items: orderedUpdatedAssets,
|
||||
shareToken,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Model3DAssets] PATCH failed:", error);
|
||||
return NextResponse.json({ success: false, error: "更新 3D 世界分享状态失败" }, { status: 500 });
|
||||
|
||||
@ -105,6 +105,7 @@ const MEMBERSHIP_CONFIG_KEY = "membership_config";
|
||||
const MEMBERSHIP_USER_OVERRIDES_KEY = "membership_user_plan_overrides";
|
||||
const STORAGE_FAILOVER_CONFIG_KEY = "storage_failover_config";
|
||||
const REALTIME_CANVAS_CONFIG_KEY = "realtime_canvas_config";
|
||||
const LAYER_SPLIT_CONFIG_KEY = "layer_split_config";
|
||||
const AGENT_CATEGORY_TREE_CONFIG_KEY = "agent_category_tree_config";
|
||||
const CANVAS_AGENT_MODE_ENABLED_KEY = "canvas_agent_mode_enabled";
|
||||
const VIDEO_CANVAS_SIDEBAR_ENABLED_KEY = "video_canvas_sidebar_enabled";
|
||||
@ -230,6 +231,13 @@ function normalizeVideoCanvasSidebarEnabled(value: unknown): boolean {
|
||||
return value !== false;
|
||||
}
|
||||
|
||||
function normalizeLayerSplitEnabled(value: unknown): boolean {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return false;
|
||||
}
|
||||
return (value as { enabled?: unknown }).enabled === true;
|
||||
}
|
||||
|
||||
function normalizeAgentCategoryTreeConfig(value: unknown): AgentCategoryTreeConfig {
|
||||
const raw =
|
||||
value && typeof value === "object" && !Array.isArray(value)
|
||||
@ -278,6 +286,7 @@ export async function GET() {
|
||||
pointsPackagesSetting,
|
||||
dulupayConfigSetting,
|
||||
realtimeCanvasConfigSetting,
|
||||
layerSplitConfigSetting,
|
||||
canvasAgentModeEnabledSetting,
|
||||
videoCanvasSidebarEnabledSetting,
|
||||
storageFailoverConfigSetting,
|
||||
@ -380,6 +389,12 @@ export async function GET() {
|
||||
.where(eq(systemSettings.key, REALTIME_CANVAS_CONFIG_KEY))
|
||||
.limit(1)
|
||||
.then(rows => rows[0]),
|
||||
db
|
||||
.select()
|
||||
.from(systemSettings)
|
||||
.where(eq(systemSettings.key, LAYER_SPLIT_CONFIG_KEY))
|
||||
.limit(1)
|
||||
.then(rows => rows[0]),
|
||||
db
|
||||
.select()
|
||||
.from(systemSettings)
|
||||
@ -500,6 +515,9 @@ export async function GET() {
|
||||
: normalizeDulupayForPublic(dulupayConfigSetting?.value);
|
||||
const realtimeCanvasServiceEnabled =
|
||||
(realtimeCanvasConfigSetting?.value as { enabled?: boolean } | undefined)?.enabled !== false;
|
||||
const layerSplitEnabled = normalizeLayerSplitEnabled(
|
||||
layerSplitConfigSetting?.value
|
||||
);
|
||||
const canvasAgentModeEnabled = normalizeCanvasAgentModeEnabled(
|
||||
canvasAgentModeEnabledSetting?.value
|
||||
);
|
||||
@ -600,6 +618,7 @@ export async function GET() {
|
||||
defaultAgentPsdTemplateCategoryTagIndex,
|
||||
realtimeCanvasEnabled: true,
|
||||
realtimeCanvasServiceEnabled: true,
|
||||
layerSplitEnabled,
|
||||
storageFailoverConfig,
|
||||
ossImageMigrationConfig,
|
||||
canvasDefaultModelId: null,
|
||||
@ -658,6 +677,7 @@ export async function GET() {
|
||||
pointsPackages,
|
||||
dulupayConfig,
|
||||
realtimeCanvasServiceEnabled,
|
||||
layerSplitEnabled,
|
||||
canvasAgentModeEnabled,
|
||||
videoCanvasSidebarEnabled,
|
||||
storageFailoverConfig,
|
||||
|
||||
@ -7,6 +7,7 @@ import {
|
||||
Bone,
|
||||
Box,
|
||||
Boxes,
|
||||
Check,
|
||||
ChevronDown,
|
||||
ChevronUp,
|
||||
Download,
|
||||
@ -618,6 +619,7 @@ export default function CanvasModel3DPage() {
|
||||
|
||||
const [taskHistory, setTaskHistory] = useState<TaskRecord[]>([]);
|
||||
const [currentTask, setCurrentTask] = useState<TaskRecord | null>(null);
|
||||
const [selectedWorldShareTaskIds, setSelectedWorldShareTaskIds] = useState<string[]>([]);
|
||||
const [autoDownloadTaskId, setAutoDownloadTaskId] = useState<string | null>(
|
||||
null
|
||||
);
|
||||
@ -1032,6 +1034,25 @@ export default function CanvasModel3DPage() {
|
||||
() => taskHistory.filter((task) => taskMatchesWorkspace(task, workspaceMode)),
|
||||
[taskHistory, workspaceMode]
|
||||
);
|
||||
const shareableWorldTasks = useMemo(
|
||||
() =>
|
||||
visibleTaskHistory.filter(
|
||||
(task) => task.provider === "worldlabs-3d" && task.status === "success"
|
||||
),
|
||||
[visibleTaskHistory]
|
||||
);
|
||||
const selectedWorldShareTasks = useMemo(() => {
|
||||
const selectedIds = new Set(selectedWorldShareTaskIds);
|
||||
return shareableWorldTasks.filter((task) => selectedIds.has(task.taskId));
|
||||
}, [selectedWorldShareTaskIds, shareableWorldTasks]);
|
||||
|
||||
useEffect(() => {
|
||||
const shareableIds = new Set(shareableWorldTasks.map((task) => task.taskId));
|
||||
setSelectedWorldShareTaskIds((prev) => {
|
||||
const next = prev.filter((taskId) => shareableIds.has(taskId));
|
||||
return next.length === prev.length ? prev : next;
|
||||
});
|
||||
}, [shareableWorldTasks]);
|
||||
|
||||
const currentModelUrl = resolveTaskModelUrl(currentTask);
|
||||
const currentPreviewImage =
|
||||
@ -1920,6 +1941,75 @@ export default function CanvasModel3DPage() {
|
||||
[updateWorldShareState]
|
||||
);
|
||||
|
||||
const handleToggleWorldShareSelection = useCallback(
|
||||
(task: TaskRecord, event?: MouseEvent<HTMLButtonElement>) => {
|
||||
event?.stopPropagation();
|
||||
if (task.provider !== "worldlabs-3d" || task.status !== "success") {
|
||||
return;
|
||||
}
|
||||
setSelectedWorldShareTaskIds((prev) =>
|
||||
prev.includes(task.taskId)
|
||||
? prev.filter((taskId) => taskId !== task.taskId)
|
||||
: [...prev, task.taskId]
|
||||
);
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
const handleSelectAllShareableWorlds = useCallback(() => {
|
||||
setSelectedWorldShareTaskIds((prev) =>
|
||||
prev.length === shareableWorldTasks.length
|
||||
? []
|
||||
: shareableWorldTasks.map((task) => task.taskId)
|
||||
);
|
||||
}, [shareableWorldTasks]);
|
||||
|
||||
const handleShareSelectedWorlds = useCallback(async () => {
|
||||
if (selectedWorldShareTasks.length < 2) {
|
||||
toast.error("请至少选择 2 个成功生成的 3D 世界");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`/api/canvas/${id}/model3d-assets`, {
|
||||
method: "PATCH",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
taskIds: selectedWorldShareTasks.map((task) => task.taskId),
|
||||
shareEnabled: true,
|
||||
}),
|
||||
});
|
||||
const payload = (await response.json().catch(() => ({}))) as {
|
||||
success?: boolean;
|
||||
error?: string;
|
||||
items?: TaskRecordApiEntity[];
|
||||
shareToken?: string | null;
|
||||
};
|
||||
if (!response.ok || payload?.success === false) {
|
||||
throw new Error(payload?.error || "批量分享世界失败");
|
||||
}
|
||||
|
||||
const updatedTasks = Array.isArray(payload.items)
|
||||
? payload.items
|
||||
.map((item) => mapApiEntityToTaskRecord(item))
|
||||
.filter((item): item is TaskRecord => Boolean(item))
|
||||
: [];
|
||||
for (const updatedTask of updatedTasks) {
|
||||
upsertTaskHistory(updatedTask, { persist: false });
|
||||
}
|
||||
|
||||
const shareToken = payload.shareToken || updatedTasks[0]?.shareToken;
|
||||
const shareUrl = buildWorldShareUrl(shareToken);
|
||||
if (!shareUrl) {
|
||||
throw new Error("分享链接生成失败");
|
||||
}
|
||||
await navigator.clipboard.writeText(shareUrl);
|
||||
toast.success(`已复制包含 ${selectedWorldShareTasks.length} 个世界的分享链接`);
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : "批量分享世界失败");
|
||||
}
|
||||
}, [id, selectedWorldShareTasks, upsertTaskHistory]);
|
||||
|
||||
const handleDisableCurrentShare = useCallback(async () => {
|
||||
if (!currentTask || currentTask.provider !== "worldlabs-3d") {
|
||||
toast.error("当前没有可关闭分享的世界资产");
|
||||
@ -3570,14 +3660,52 @@ export default function CanvasModel3DPage() {
|
||||
<X className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
{workspaceMode === "world" ? (
|
||||
<div className="space-y-2 border-white/10 border-b px-3 py-3">
|
||||
<div className="flex items-center justify-between gap-2 text-[11px] text-zinc-500">
|
||||
<span>多世界分享</span>
|
||||
<span>已选 {selectedWorldShareTasks.length}</span>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<Button
|
||||
className="h-8 border-white/10 bg-zinc-950/60 text-xs text-zinc-200 hover:bg-zinc-800"
|
||||
disabled={shareableWorldTasks.length === 0}
|
||||
onClick={handleSelectAllShareableWorlds}
|
||||
size="sm"
|
||||
variant="outline"
|
||||
>
|
||||
{selectedWorldShareTaskIds.length === shareableWorldTasks.length &&
|
||||
shareableWorldTasks.length > 0
|
||||
? "取消全选"
|
||||
: "全选世界"}
|
||||
</Button>
|
||||
<Button
|
||||
className="h-8 border-cyan-400/30 bg-cyan-400/15 text-xs text-cyan-100 hover:bg-cyan-400/25"
|
||||
disabled={selectedWorldShareTasks.length < 2}
|
||||
onClick={handleShareSelectedWorlds}
|
||||
size="sm"
|
||||
variant="outline"
|
||||
>
|
||||
分享所选
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
<ScrollArea className="flex-1">
|
||||
<div className="grid grid-cols-2 gap-2 p-3">
|
||||
{visibleTaskHistory.map((task) => (
|
||||
{visibleTaskHistory.map((task) => {
|
||||
const canShareWorld =
|
||||
task.provider === "worldlabs-3d" && task.status === "success";
|
||||
const isSelectedForShare = selectedWorldShareTaskIds.includes(task.taskId);
|
||||
|
||||
return (
|
||||
<button
|
||||
className={cn(
|
||||
"group relative aspect-square overflow-hidden rounded-lg border border-white/5 bg-zinc-950 transition-all hover:border-yellow-500/50",
|
||||
currentTask?.taskId === task.taskId &&
|
||||
"border-yellow-500 ring-2 ring-yellow-500/30 shadow-[0_18px_40px_rgba(0,0,0,0.28)]"
|
||||
"border-yellow-500 ring-2 ring-yellow-500/30 shadow-[0_18px_40px_rgba(0,0,0,0.28)]",
|
||||
isSelectedForShare &&
|
||||
"border-cyan-300 ring-2 ring-cyan-300/40"
|
||||
)}
|
||||
key={task.taskId}
|
||||
onClick={() => handleSelectTask(task)}
|
||||
@ -3588,7 +3716,7 @@ export default function CanvasModel3DPage() {
|
||||
当前
|
||||
</Badge>
|
||||
) : null}
|
||||
{task.provider === "worldlabs-3d" && task.status === "success" ? (
|
||||
{canShareWorld ? (
|
||||
<Button
|
||||
className={cn(
|
||||
"absolute left-2 top-2 z-10 h-7 w-7 rounded-full border border-white/10 bg-black/60 text-zinc-100 opacity-0 shadow-lg backdrop-blur-sm transition-opacity hover:bg-black/80 group-hover:opacity-100",
|
||||
@ -3603,6 +3731,24 @@ export default function CanvasModel3DPage() {
|
||||
<Link2 className="h-3.5 w-3.5" />
|
||||
</Button>
|
||||
) : null}
|
||||
{canShareWorld ? (
|
||||
<Button
|
||||
className={cn(
|
||||
"absolute left-10 top-2 z-10 h-7 rounded-full border border-white/10 bg-black/60 px-2 text-[10px] text-zinc-100 opacity-0 shadow-lg backdrop-blur-sm transition-opacity hover:bg-black/80 group-hover:opacity-100",
|
||||
isSelectedForShare &&
|
||||
"border-cyan-300/60 bg-cyan-400/25 text-cyan-50 opacity-100"
|
||||
)}
|
||||
onClick={(event) => handleToggleWorldShareSelection(task, event)}
|
||||
size="sm"
|
||||
type="button"
|
||||
variant="ghost"
|
||||
>
|
||||
{isSelectedForShare ? (
|
||||
<Check className="mr-1 h-3 w-3" />
|
||||
) : null}
|
||||
{isSelectedForShare ? "已选" : "选择"}
|
||||
</Button>
|
||||
) : null}
|
||||
{task.status === "success" && resolveTaskModelUrl(task) ? (
|
||||
<Button
|
||||
className="absolute right-2 top-2 z-10 h-7 w-7 rounded-full border border-white/10 bg-black/60 text-zinc-100 opacity-0 shadow-lg backdrop-blur-sm transition-opacity hover:bg-black/80 group-hover:opacity-100"
|
||||
@ -3663,7 +3809,8 @@ export default function CanvasModel3DPage() {
|
||||
) : null}
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
);
|
||||
})}
|
||||
{visibleTaskHistory.length === 0 && (
|
||||
<div className="col-span-2 py-10 text-center text-xs text-zinc-500">
|
||||
{workspaceMode === "world" ? "暂无世界记录" : "暂无模型记录"}
|
||||
|
||||
@ -11,7 +11,6 @@ import {
|
||||
ChevronRight,
|
||||
Clapperboard,
|
||||
CheckCircle2,
|
||||
Crosshair,
|
||||
Download,
|
||||
Eye,
|
||||
FileImage,
|
||||
@ -1402,10 +1401,6 @@ export default function CanvasEditorPage({ params }: PageProps) {
|
||||
const updateNode = useCanvasStore((state) => state.updateNode);
|
||||
const pushHistory = useCanvasStore((state) => state.pushHistory);
|
||||
const setGenerating = useCanvasStore((state) => state.setGenerating);
|
||||
const setPickerMode = useCanvasStore((state) => state.setPickerMode);
|
||||
const requestImageNodeAction = useCanvasStore(
|
||||
(state) => state.requestImageNodeAction
|
||||
);
|
||||
|
||||
// Local state
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
@ -1537,6 +1532,7 @@ export default function CanvasEditorPage({ params }: PageProps) {
|
||||
agentEnabled?: boolean;
|
||||
canvasAgentModeEnabled?: boolean;
|
||||
videoCanvasSidebarEnabled?: boolean;
|
||||
layerSplitEnabled?: boolean;
|
||||
realtimeCanvasEnabled?: boolean;
|
||||
realtimeCanvasServiceEnabled?: boolean;
|
||||
agentWorkflowShowcaseConfig?: AgentWorkflowShowcaseConfig;
|
||||
@ -8616,6 +8612,7 @@ export default function CanvasEditorPage({ params }: PageProps) {
|
||||
showEmptyState={!isVideoMode}
|
||||
workspaceLabel={workspaceLabel}
|
||||
workspaceVariant={isVideoMode ? "video" : "canvas"}
|
||||
layerSplitEnabled={siteSettingsData?.layerSplitEnabled === true}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@ -9736,61 +9733,6 @@ export default function CanvasEditorPage({ params }: PageProps) {
|
||||
</DropdownMenu>
|
||||
) : null}
|
||||
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
className={cn(
|
||||
isVideoMode
|
||||
? VIDEO_CANVAS_TOOLBAR_BUTTON_CLASS
|
||||
: "h-9 w-9 rounded-xl"
|
||||
)}
|
||||
disabled={!selectedImageNode}
|
||||
onClick={() => {
|
||||
if (!selectedImageNode) {
|
||||
toast.info("请先选中一张图片");
|
||||
return;
|
||||
}
|
||||
setPickerMode(true, selectedImageNode.id);
|
||||
toast.info("点击图片中的物品进行拾取");
|
||||
}}
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
>
|
||||
<Crosshair className="h-4 w-4" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top">
|
||||
{selectedImageNode ? "识物" : "先选中图片后识物"}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
className={cn(
|
||||
isVideoMode
|
||||
? VIDEO_CANVAS_TOOLBAR_BUTTON_CLASS
|
||||
: "h-9 w-9 rounded-xl"
|
||||
)}
|
||||
disabled={!selectedImageNode}
|
||||
onClick={() => {
|
||||
if (!selectedImageNode) {
|
||||
toast.info("请先选中一张图片");
|
||||
return;
|
||||
}
|
||||
requestImageNodeAction(selectedImageNode.id, "box-cutout");
|
||||
}}
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
>
|
||||
<Scan className="h-4 w-4" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top">
|
||||
{selectedImageNode ? "框选抠图" : "先选中图片后框选抠图"}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
|
||||
{siteSettingsData?.realtimeCanvasEnabled === true &&
|
||||
siteSettingsData?.realtimeCanvasServiceEnabled !== false ? (
|
||||
<Tooltip>
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { and, desc, eq } from "drizzle-orm";
|
||||
import { notFound } from "next/navigation";
|
||||
import { WorldSharePage } from "@/components/canvas/world3d/WorldSharePage";
|
||||
import { db } from "@/lib/db";
|
||||
@ -46,7 +46,7 @@ export default async function SharedWorldPage({
|
||||
notFound();
|
||||
}
|
||||
|
||||
const [asset] = await db
|
||||
const assets = await db
|
||||
.select({
|
||||
taskId: model3dAsset.taskId,
|
||||
prompt: model3dAsset.prompt,
|
||||
@ -54,6 +54,8 @@ export default async function SharedWorldPage({
|
||||
previewImageUrl: model3dAsset.previewImageUrl,
|
||||
modelUrl: model3dAsset.modelUrl,
|
||||
assetPayload: model3dAsset.assetPayload,
|
||||
sharedAt: model3dAsset.sharedAt,
|
||||
updatedAt: model3dAsset.updatedAt,
|
||||
})
|
||||
.from(model3dAsset)
|
||||
.where(
|
||||
@ -64,28 +66,47 @@ export default async function SharedWorldPage({
|
||||
eq(model3dAsset.status, "success")
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
.orderBy(desc(model3dAsset.sharedAt), desc(model3dAsset.updatedAt))
|
||||
.limit(30);
|
||||
|
||||
if (!asset) {
|
||||
if (assets.length === 0) {
|
||||
notFound();
|
||||
}
|
||||
|
||||
const assetPayload = readAssetPayload(asset.assetPayload);
|
||||
const sharePath = `/worlds/share/${shareToken}`;
|
||||
const worlds = assets.map((asset) => {
|
||||
const assetPayload = readAssetPayload(asset.assetPayload);
|
||||
|
||||
return {
|
||||
id: asset.taskId,
|
||||
caption: assetPayload?.caption || null,
|
||||
imageUrl: assetPayload?.panoUrl || assetPayload?.thumbnailUrl || asset.previewImageUrl || null,
|
||||
meshUrl: asset.modelUrl || assetPayload?.meshUrl || null,
|
||||
model: assetPayload?.model || asset.modelVersion || null,
|
||||
previewImageUrl: asset.previewImageUrl || assetPayload?.thumbnailUrl || null,
|
||||
prompt: asset.prompt || null,
|
||||
semanticsMetadata: assetPayload?.semanticsMetadata || null,
|
||||
splatUrl: pickBestWorldLabsSplatUrl(assetPayload?.spzUrls),
|
||||
title: assetPayload?.displayName || asset.prompt || `World ${asset.taskId.slice(0, 8)}`,
|
||||
worldUrl: assetPayload?.worldUrl || null,
|
||||
};
|
||||
});
|
||||
const primaryWorld = worlds[0];
|
||||
|
||||
return (
|
||||
<WorldSharePage
|
||||
caption={assetPayload?.caption || null}
|
||||
imageUrl={assetPayload?.panoUrl || assetPayload?.thumbnailUrl || asset.previewImageUrl || null}
|
||||
meshUrl={asset.modelUrl || assetPayload?.meshUrl || null}
|
||||
model={assetPayload?.model || asset.modelVersion || null}
|
||||
previewImageUrl={asset.previewImageUrl || assetPayload?.thumbnailUrl || null}
|
||||
prompt={asset.prompt || null}
|
||||
semanticsMetadata={assetPayload?.semanticsMetadata || null}
|
||||
caption={primaryWorld?.caption || null}
|
||||
imageUrl={primaryWorld?.imageUrl || null}
|
||||
meshUrl={primaryWorld?.meshUrl || null}
|
||||
model={primaryWorld?.model || null}
|
||||
previewImageUrl={primaryWorld?.previewImageUrl || null}
|
||||
prompt={primaryWorld?.prompt || null}
|
||||
semanticsMetadata={primaryWorld?.semanticsMetadata || null}
|
||||
shareUrl={sharePath}
|
||||
splatUrl={pickBestWorldLabsSplatUrl(assetPayload?.spzUrls)}
|
||||
title={assetPayload?.displayName || asset.prompt || `World ${asset.taskId.slice(0, 8)}`}
|
||||
worldUrl={assetPayload?.worldUrl || null}
|
||||
splatUrl={primaryWorld?.splatUrl || null}
|
||||
title={primaryWorld?.title || "3D 世界分享"}
|
||||
worldUrl={primaryWorld?.worldUrl || null}
|
||||
worlds={worlds}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
@ -25,6 +25,10 @@ import {
|
||||
} from "@/components/ui/select";
|
||||
import { ScrollArea } from "@/components/ui/scroll-area";
|
||||
import { cn, fetcher } from "@/lib/utils";
|
||||
import {
|
||||
isImageGenerationModelDescriptor,
|
||||
isVideoModelDescriptor,
|
||||
} from "@/lib/ai/model-classification";
|
||||
import {
|
||||
defaultAgentPresetModeConfig,
|
||||
normalizeAgentPresetModeConfig,
|
||||
@ -137,8 +141,10 @@ export function AgentPresetDialog({
|
||||
models?: Array<{
|
||||
id: string;
|
||||
displayName: string;
|
||||
modelId?: string;
|
||||
supportsImageGen?: boolean;
|
||||
providerType?: string;
|
||||
capabilities?: unknown;
|
||||
isMultimodal?: boolean;
|
||||
}>;
|
||||
}>("/api/models", fetcher);
|
||||
@ -239,27 +245,46 @@ export function AgentPresetDialog({
|
||||
[presetLevel3Options, selectedPresetLevel3Id]
|
||||
);
|
||||
|
||||
const videoProviderTypes = useMemo(
|
||||
() => new Set(["volcano-video", "doubao-video", "qianwen-video", "fal-video", "ruxa-video"]),
|
||||
[]
|
||||
);
|
||||
const filteredModels = useMemo(() => {
|
||||
const models = modelsData?.models || [];
|
||||
if (type === "video") {
|
||||
return models.filter((model) => videoProviderTypes.has(String(model.providerType || "")));
|
||||
return models.filter((model) =>
|
||||
isVideoModelDescriptor({
|
||||
providerType: model.providerType,
|
||||
modelId: model.modelId,
|
||||
capabilities: model.capabilities,
|
||||
supportsImageGen: model.supportsImageGen,
|
||||
})
|
||||
);
|
||||
}
|
||||
if (type === "text") {
|
||||
return models.filter(
|
||||
(model) =>
|
||||
model.isMultimodal === true &&
|
||||
!videoProviderTypes.has(String(model.providerType || ""))
|
||||
!isVideoModelDescriptor({
|
||||
providerType: model.providerType,
|
||||
modelId: model.modelId,
|
||||
capabilities: model.capabilities,
|
||||
supportsImageGen: model.supportsImageGen,
|
||||
})
|
||||
);
|
||||
}
|
||||
return models.filter(
|
||||
(model) =>
|
||||
model.supportsImageGen && !videoProviderTypes.has(String(model.providerType || ""))
|
||||
isImageGenerationModelDescriptor({
|
||||
providerType: model.providerType,
|
||||
modelId: model.modelId,
|
||||
capabilities: model.capabilities,
|
||||
supportsImageGen: model.supportsImageGen,
|
||||
}) &&
|
||||
!isVideoModelDescriptor({
|
||||
providerType: model.providerType,
|
||||
modelId: model.modelId,
|
||||
capabilities: model.capabilities,
|
||||
supportsImageGen: model.supportsImageGen,
|
||||
})
|
||||
);
|
||||
}, [modelsData?.models, type, videoProviderTypes]);
|
||||
}, [modelsData?.models, type]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
|
||||
@ -182,6 +182,7 @@ interface CanvasWorkspaceProps {
|
||||
workspaceLabel?: string;
|
||||
emptyStateDescription?: string;
|
||||
workspaceVariant?: "canvas" | "video";
|
||||
layerSplitEnabled?: boolean;
|
||||
showEmptyState?: boolean;
|
||||
onBlankCanvasDoubleClick?: (payload: {
|
||||
screenPosition: Position;
|
||||
@ -194,6 +195,7 @@ export function CanvasWorkspace({
|
||||
workspaceLabel = "画布",
|
||||
emptyStateDescription = '点击顶部"添加"按钮开始创作',
|
||||
workspaceVariant = "canvas",
|
||||
layerSplitEnabled = false,
|
||||
showEmptyState = true,
|
||||
onBlankCanvasDoubleClick,
|
||||
}: CanvasWorkspaceProps) {
|
||||
@ -1488,7 +1490,11 @@ export function CanvasWorkspace({
|
||||
) : null}
|
||||
{isWorkflowCanvas ? <VideoCanvasNodeMarkers node={node} /> : null}
|
||||
{node.type === 'image' && (
|
||||
<ImageNode node={node} isSelected={isSelected} />
|
||||
<ImageNode
|
||||
isLayerSplitEnabled={layerSplitEnabled}
|
||||
isSelected={isSelected}
|
||||
node={node}
|
||||
/>
|
||||
)}
|
||||
{node.type === 'text' && (
|
||||
<TextNode node={node} isSelected={isSelected} />
|
||||
|
||||
@ -847,6 +847,7 @@ export function RealtimeCanvasDialog({
|
||||
const [refreshToken, setRefreshToken] = useState(0);
|
||||
const [siteName, setSiteName] = useState("AI Chatbot");
|
||||
const [siteLogo, setSiteLogo] = useState<string | null>(null);
|
||||
const [isLayerSplitEnabled, setIsLayerSplitEnabled] = useState(false);
|
||||
|
||||
const statusBadge = useMemo(() => {
|
||||
if (status === "live")
|
||||
@ -908,6 +909,7 @@ export function RealtimeCanvasDialog({
|
||||
) {
|
||||
setSiteLogo(data.siteLogoDark.trim());
|
||||
}
|
||||
setIsLayerSplitEnabled(data.layerSplitEnabled === true);
|
||||
})
|
||||
.catch((error) => {
|
||||
console.warn("[RealtimeCanvas] Failed to load site settings:", error);
|
||||
@ -1731,6 +1733,9 @@ export function RealtimeCanvasDialog({
|
||||
|
||||
const importFilesAsSplitLayers = useCallback(
|
||||
async (files: File[]) => {
|
||||
if (!isLayerSplitEnabled) {
|
||||
throw new Error("图层拆分功能未开启");
|
||||
}
|
||||
if (files.length === 0) return 0;
|
||||
const invalidFile = files.find((file) => !file.type.startsWith("image/"));
|
||||
if (invalidFile) {
|
||||
@ -1778,7 +1783,7 @@ export function RealtimeCanvasDialog({
|
||||
|
||||
return appendLayerBatch(nextLayerBatch);
|
||||
},
|
||||
[appendLayerBatch, createPlacedLayer, fileToDataUrl]
|
||||
[appendLayerBatch, createPlacedLayer, fileToDataUrl, isLayerSplitEnabled]
|
||||
);
|
||||
|
||||
const triggerGenerate = useCallback(() => {
|
||||
@ -3534,23 +3539,25 @@ export function RealtimeCanvasDialog({
|
||||
)}
|
||||
</div>
|
||||
<div className="mt-2 flex flex-col gap-2">
|
||||
<Button
|
||||
className={cn(
|
||||
"h-10 rounded-xl px-0 hover:bg-[#f5f5f5]",
|
||||
controlSurfaceClass
|
||||
)}
|
||||
disabled={isLayerSplitImporting}
|
||||
onClick={() => splitUploadInputRef.current?.click()}
|
||||
title="图层拆分导入"
|
||||
type="button"
|
||||
variant="outline"
|
||||
>
|
||||
{isLayerSplitImporting ? (
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
) : (
|
||||
<Layers3 className="h-4 w-4" />
|
||||
)}
|
||||
</Button>
|
||||
{isLayerSplitEnabled ? (
|
||||
<Button
|
||||
className={cn(
|
||||
"h-10 rounded-xl px-0 hover:bg-[#f5f5f5]",
|
||||
controlSurfaceClass
|
||||
)}
|
||||
disabled={isLayerSplitImporting}
|
||||
onClick={() => splitUploadInputRef.current?.click()}
|
||||
title="图层拆分导入"
|
||||
type="button"
|
||||
variant="outline"
|
||||
>
|
||||
{isLayerSplitImporting ? (
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
) : (
|
||||
<Layers3 className="h-4 w-4" />
|
||||
)}
|
||||
</Button>
|
||||
) : null}
|
||||
<Button
|
||||
className={cn(
|
||||
"h-10 rounded-xl px-0 hover:bg-[#f5f5f5]",
|
||||
|
||||
@ -351,9 +351,14 @@ async function drawMarkerOnImage(
|
||||
interface ImageNodeProps {
|
||||
node: CanvasNode;
|
||||
isSelected: boolean;
|
||||
isLayerSplitEnabled?: boolean;
|
||||
}
|
||||
|
||||
export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
export function ImageNode({
|
||||
node,
|
||||
isSelected,
|
||||
isLayerSplitEnabled = false,
|
||||
}: ImageNodeProps) {
|
||||
const data = node.data as ImageNodeData;
|
||||
const contentScale = useMemo(
|
||||
() =>
|
||||
@ -1150,6 +1155,24 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
setPickerMode(false);
|
||||
}, [setPickerMode]);
|
||||
|
||||
const handleStartPicking = useCallback(
|
||||
(event?: React.MouseEvent) => {
|
||||
event?.stopPropagation();
|
||||
if (!data.src) {
|
||||
toast.error("请先上传图片");
|
||||
return;
|
||||
}
|
||||
if (isProcessing) {
|
||||
toast.info("正在处理中,请稍候...");
|
||||
return;
|
||||
}
|
||||
|
||||
setPickerMode(true, node.id);
|
||||
toast.info("点击图片中的物品进行拾取");
|
||||
},
|
||||
[data.src, isProcessing, node.id, setPickerMode]
|
||||
);
|
||||
|
||||
const handleEnterBoxCutoutMode = useCallback(
|
||||
(event?: React.MouseEvent | React.PointerEvent) => {
|
||||
event?.stopPropagation();
|
||||
@ -2248,6 +2271,10 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
return;
|
||||
}
|
||||
if (action === "layer-split") {
|
||||
if (!isLayerSplitEnabled) {
|
||||
toast.info("图层拆分功能未开启");
|
||||
return;
|
||||
}
|
||||
setIsProcessing(true);
|
||||
try {
|
||||
const result = await imageGenerationService.splitImageLayers(data.src);
|
||||
@ -3247,10 +3274,12 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
<StickyNote className="mr-2 h-4 w-4" />
|
||||
备注
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onClick={() => handleAction("layer-split")}>
|
||||
<Layers className="mr-2 h-4 w-4" />
|
||||
图层拆分
|
||||
</DropdownMenuItem>
|
||||
{isLayerSplitEnabled ? (
|
||||
<DropdownMenuItem onClick={() => handleAction("layer-split")}>
|
||||
<Layers className="mr-2 h-4 w-4" />
|
||||
图层拆分
|
||||
</DropdownMenuItem>
|
||||
) : null}
|
||||
<DropdownMenuItem onClick={() => handleAction("resize")}>
|
||||
<Maximize2 className="mr-2 h-4 w-4" />
|
||||
调尺
|
||||
@ -3451,6 +3480,16 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
|
||||
<div className="mx-1 h-6 w-px bg-slate-200 dark:bg-slate-700" />
|
||||
|
||||
<Button
|
||||
className="h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
|
||||
disabled={isProcessing || isPickerLoading || isThisNodePicking}
|
||||
onClick={handleStartPicking}
|
||||
variant="ghost"
|
||||
>
|
||||
<Crosshair className="mr-1.5 h-4 w-4" />
|
||||
识物
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
className="h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
|
||||
disabled={isProcessing}
|
||||
@ -3460,7 +3499,7 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
<span className="mr-1.5 rounded-md border border-slate-300 px-1.5 text-[11px] text-slate-600 leading-5 dark:border-slate-500 dark:text-slate-100">
|
||||
HD
|
||||
</span>
|
||||
高清增强
|
||||
高清
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
@ -3473,6 +3512,16 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
抠图
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
className="h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
|
||||
disabled={isProcessing}
|
||||
onClick={() => handleAction("box-cutout")}
|
||||
variant="ghost"
|
||||
>
|
||||
<Scan className="mr-1.5 h-4 w-4" />
|
||||
框选抠图
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
className="h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
|
||||
disabled={isProcessing}
|
||||
@ -3490,7 +3539,7 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
variant="ghost"
|
||||
>
|
||||
<Maximize2 className="mr-1.5 h-4 w-4" />
|
||||
尺寸调整
|
||||
尺寸
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
@ -3500,7 +3549,7 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
variant="ghost"
|
||||
>
|
||||
<Type className="mr-1.5 h-4 w-4" />
|
||||
文字替换
|
||||
文字
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
@ -3510,18 +3559,20 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
variant="ghost"
|
||||
>
|
||||
<Box className="mr-1.5 h-4 w-4" />
|
||||
视角生成
|
||||
多视角
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
className="h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
|
||||
disabled={isProcessing}
|
||||
onClick={() => handleAction("layer-split")}
|
||||
variant="ghost"
|
||||
>
|
||||
<Layers className="mr-1.5 h-4 w-4" />
|
||||
图层拆分
|
||||
</Button>
|
||||
{isLayerSplitEnabled ? (
|
||||
<Button
|
||||
className="h-10 rounded-xl px-3 font-medium text-[13px] text-slate-700 hover:bg-slate-100 hover:text-slate-900 dark:text-slate-100 dark:hover:bg-zinc-800"
|
||||
disabled={isProcessing}
|
||||
onClick={() => handleAction("layer-split")}
|
||||
variant="ghost"
|
||||
>
|
||||
<Layers className="mr-1.5 h-4 w-4" />
|
||||
图层拆分
|
||||
</Button>
|
||||
) : null}
|
||||
|
||||
<div className="mx-1 h-6 w-px bg-slate-200 dark:bg-slate-700" />
|
||||
|
||||
|
||||
@ -8,7 +8,7 @@ import {
|
||||
Sparkles,
|
||||
Waypoints,
|
||||
} from "lucide-react";
|
||||
import { useMemo, useState } from "react";
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { World3DViewer } from "@/components/canvas/world3d/World3DViewer";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
@ -19,6 +19,7 @@ type SharedWorldPageProps = {
|
||||
title: string;
|
||||
prompt?: string | null;
|
||||
caption?: string | null;
|
||||
model?: string | null;
|
||||
worldUrl?: string | null;
|
||||
imageUrl?: string | null;
|
||||
meshUrl?: string | null;
|
||||
@ -26,6 +27,21 @@ type SharedWorldPageProps = {
|
||||
previewImageUrl?: string | null;
|
||||
semanticsMetadata?: Record<string, unknown> | null;
|
||||
shareUrl: string;
|
||||
worlds?: SharedWorldItem[];
|
||||
};
|
||||
|
||||
type SharedWorldItem = {
|
||||
id: string;
|
||||
title: string;
|
||||
prompt?: string | null;
|
||||
caption?: string | null;
|
||||
model?: string | null;
|
||||
worldUrl?: string | null;
|
||||
imageUrl?: string | null;
|
||||
meshUrl?: string | null;
|
||||
splatUrl?: string | null;
|
||||
previewImageUrl?: string | null;
|
||||
semanticsMetadata?: Record<string, unknown> | null;
|
||||
};
|
||||
|
||||
const VIEW_OPTIONS: Array<{
|
||||
@ -62,6 +78,7 @@ export function WorldSharePage({
|
||||
title,
|
||||
prompt,
|
||||
caption,
|
||||
model,
|
||||
worldUrl,
|
||||
imageUrl,
|
||||
meshUrl,
|
||||
@ -69,10 +86,55 @@ export function WorldSharePage({
|
||||
previewImageUrl,
|
||||
semanticsMetadata,
|
||||
shareUrl,
|
||||
worlds,
|
||||
}: SharedWorldPageProps) {
|
||||
const worldItems = useMemo<SharedWorldItem[]>(
|
||||
() =>
|
||||
worlds?.length
|
||||
? worlds
|
||||
: [
|
||||
{
|
||||
id: "shared-world",
|
||||
title,
|
||||
prompt,
|
||||
caption,
|
||||
model,
|
||||
worldUrl,
|
||||
imageUrl,
|
||||
meshUrl,
|
||||
splatUrl,
|
||||
previewImageUrl,
|
||||
semanticsMetadata,
|
||||
},
|
||||
],
|
||||
[
|
||||
caption,
|
||||
imageUrl,
|
||||
meshUrl,
|
||||
model,
|
||||
previewImageUrl,
|
||||
prompt,
|
||||
semanticsMetadata,
|
||||
splatUrl,
|
||||
title,
|
||||
worldUrl,
|
||||
worlds,
|
||||
]
|
||||
);
|
||||
const [activeWorldId, setActiveWorldId] = useState(
|
||||
() => worldItems[0]?.id || "shared-world"
|
||||
);
|
||||
const activeWorld =
|
||||
worldItems.find((item) => item.id === activeWorldId) || worldItems[0];
|
||||
const availableViews = useMemo(
|
||||
() => resolveAvailableViews({ worldUrl, imageUrl, meshUrl, splatUrl }),
|
||||
[imageUrl, meshUrl, splatUrl, worldUrl]
|
||||
() =>
|
||||
resolveAvailableViews({
|
||||
worldUrl: activeWorld?.worldUrl,
|
||||
imageUrl: activeWorld?.imageUrl,
|
||||
meshUrl: activeWorld?.meshUrl,
|
||||
splatUrl: activeWorld?.splatUrl,
|
||||
}),
|
||||
[activeWorld?.imageUrl, activeWorld?.meshUrl, activeWorld?.splatUrl, activeWorld?.worldUrl]
|
||||
);
|
||||
const [activeView, setActiveView] = useState<World3DViewMode>(
|
||||
inferInitialView(availableViews)
|
||||
@ -80,7 +142,24 @@ export function WorldSharePage({
|
||||
const [firstPerson, setFirstPerson] = useState(false);
|
||||
const [copied, setCopied] = useState(false);
|
||||
|
||||
const primaryDescription = caption || prompt || "这个 3D 世界已开启只读分享。";
|
||||
useEffect(() => {
|
||||
if (worldItems.some((item) => item.id === activeWorldId)) return;
|
||||
setActiveWorldId(worldItems[0]?.id || "shared-world");
|
||||
}, [activeWorldId, worldItems]);
|
||||
|
||||
useEffect(() => {
|
||||
if (availableViews.includes(activeView)) return;
|
||||
setActiveView(inferInitialView(availableViews));
|
||||
}, [activeView, availableViews]);
|
||||
|
||||
useEffect(() => {
|
||||
setFirstPerson(false);
|
||||
}, [activeWorld?.id]);
|
||||
|
||||
const primaryDescription =
|
||||
activeWorld?.caption || activeWorld?.prompt || "这个 3D 世界已开启只读分享。";
|
||||
const activeTitle = activeWorld?.title || title;
|
||||
const activeMeshUrl = activeWorld?.meshUrl || null;
|
||||
|
||||
const handleCopyShareUrl = async () => {
|
||||
try {
|
||||
@ -94,10 +173,10 @@ export function WorldSharePage({
|
||||
};
|
||||
|
||||
const handleDownloadMesh = () => {
|
||||
if (!meshUrl) return;
|
||||
if (!activeMeshUrl) return;
|
||||
const link = document.createElement("a");
|
||||
link.href = meshUrl;
|
||||
link.download = "shared-world.glb";
|
||||
link.href = activeMeshUrl;
|
||||
link.download = `shared-world-${activeWorld?.id?.slice(0, 8) || "asset"}.glb`;
|
||||
link.click();
|
||||
};
|
||||
|
||||
@ -108,7 +187,7 @@ export function WorldSharePage({
|
||||
<div className="space-y-3">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge className="border-cyan-500/30 bg-cyan-500/10 text-cyan-200" variant="outline">
|
||||
3D 世界分享
|
||||
{worldItems.length > 1 ? `${worldItems.length} 个 3D 世界分享` : "3D 世界分享"}
|
||||
</Badge>
|
||||
{availableViews.map((view) => (
|
||||
<Badge className="border-white/10 bg-white/5 text-zinc-300" key={view} variant="outline">
|
||||
@ -118,7 +197,7 @@ export function WorldSharePage({
|
||||
</div>
|
||||
<div>
|
||||
<h1 className="text-2xl font-semibold tracking-tight text-white md:text-3xl">
|
||||
{title}
|
||||
{activeTitle}
|
||||
</h1>
|
||||
<p className="mt-2 max-w-3xl text-sm leading-6 text-zinc-300">
|
||||
{primaryDescription}
|
||||
@ -135,7 +214,7 @@ export function WorldSharePage({
|
||||
<Link2 className="mr-2 h-4 w-4" />
|
||||
{copied ? "已复制链接" : "复制分享链接"}
|
||||
</Button>
|
||||
{meshUrl ? (
|
||||
{activeMeshUrl ? (
|
||||
<Button
|
||||
className="border-white/10 bg-white/5 text-zinc-100 hover:bg-white/10"
|
||||
onClick={handleDownloadMesh}
|
||||
@ -176,7 +255,7 @@ export function WorldSharePage({
|
||||
))}
|
||||
</div>
|
||||
|
||||
{activeView === "splat" && splatUrl ? (
|
||||
{activeView === "splat" && activeWorld?.splatUrl ? (
|
||||
<Button
|
||||
className={cn(
|
||||
"rounded-full border border-white/10",
|
||||
@ -196,21 +275,67 @@ export function WorldSharePage({
|
||||
|
||||
<div className="h-[62vh] min-h-[520px]">
|
||||
<World3DViewer
|
||||
key={`${activeWorld?.id || "shared-world"}-${activeView}`}
|
||||
className="h-full w-full"
|
||||
firstPerson={firstPerson}
|
||||
imageUrl={imageUrl}
|
||||
imageUrl={activeWorld?.imageUrl || null}
|
||||
interactionMode="navigate"
|
||||
meshUrl={meshUrl}
|
||||
meshUrl={activeWorld?.meshUrl || null}
|
||||
mode={activeView}
|
||||
previewImageUrl={previewImageUrl}
|
||||
semanticsMetadata={semanticsMetadata}
|
||||
splatUrl={splatUrl}
|
||||
worldUrl={worldUrl}
|
||||
previewImageUrl={activeWorld?.previewImageUrl || null}
|
||||
semanticsMetadata={activeWorld?.semanticsMetadata || null}
|
||||
splatUrl={activeWorld?.splatUrl || null}
|
||||
worldUrl={activeWorld?.worldUrl || null}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-4">
|
||||
{worldItems.length > 1 ? (
|
||||
<div className="rounded-3xl border border-white/10 bg-black/30 p-4 shadow-2xl backdrop-blur">
|
||||
<h2 className="text-sm font-medium text-white">世界列表</h2>
|
||||
<div className="mt-3 space-y-2">
|
||||
{worldItems.map((item, index) => {
|
||||
const thumbnail = item.previewImageUrl || item.imageUrl || null;
|
||||
const isActive = item.id === activeWorld?.id;
|
||||
|
||||
return (
|
||||
<button
|
||||
className={cn(
|
||||
"flex w-full items-center gap-3 rounded-2xl border border-white/10 bg-white/5 p-2 text-left transition hover:bg-white/10",
|
||||
isActive && "border-cyan-300/60 bg-cyan-400/10"
|
||||
)}
|
||||
key={item.id}
|
||||
onClick={() => setActiveWorldId(item.id)}
|
||||
type="button"
|
||||
>
|
||||
<div className="flex h-14 w-16 shrink-0 items-center justify-center overflow-hidden rounded-xl bg-white/10">
|
||||
{thumbnail ? (
|
||||
<img
|
||||
alt={item.title}
|
||||
className="h-full w-full object-cover"
|
||||
src={thumbnail}
|
||||
/>
|
||||
) : (
|
||||
<Box className="h-5 w-5 text-zinc-500" />
|
||||
)}
|
||||
</div>
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="text-[11px] text-cyan-200">世界 {index + 1}</div>
|
||||
<div className="truncate text-sm font-medium text-white">
|
||||
{item.title}
|
||||
</div>
|
||||
<div className="mt-1 truncate text-xs text-zinc-400">
|
||||
{item.caption || item.prompt || item.model || "暂无描述"}
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="rounded-3xl border border-white/10 bg-black/30 p-4 shadow-2xl backdrop-blur">
|
||||
<h2 className="text-sm font-medium text-white">查看说明</h2>
|
||||
<div className="mt-3 space-y-2 text-sm leading-6 text-zinc-300">
|
||||
@ -224,15 +349,17 @@ export function WorldSharePage({
|
||||
<div className="mt-3 grid grid-cols-1 gap-2 text-xs text-zinc-300">
|
||||
<div className="rounded-2xl border border-white/10 bg-white/5 px-3 py-2">
|
||||
<div className="text-zinc-500">World</div>
|
||||
<div className="mt-1">{splatUrl ? "可查看" : "暂无"}</div>
|
||||
<div className="mt-1">{activeWorld?.splatUrl ? "可查看" : "暂无"}</div>
|
||||
</div>
|
||||
<div className="rounded-2xl border border-white/10 bg-white/5 px-3 py-2">
|
||||
<div className="text-zinc-500">图像</div>
|
||||
<div className="mt-1">{imageUrl || previewImageUrl ? "可查看" : "暂无"}</div>
|
||||
<div className="mt-1">
|
||||
{activeWorld?.imageUrl || activeWorld?.previewImageUrl ? "可查看" : "暂无"}
|
||||
</div>
|
||||
</div>
|
||||
<div className="rounded-2xl border border-white/10 bg-white/5 px-3 py-2">
|
||||
<div className="text-zinc-500">Mesh</div>
|
||||
<div className="mt-1">{meshUrl ? "可下载" : "暂无"}</div>
|
||||
<div className="mt-1">{activeWorld?.meshUrl ? "可下载" : "暂无"}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -66,6 +66,11 @@ function normalizeImageParamsForProvider(
|
||||
return normalized;
|
||||
}
|
||||
|
||||
if (providerType === "krea") {
|
||||
normalized.imageCount = Math.min(Math.max(currentCount, 1), 4);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
if (providerType === "alibaba-image") {
|
||||
normalized.imageCount = Math.min(Math.max(currentCount, 1), 6);
|
||||
return normalized;
|
||||
@ -1137,7 +1142,9 @@ export function Chat({
|
||||
? "fal-video"
|
||||
: model?.providerType === "ruxa-video"
|
||||
? "ruxa-video"
|
||||
: "volcano-video";
|
||||
: model?.providerType === "krea"
|
||||
? "krea-video"
|
||||
: "volcano-video";
|
||||
else if (
|
||||
isImageGenerationModelDescriptor({
|
||||
providerType: model?.providerType,
|
||||
@ -1999,6 +2006,7 @@ export function Chat({
|
||||
if (provider === "qianwen-video") return "qianwen-video";
|
||||
if (provider === "fal-video") return "fal-video";
|
||||
if (provider === "ruxa-video") return "ruxa-video";
|
||||
if (provider === "krea") return "krea-video";
|
||||
if (provider === "volcano-video" || provider === "doubao-video") return "volcano-video";
|
||||
return "other";
|
||||
};
|
||||
|
||||
@ -12,6 +12,7 @@ import {
|
||||
type ThemeColorMode,
|
||||
Transforms,
|
||||
type Viewport,
|
||||
getHitElementsBySelection,
|
||||
toHostPoint,
|
||||
toViewBoxPoint,
|
||||
} from "@plait/core";
|
||||
@ -19,8 +20,9 @@ import { DrawTransforms, MemorizeKey } from "@plait/draw";
|
||||
import {
|
||||
canSetAbstract,
|
||||
getSelectedMindElements,
|
||||
type MindElement,
|
||||
MindElement,
|
||||
MindTransforms,
|
||||
PlaitMind,
|
||||
} from "@plait/mind";
|
||||
import type { BoardChangeData } from "@plait-board/react-board";
|
||||
import {
|
||||
@ -441,6 +443,48 @@ function getExistingFlowBoardMindElements(
|
||||
});
|
||||
}
|
||||
|
||||
function uniqueFlowBoardMindElements(elements: MindElement[]) {
|
||||
const seen = new Set<MindElement>();
|
||||
return elements.filter((element) => {
|
||||
if (seen.has(element)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(element);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function preferFlowBoardNodeElements(elements: MindElement[]) {
|
||||
const nodeElements = elements.filter((element) => !PlaitMind.isMind(element));
|
||||
return nodeElements.length > 0 ? nodeElements : elements;
|
||||
}
|
||||
|
||||
function getFlowBoardMindElementsBySelection(
|
||||
board: PlaitBoard,
|
||||
selection: Selection | null | undefined
|
||||
) {
|
||||
if (!selection) {
|
||||
return [] as MindElement[];
|
||||
}
|
||||
|
||||
const hitElements = getHitElementsBySelection(board, selection, (element) =>
|
||||
MindElement.isMindElement(board, element)
|
||||
);
|
||||
return getSelectedMindElements(board, hitElements);
|
||||
}
|
||||
|
||||
function getFlowBoardSelectedMindElements(
|
||||
board: PlaitBoard,
|
||||
selection: Selection | null | undefined
|
||||
) {
|
||||
return preferFlowBoardNodeElements(
|
||||
uniqueFlowBoardMindElements([
|
||||
...getSelectedMindElements(board),
|
||||
...getFlowBoardMindElementsBySelection(board, selection),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
function getFlowBoardMarkerSelectionState(
|
||||
board: PlaitBoard | null,
|
||||
selectedElements?: MindElement[]
|
||||
@ -599,6 +643,7 @@ export function DrawnixWorkspace({
|
||||
const boardRef = useRef<PlaitBoard | null>(null);
|
||||
const mindMapFileInputRef = useRef<HTMLInputElement | null>(null);
|
||||
const selectedMindElementsRef = useRef<MindElement[]>([]);
|
||||
const selectedMindSelectionRef = useRef<Selection | null>(null);
|
||||
const [isMarkerPanelOpen, setIsMarkerPanelOpen] = useState(false);
|
||||
const [isImportingMindMap, setIsImportingMindMap] = useState(false);
|
||||
const [markerSelectionState, setMarkerSelectionState] =
|
||||
@ -621,7 +666,10 @@ export function DrawnixWorkspace({
|
||||
return [] as MindElement[];
|
||||
}
|
||||
|
||||
const selectedElements = getSelectedMindElements(board);
|
||||
const selectedElements = getFlowBoardSelectedMindElements(
|
||||
board,
|
||||
selectedMindSelectionRef.current || board.selection
|
||||
);
|
||||
if (selectedElements.length > 0) {
|
||||
selectedMindElementsRef.current = selectedElements;
|
||||
return selectedElements;
|
||||
@ -636,7 +684,12 @@ export function DrawnixWorkspace({
|
||||
}, []);
|
||||
|
||||
const refreshMarkerSelectionState = useCallback(
|
||||
(options: { preserveFallback?: boolean } = {}) => {
|
||||
(
|
||||
options: {
|
||||
preserveFallback?: boolean;
|
||||
selection?: Selection | null;
|
||||
} = {}
|
||||
) => {
|
||||
const board = boardRef.current;
|
||||
if (!board) {
|
||||
selectedMindElementsRef.current = [];
|
||||
@ -644,7 +697,10 @@ export function DrawnixWorkspace({
|
||||
return;
|
||||
}
|
||||
|
||||
const selectedElements = getSelectedMindElements(board);
|
||||
const selectedElements = getFlowBoardSelectedMindElements(
|
||||
board,
|
||||
options.selection ?? selectedMindSelectionRef.current ?? board.selection
|
||||
);
|
||||
if (selectedElements.length > 0) {
|
||||
selectedMindElementsRef.current = selectedElements;
|
||||
setMarkerSelectionState(
|
||||
@ -674,7 +730,11 @@ export function DrawnixWorkspace({
|
||||
const handleDrawnixInit = useCallback((board: PlaitBoard) => {
|
||||
boardRef.current = board;
|
||||
seedFlowBoardDefaultTextColor();
|
||||
selectedMindElementsRef.current = getSelectedMindElements(board);
|
||||
selectedMindSelectionRef.current = board.selection;
|
||||
selectedMindElementsRef.current = getFlowBoardSelectedMindElements(
|
||||
board,
|
||||
board.selection
|
||||
);
|
||||
setMarkerSelectionState(
|
||||
getFlowBoardMarkerSelectionState(board, selectedMindElementsRef.current)
|
||||
);
|
||||
@ -936,8 +996,9 @@ export function DrawnixWorkspace({
|
||||
);
|
||||
|
||||
const handleDrawnixSelectionChange = useCallback(
|
||||
(_selection: Selection | null) => {
|
||||
refreshMarkerSelectionState();
|
||||
(selection: Selection | null) => {
|
||||
selectedMindSelectionRef.current = selection;
|
||||
refreshMarkerSelectionState({ selection });
|
||||
},
|
||||
[refreshMarkerSelectionState]
|
||||
);
|
||||
@ -1471,6 +1532,12 @@ export function DrawnixWorkspace({
|
||||
.drawnix .plait-text-container {
|
||||
color: ${FLOW_BOARD_DEFAULT_TEXT_COLOR};
|
||||
}
|
||||
|
||||
.drawnix:not(.drawnix--mobile) .app-toolbar {
|
||||
top: auto;
|
||||
bottom: 36px;
|
||||
left: 36px;
|
||||
}
|
||||
`}
|
||||
</style>
|
||||
<input
|
||||
@ -1502,10 +1569,10 @@ export function DrawnixWorkspace({
|
||||
</FloatingSidebarShell>
|
||||
|
||||
<FloatingSidebarShell
|
||||
className="absolute top-9 right-[180px] z-20"
|
||||
className="absolute left-6 top-[92px] z-20"
|
||||
flowBoardControl
|
||||
>
|
||||
<FloatingSidebarRail orientation="horizontal">
|
||||
<FloatingSidebarRail className="max-h-[calc(100vh-172px)]">
|
||||
<FloatingSidebarAction
|
||||
disabled={isSavingNow}
|
||||
icon={Save}
|
||||
@ -1653,6 +1720,9 @@ export function DrawnixWorkspace({
|
||||
onMouseDownCapture={(event) => {
|
||||
event.stopPropagation();
|
||||
}}
|
||||
onPointerDownCapture={(event) => {
|
||||
event.stopPropagation();
|
||||
}}
|
||||
>
|
||||
<div className="flex items-center justify-between border-black/5 border-b px-5 py-4">
|
||||
<div>
|
||||
@ -1723,6 +1793,10 @@ export function DrawnixWorkspace({
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}}
|
||||
onPointerDown={(event) => {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}}
|
||||
onClick={() =>
|
||||
handleApplyFlowBoardMarker(group, marker)
|
||||
}
|
||||
@ -1747,6 +1821,10 @@ export function DrawnixWorkspace({
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}}
|
||||
onPointerDown={(event) => {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}}
|
||||
onClick={handleClearFlowBoardMarkers}
|
||||
type="button"
|
||||
>
|
||||
|
||||
@ -156,6 +156,10 @@ export function FloatingSidebarAction({
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}}
|
||||
onPointerDown={(event) => {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}}
|
||||
onClick={onClick}
|
||||
type="button"
|
||||
>
|
||||
|
||||
@ -24,6 +24,7 @@ const PROVIDER_MODEL_LABEL: Record<string, string> = {
|
||||
midjourney: "Midjourney",
|
||||
"volcano-vision": "即梦 4.5",
|
||||
gemini: "Gemini 图片",
|
||||
krea: "Krea 图片",
|
||||
};
|
||||
|
||||
const PHASE_HINT: Record<NonNullable<ImageGenerationLoadingProps["phase"]>, string> = {
|
||||
|
||||
@ -25,6 +25,8 @@ type DynamicModel = {
|
||||
providerType: string;
|
||||
modelId: string;
|
||||
displayName: string;
|
||||
supportsImageGen?: boolean;
|
||||
capabilities?: unknown;
|
||||
};
|
||||
|
||||
const fetcher = (url: string) => fetch(url).then((res) => res.json());
|
||||
@ -114,7 +116,7 @@ function PureMessages({
|
||||
const isQianwenVideoModel = modelsData?.models?.some(
|
||||
(m: DynamicModel) =>
|
||||
m.id === selectedModelId &&
|
||||
(m.providerType === "qianwen-video" || m.providerType === "fal-video" || m.providerType === "ruxa-video")
|
||||
(m.providerType === "qianwen-video" || m.providerType === "fal-video" || m.providerType === "ruxa-video" || m.providerType === "krea")
|
||||
) ?? false;
|
||||
|
||||
const isVolcanoVisionModel = modelsData?.models?.some(
|
||||
@ -615,6 +617,7 @@ function PureMessages({
|
||||
modelType === "qianwen-video" ||
|
||||
modelType === "fal-video" ||
|
||||
modelType === "ruxa-video" ||
|
||||
modelType === "krea-video" ||
|
||||
(!modelType && (isQianwenVideoModel || isVideoModel));
|
||||
|
||||
const isAutoMode = selectedModelId === "auto";
|
||||
@ -657,13 +660,15 @@ function PureMessages({
|
||||
? "ruxa-video"
|
||||
: modelType === "fal-video"
|
||||
? "fal-video"
|
||||
: modelType === "volcano-video"
|
||||
? "volcano-video"
|
||||
: modelType === "qianwen-video"
|
||||
? "qianwen-video"
|
||||
: currentModelProviderType === "doubao-video"
|
||||
? "volcano-video"
|
||||
: currentModelProviderType;
|
||||
: modelType === "krea-video"
|
||||
? "krea"
|
||||
: modelType === "volcano-video"
|
||||
? "volcano-video"
|
||||
: modelType === "qianwen-video"
|
||||
? "qianwen-video"
|
||||
: currentModelProviderType === "doubao-video"
|
||||
? "volcano-video"
|
||||
: currentModelProviderType;
|
||||
const videoPhase =
|
||||
imageGenerationPhase === "idle" ? "processing" : imageGenerationPhase;
|
||||
|
||||
|
||||
@ -1554,7 +1554,7 @@ function PureModelSelectorCompact({
|
||||
|
||||
const modeFilteredModels = useMemo(() => {
|
||||
if (modeFilter === "video") {
|
||||
return dynamicModels.filter((model) => isVideoProviderType(model.providerType));
|
||||
return dynamicModels.filter((model) => isVideoProviderType(model));
|
||||
}
|
||||
if (modeFilter === "image") {
|
||||
return dynamicModels.filter((model) => isImageGenerationModel(model));
|
||||
|
||||
@ -19,6 +19,7 @@ const PROVIDER_MODEL_LABEL: Record<string, string> = {
|
||||
"ruxa-video": "RUXA 视频",
|
||||
"volcano-video": "火山视频",
|
||||
runway: "Runway 视频",
|
||||
krea: "Krea 视频",
|
||||
};
|
||||
|
||||
const PHASE_HINT: Record<VideoGenerationPhase, string> = {
|
||||
@ -35,6 +36,7 @@ const PROVIDER_GRADIENT_CLASS: Record<string, string> = {
|
||||
"ruxa-video": "from-violet-400 via-fuchsia-400 to-pink-400",
|
||||
"volcano-video": "from-orange-400 via-amber-400 to-yellow-400",
|
||||
runway: "from-rose-400 via-orange-400 to-amber-400",
|
||||
krea: "from-blue-400 via-cyan-400 to-emerald-400",
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@ -8,7 +8,7 @@ import { useEffect, useState, useCallback } from "react";
|
||||
export interface PendingGeneration {
|
||||
chatId: string;
|
||||
modelId: string;
|
||||
modelType: "gemini" | "volcano-vision" | "volcano-video" | "qianwen-video" | "fal-video" | "ruxa-video" | "alibaba-image" | "other";
|
||||
modelType: "gemini" | "volcano-vision" | "volcano-video" | "qianwen-video" | "fal-video" | "ruxa-video" | "krea-video" | "alibaba-image" | "other";
|
||||
timestamp: number;
|
||||
userMessageId?: string;
|
||||
}
|
||||
|
||||
@ -26,6 +26,7 @@ import { createMidjourney } from "./providers/midjourney";
|
||||
import { createFalVideo } from "./providers/fal-video";
|
||||
import { createRuxaVideo } from "./providers/ruxa-video";
|
||||
import { createRunway } from "./providers/runway";
|
||||
import { createKrea } from "./providers/krea";
|
||||
import { createVolcanoMultimodal } from "./providers/volcano-multimodal";
|
||||
import { getProviderDefaultBaseUrl } from "./provider-defaults";
|
||||
|
||||
@ -247,6 +248,15 @@ function getOrCreateProvider(provider: Pick<AiProvider, "id" | "type" | "baseUrl
|
||||
return runwayProvider as any;
|
||||
}
|
||||
|
||||
case "krea": {
|
||||
const kreaProvider = createKrea({
|
||||
baseURL: resolvedBaseUrl,
|
||||
apiKey: provider.apiKey,
|
||||
});
|
||||
console.log(`[Provider] Created Krea provider: ${resolvedBaseUrl}`);
|
||||
return kreaProvider as any;
|
||||
}
|
||||
|
||||
case "custom":
|
||||
default: {
|
||||
if (isVolcanoMultimodalBaseUrl(resolvedBaseUrl)) {
|
||||
@ -381,6 +391,10 @@ export async function getDynamicModel(
|
||||
console.log("[getDynamicModel] Getting Runway chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Runway model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "krea") {
|
||||
console.log("[getDynamicModel] Getting Krea chat model...");
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Krea model retrieved:", baseModel ? "success" : "failed");
|
||||
} else if (provider.type === "huanke-gemini") {
|
||||
// 环科 Gemini 原生格式使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Huanke Gemini chat model..."); // 调试日志
|
||||
|
||||
294
lib/ai/krea-models.ts
Normal file
294
lib/ai/krea-models.ts
Normal file
@ -0,0 +1,294 @@
|
||||
export type KreaModelKind = "image" | "image-enhance" | "video";
|
||||
|
||||
export interface KreaModelPreset {
|
||||
modelId: string;
|
||||
displayName: string;
|
||||
description: string;
|
||||
kind: KreaModelKind;
|
||||
supportsVision: boolean;
|
||||
supportsImageGen: boolean;
|
||||
requestFields: string[];
|
||||
requiredFields: string[];
|
||||
capabilities?: Record<string, any>;
|
||||
defaultParams: Record<string, any>;
|
||||
imageGenConfig?: Record<string, any>;
|
||||
sortOrder: number;
|
||||
}
|
||||
|
||||
const KREA_OPENAPI_URL = "https://api.krea.ai/openapi.json";
|
||||
|
||||
const REFERENCE_INPUT_FIELDS = new Set([
|
||||
"imageUrl",
|
||||
"imageUrls",
|
||||
"image_url",
|
||||
"styleImages",
|
||||
"imageStyleRefs",
|
||||
"characterReferenceImages",
|
||||
"backgroundImage",
|
||||
"referenceImages",
|
||||
"startImage",
|
||||
"endImage",
|
||||
"init_image_url",
|
||||
"references",
|
||||
"subjects",
|
||||
"styleReferences",
|
||||
"elementReferences",
|
||||
"videoReference",
|
||||
"startVideo",
|
||||
"init_video",
|
||||
"referenceVideos",
|
||||
"referenceAudios",
|
||||
]);
|
||||
|
||||
function hasReferenceInput(fields: string[]) {
|
||||
return fields.some((field) => REFERENCE_INPUT_FIELDS.has(field));
|
||||
}
|
||||
|
||||
function createDefaultParams(
|
||||
kind: KreaModelKind,
|
||||
modelId: string,
|
||||
requestFields: string[],
|
||||
requiredFields: string[]
|
||||
) {
|
||||
return {
|
||||
provider: "krea",
|
||||
kind,
|
||||
endpoint: modelId,
|
||||
requestFields,
|
||||
requiredFields,
|
||||
officialApi: KREA_OPENAPI_URL,
|
||||
};
|
||||
}
|
||||
|
||||
function createImageConfig(fields: string[]) {
|
||||
return {
|
||||
maxImages: fields.includes("batchSize") ? 4 : 1,
|
||||
supportsSize: fields.includes("width") && fields.includes("height"),
|
||||
supports1K: true,
|
||||
supports2K: true,
|
||||
supports4K: fields.includes("resolution") || fields.includes("width"),
|
||||
supportsPromptExtend: fields.includes("skipPromptExpansion"),
|
||||
supportsNegativePrompt: fields.includes("negative_prompt"),
|
||||
supportsWatermark: false,
|
||||
supportsSeed: fields.includes("seed"),
|
||||
creditCostPerImage: 0,
|
||||
enableLoadingPlaceholder: true,
|
||||
loadingColor: "blue",
|
||||
};
|
||||
}
|
||||
|
||||
function createVideoDurationOptions(modelId: string) {
|
||||
if (modelId.includes("/minimax/hailuo-02") || modelId.includes("/minimax/hailuo-2.3")) {
|
||||
return [6, 10];
|
||||
}
|
||||
if (modelId.includes("/google/veo")) {
|
||||
return [4, 8];
|
||||
}
|
||||
if (modelId.includes("/openai/sora-2")) {
|
||||
return [4, 8, 12, 20];
|
||||
}
|
||||
return [5, 10];
|
||||
}
|
||||
|
||||
function createVideoCapabilities(modelId: string, fields: string[]) {
|
||||
const supportedModes = new Set<string>();
|
||||
if (fields.includes("prompt")) supportedModes.add("text-to-video");
|
||||
if (
|
||||
fields.includes("startImage") ||
|
||||
fields.includes("image_url") ||
|
||||
fields.includes("init_image_url")
|
||||
) {
|
||||
supportedModes.add("first-frame");
|
||||
}
|
||||
if (fields.includes("endImage")) supportedModes.add("first-last-frame");
|
||||
if (
|
||||
fields.includes("referenceImages") ||
|
||||
fields.includes("styleReferences") ||
|
||||
fields.includes("elementReferences") ||
|
||||
fields.includes("subjects") ||
|
||||
fields.includes("references")
|
||||
) {
|
||||
supportedModes.add("reference-images");
|
||||
}
|
||||
if (supportedModes.size === 0) supportedModes.add("text-to-video");
|
||||
|
||||
const durationOptions = fields.includes("duration") ? createVideoDurationOptions(modelId) : [5];
|
||||
const resolutionOptions = fields.includes("resolution")
|
||||
? (modelId.includes("/minimax/") ? ["720p", "1080p"] : ["480p", "720p", "1080p"])
|
||||
: ["720p"];
|
||||
|
||||
return {
|
||||
supportedModes: Array.from(supportedModes),
|
||||
supportedResolutions: resolutionOptions,
|
||||
supportedRatios: ["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"],
|
||||
durationRange: {
|
||||
min: Math.min(...durationOptions),
|
||||
max: Math.max(...durationOptions),
|
||||
},
|
||||
durationOptions,
|
||||
supportWatermark: false,
|
||||
supportCameraFixed: fields.includes("cameraControl"),
|
||||
supportSeed: fields.includes("seed"),
|
||||
supportReturnLastFrame: fields.includes("endImage"),
|
||||
maxReferenceImages: fields.includes("referenceImages")
|
||||
? modelId.includes("/seedance-2")
|
||||
? 9
|
||||
: 4
|
||||
: fields.includes("styleReferences") || fields.includes("elementReferences")
|
||||
? 3
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function imagePreset(
|
||||
sortOrder: number,
|
||||
modelId: string,
|
||||
displayName: string,
|
||||
fields: string[],
|
||||
requiredFields: string[]
|
||||
): KreaModelPreset {
|
||||
return {
|
||||
modelId,
|
||||
displayName,
|
||||
description: `Krea official image endpoint: ${modelId}`,
|
||||
kind: "image",
|
||||
supportsVision: hasReferenceInput(fields),
|
||||
supportsImageGen: true,
|
||||
requestFields: fields,
|
||||
requiredFields,
|
||||
imageGenConfig: createImageConfig(fields),
|
||||
defaultParams: createDefaultParams("image", modelId, fields, requiredFields),
|
||||
sortOrder,
|
||||
};
|
||||
}
|
||||
|
||||
function enhancePreset(
|
||||
sortOrder: number,
|
||||
modelId: string,
|
||||
displayName: string,
|
||||
fields: string[],
|
||||
requiredFields: string[]
|
||||
): KreaModelPreset {
|
||||
return {
|
||||
modelId,
|
||||
displayName,
|
||||
description: `Krea official image enhance endpoint: ${modelId}`,
|
||||
kind: "image-enhance",
|
||||
supportsVision: true,
|
||||
supportsImageGen: true,
|
||||
requestFields: fields,
|
||||
requiredFields,
|
||||
imageGenConfig: createImageConfig(fields),
|
||||
defaultParams: createDefaultParams("image-enhance", modelId, fields, requiredFields),
|
||||
sortOrder,
|
||||
};
|
||||
}
|
||||
|
||||
function videoPreset(
|
||||
sortOrder: number,
|
||||
modelId: string,
|
||||
displayName: string,
|
||||
fields: string[],
|
||||
requiredFields: string[]
|
||||
): KreaModelPreset {
|
||||
const durationOptions = fields.includes("duration") ? createVideoDurationOptions(modelId) : [];
|
||||
return {
|
||||
modelId,
|
||||
displayName,
|
||||
description: `Krea official video endpoint: ${modelId}`,
|
||||
kind: "video",
|
||||
supportsVision: hasReferenceInput(fields),
|
||||
supportsImageGen: false,
|
||||
requestFields: fields,
|
||||
requiredFields,
|
||||
capabilities: createVideoCapabilities(modelId, fields),
|
||||
defaultParams: createDefaultParams("video", modelId, fields, requiredFields),
|
||||
imageGenConfig: {
|
||||
creditCostPerVideo: 0,
|
||||
videoDurationOptions: durationOptions,
|
||||
creditCostPerVideoByDuration: {},
|
||||
enableLoadingPlaceholder: true,
|
||||
loadingColor: "blue",
|
||||
},
|
||||
sortOrder,
|
||||
};
|
||||
}
|
||||
|
||||
export const KREA_MODEL_PRESETS: KreaModelPreset[] = [
|
||||
imagePreset(10, "/generate/image/bfl/flux-1-dev", "Flux", ["strength", "seed", "steps", "styleImages", "imageStyleRefs", "prompt", "width", "height", "guidance_scale_flux", "imageUrl", "relaxedModeAccess", "styles"], ["prompt"]),
|
||||
imagePreset(20, "/generate/image/bfl/flux-1-kontext-dev", "Flux Kontext", ["strength", "seed", "steps", "styleImages", "imageStyleRefs", "prompt", "width", "height", "guidance_scale_flux", "imageUrl", "relaxedModeAccess", "styles"], ["prompt"]),
|
||||
imagePreset(30, "/generate/image/google/nano-banana-pro", "Nano Banana Pro", ["batchSize", "prompt", "width", "height", "imageUrls", "aspectRatio", "styleImages", "resolution"], ["prompt"]),
|
||||
imagePreset(40, "/generate/image/google/nano-banana-2", "Nano Banana 2", ["batchSize", "prompt", "width", "height", "imageUrls", "aspectRatio", "styleImages", "resolution"], ["prompt"]),
|
||||
imagePreset(50, "/generate/image/google/nano-banana", "Nano Banana", ["batchSize", "prompt", "width", "height", "imageUrls", "styleImages", "aspectRatio"], ["prompt"]),
|
||||
imagePreset(60, "/generate/image/bfl/flux-1.1-pro", "Flux 1.1 Pro", ["batchSize", "prompt", "width", "height", "seed"], ["prompt", "width", "height"]),
|
||||
imagePreset(70, "/generate/image/bfl/flux-1.1-pro-ultra", "Flux 1.1 Pro Ultra", ["batchSize", "prompt", "width", "height", "seed", "raw"], ["prompt"]),
|
||||
imagePreset(80, "/generate/image/ideogram/ideogram-2-turbo", "Ideogram 2.0A Turbo", ["batchSize", "prompt", "width", "height", "seed"], ["prompt"]),
|
||||
imagePreset(90, "/generate/image/ideogram/ideogram-3", "Ideogram 3.0", ["batchSize", "prompt", "width", "height", "seed", "styleImages", "characterReferenceImages"], ["prompt"]),
|
||||
imagePreset(100, "/generate/image/google/imagen-3", "Imagen 3", ["batchSize", "prompt", "width", "height", "seed"], ["prompt"]),
|
||||
imagePreset(110, "/generate/image/google/imagen-4", "Imagen 4", ["batchSize", "prompt", "width", "height", "seed"], ["prompt"]),
|
||||
imagePreset(120, "/generate/image/google/imagen-4-fast", "Imagen 4 Fast", ["batchSize", "prompt", "width", "height", "seed"], ["prompt"]),
|
||||
imagePreset(130, "/generate/image/google/imagen-4-ultra", "Imagen 4 Ultra", ["batchSize", "prompt", "width", "height", "seed"], ["prompt"]),
|
||||
imagePreset(140, "/generate/image/runway/gen-4-image", "Runway Gen-4", ["batchSize", "prompt", "width", "height", "seed", "referenceImages"], ["prompt", "referenceImages"]),
|
||||
imagePreset(150, "/generate/image/openai/gpt-image", "ChatGPT Image", ["batchSize", "prompt", "width", "height", "quality", "backgroundImage", "imageUrls", "styleImages", "styles"], ["prompt"]),
|
||||
imagePreset(160, "/generate/image/bytedance/seedream-3", "Seedream 3", ["batchSize", "prompt", "width", "height", "seed", "model"], ["prompt", "model"]),
|
||||
imagePreset(170, "/generate/image/bytedance/seedream-4", "Seedream 4", ["batchSize", "prompt", "seed", "width", "height", "styleImages"], ["prompt", "width", "height"]),
|
||||
imagePreset(180, "/generate/image/bytedance/seedream-5-lite", "Seedream 5 Lite", ["batchSize", "prompt", "seed", "width", "height", "styleImages"], ["prompt", "width", "height"]),
|
||||
imagePreset(190, "/generate/image/qwen/2512", "Qwen 2512", ["prompt", "negative_prompt", "width", "height", "num_inference_steps", "cfg_scale", "seed", "batchSize", "skipPromptExpansion", "styles"], ["prompt"]),
|
||||
imagePreset(200, "/generate/image/z-image/z-image", "Z Image", ["prompt", "width", "height", "batchSize", "seed", "imageUrl", "denoising_strength", "styleImages", "skipPromptExpansion", "styles"], ["prompt", "width", "height"]),
|
||||
imagePreset(210, "/generate/image/bytedance/seededit", "SeedEdit", ["batchSize", "prompt", "seed", "imageUrl"], ["prompt", "imageUrl"]),
|
||||
enhancePreset(220, "/generate/enhance/topaz/generative-enhance", "Topaz Generative", ["batchSize", "width", "height", "seed", "prompt", "output_format", "subject_detection", "face_enhancement", "face_enhancement_creativity", "face_enhancement_strength", "crop_to_fill", "image_url", "model", "upscaling_activated", "image_scaling_factor", "creativity", "texture", "sharpen", "denoise", "detail"], ["width", "height", "image_url"]),
|
||||
enhancePreset(230, "/generate/enhance/topaz/standard-enhance", "Topaz", ["batchSize", "width", "height", "seed", "prompt", "output_format", "subject_detection", "face_enhancement", "face_enhancement_creativity", "face_enhancement_strength", "crop_to_fill", "image_url", "model", "upscaling_activated", "image_scaling_factor", "sharpen", "denoise", "fix_compression", "strength"], ["width", "height", "image_url", "model"]),
|
||||
enhancePreset(240, "/generate/enhance/topaz/bloom-enhance", "Topaz Bloom", ["batchSize", "width", "height", "seed", "prompt", "output_format", "crop_to_fill", "image_url", "model", "creativity", "face_preservation", "color_preservation", "upscaling_activated", "image_scaling_factor"], ["width", "height", "image_url", "model"]),
|
||||
videoPreset(250, "/generate/video/kling/kling-1", "Kling 1.0", ["startImage", "prompt", "aspectRatio", "duration", "cameraControl", "endImage", "mode"], ["prompt"]),
|
||||
videoPreset(260, "/generate/video/kling/kling-1.5", "Kling 1.5", ["startImage", "prompt", "aspectRatio", "duration", "cameraControl", "endImage"], ["prompt"]),
|
||||
videoPreset(270, "/generate/video/kling/kling-1.6", "Kling 1.6", ["startImage", "prompt", "aspectRatio", "duration", "cameraControl", "endImage", "mode"], ["prompt"]),
|
||||
videoPreset(280, "/generate/video/kling/kling-2", "Kling 2.0", ["startImage", "prompt", "aspectRatio", "endImage", "mode"], ["prompt"]),
|
||||
videoPreset(290, "/generate/video/kling/kling-2.1", "Kling 2.1", ["startImage", "prompt", "aspectRatio", "duration", "endImage", "mode"], ["prompt"]),
|
||||
videoPreset(300, "/generate/video/kling/kling-2.5", "Kling 2.5", ["startImage", "prompt", "aspectRatio", "duration", "endImage"], ["prompt"]),
|
||||
videoPreset(310, "/generate/video/kling/kling-2.6", "Kling 2.6", ["startImage", "prompt", "aspectRatio", "duration", "endImage", "generateAudio"], ["prompt"]),
|
||||
videoPreset(320, "/generate/video/kling/kling-o1", "Kling o1", ["startImage", "prompt", "aspectRatio", "duration", "endImage", "styleReferences", "elementReferences", "videoReference"], ["prompt"]),
|
||||
videoPreset(330, "/generate/video/minimax/hailuo", "Hailuo", ["startImage", "prompt", "expandPrompt", "cameraControl", "model", "subjects"], ["prompt"]),
|
||||
videoPreset(340, "/generate/video/minimax/hailuo-02", "Hailuo 02", ["startImage", "prompt", "expandPrompt", "endImage", "resolution", "duration"], ["prompt"]),
|
||||
videoPreset(350, "/generate/video/minimax/hailuo-2.3", "Hailuo 2.3", ["startImage", "prompt", "expandPrompt", "endImage", "resolution", "duration"], ["prompt"]),
|
||||
videoPreset(360, "/generate/video/minimax/hailuo-2.3-fast", "Hailuo 2.3 Fast", ["startImage", "prompt", "expandPrompt", "endImage", "resolution", "duration"], ["prompt"]),
|
||||
videoPreset(370, "/generate/video/alibaba/wan-2.1", "Wan 2.1", ["startImage", "prompt", "width", "height", "endImage", "aspectRatio", "frame_num", "styles"], ["prompt", "width", "height"]),
|
||||
videoPreset(380, "/generate/video/alibaba/wan-2.2", "Wan 2.2", ["startImage", "guide_scale", "prompt", "width", "height", "aspectRatio", "frame_num", "styles"], ["prompt", "width", "height"]),
|
||||
videoPreset(390, "/generate/video/alibaba/wan-2.5", "Wan 2.5", ["startImage", "prompt", "aspectRatio", "resolution", "enablePromptExpansion", "duration", "seed", "audio_url", "generateAudio"], ["prompt"]),
|
||||
videoPreset(400, "/generate/video/google/veo-2", "Veo 2", ["startImage", "prompt", "aspectRatio", "duration", "image_url"], ["prompt"]),
|
||||
videoPreset(410, "/generate/video/google/veo-3", "Veo 3", ["startImage", "prompt", "aspectRatio", "duration", "generateAudio", "resolution"], ["prompt"]),
|
||||
videoPreset(420, "/generate/video/google/veo-3-fast", "Veo 3 Fast", ["startImage", "prompt", "aspectRatio", "duration", "generateAudio", "resolution"], ["prompt"]),
|
||||
videoPreset(430, "/generate/video/google/veo-3.1", "Veo 3.1", ["startImage", "prompt", "aspectRatio", "duration", "generateAudio", "resolution", "endImage", "referenceImages"], ["prompt"]),
|
||||
videoPreset(440, "/generate/video/google/veo-3.1-fast", "Veo 3.1 Fast", ["startImage", "prompt", "aspectRatio", "duration", "generateAudio", "resolution", "endImage", "referenceImages"], ["prompt"]),
|
||||
videoPreset(450, "/generate/video/google/veo-3.1-lite", "Veo 3.1 Lite", ["startImage", "prompt", "aspectRatio", "duration", "resolution", "endImage"], ["prompt"]),
|
||||
videoPreset(460, "/generate/video/bytedance/seedance-1.0-lite", "Seedance Lite", ["startImage", "prompt", "aspectRatio", "endImage", "referenceImages", "duration", "resolution"], ["prompt"]),
|
||||
videoPreset(470, "/generate/video/bytedance/seedance-1.0-pro", "Seedance Pro", ["startImage", "prompt", "aspectRatio", "endImage", "referenceImages", "duration", "resolution"], ["prompt"]),
|
||||
videoPreset(480, "/generate/video/bytedance/seedance-1.0-pro-fast", "Seedance Pro Fast", ["startImage", "prompt", "aspectRatio", "duration", "resolution"], ["prompt"]),
|
||||
videoPreset(490, "/generate/video/bytedance/seedance-2", "Seedance 2.0", ["startImage", "prompt", "aspectRatio", "endImage", "referenceImages", "referenceVideos", "referenceAudios", "duration", "generateAudio", "resolution", "seed"], ["prompt"]),
|
||||
videoPreset(500, "/generate/video/bytedance/seedance-2-fast", "Seedance 2.0 Fast", ["startImage", "prompt", "aspectRatio", "endImage", "referenceImages", "referenceVideos", "referenceAudios", "duration", "generateAudio", "resolution", "seed"], ["prompt"]),
|
||||
videoPreset(510, "/generate/video/openai/sora-2", "Sora 2", ["startImage", "prompt", "duration", "aspectRatio"], ["prompt"]),
|
||||
videoPreset(520, "/generate/video/luma/ray-2", "Ray 2", ["startImage", "prompt", "aspectRatio", "loop", "width", "height"], ["prompt", "width", "height"]),
|
||||
videoPreset(530, "/generate/video/runway/gen-3", "Runway Gen-3", ["startImage", "prompt", "aspectRatio", "duration", "seed"], ["prompt"]),
|
||||
videoPreset(540, "/generate/video/runway/gen-4-video", "Runway Gen-4", ["startImage", "prompt", "aspectRatio", "duration", "seed"], ["prompt"]),
|
||||
videoPreset(550, "/generate/video/runway/gen-4.5", "Runway Gen-4.5", ["startImage", "prompt", "aspectRatio", "duration", "seed"], ["prompt"]),
|
||||
videoPreset(560, "/generate/video/runway/aleph", "Runway Aleph", ["startImage", "startVideo", "startVideoTrim", "endVideoTrim", "prompt", "init_video", "init_image_url", "references", "ratio", "seed", "contentModeration"], ["prompt", "init_video"]),
|
||||
];
|
||||
|
||||
export function getKreaModelPresetById(modelId: string) {
|
||||
const normalized = String(modelId || "").trim();
|
||||
return KREA_MODEL_PRESETS.find((preset) => preset.modelId === normalized);
|
||||
}
|
||||
|
||||
export function inferKreaModelKind(modelId?: string | null): KreaModelKind | null {
|
||||
const normalized = String(modelId || "").trim().toLowerCase();
|
||||
if (!normalized) return null;
|
||||
if (normalized.startsWith("/generate/video/") || normalized.includes("/generate/video/")) {
|
||||
return "video";
|
||||
}
|
||||
if (normalized.startsWith("/generate/enhance/") || normalized.includes("/generate/enhance/")) {
|
||||
return "image-enhance";
|
||||
}
|
||||
if (normalized.startsWith("/generate/image/") || normalized.includes("/generate/image/")) {
|
||||
return "image";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@ -1,3 +1,5 @@
|
||||
import { inferKreaModelKind } from "@/lib/ai/krea-models";
|
||||
|
||||
const STATIC_VIDEO_PROVIDER_TYPES = new Set([
|
||||
"doubao-video",
|
||||
"volcano-video",
|
||||
@ -88,6 +90,10 @@ export function isVideoModelDescriptor(params: {
|
||||
return inferRunwayModelKind(params.modelId) === "video";
|
||||
}
|
||||
|
||||
if (providerType === "krea") {
|
||||
return inferKreaModelKind(params.modelId) === "video";
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -105,6 +111,12 @@ export function isImageGenerationModelDescriptor(params: {
|
||||
if (runwayModelKind === "video") return false;
|
||||
}
|
||||
|
||||
if (providerType === "krea") {
|
||||
const kreaModelKind = inferKreaModelKind(params.modelId);
|
||||
if (kreaModelKind === "image" || kreaModelKind === "image-enhance") return true;
|
||||
if (kreaModelKind === "video") return false;
|
||||
}
|
||||
|
||||
if (isVideoModelDescriptor(params)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -155,7 +155,7 @@ const INTENT_MODEL_MAP: Record<
|
||||
> = {
|
||||
// 图片编辑:优先 alibaba-image(灵魂4.5,支持提示词扩展)
|
||||
image_edit: {
|
||||
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "huanke-gemini"],
|
||||
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini", "huanke-gemini"],
|
||||
requireVision: true,
|
||||
requireImageGen: true,
|
||||
description: "图片编辑",
|
||||
@ -170,14 +170,14 @@ const INTENT_MODEL_MAP: Record<
|
||||
|
||||
// 图片生成:优先灵魂4.5
|
||||
image_gen: {
|
||||
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "huanke-gemini", "volcano-vision"],
|
||||
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini", "huanke-gemini", "volcano-vision"],
|
||||
requireImageGen: true,
|
||||
description: "图片生成",
|
||||
},
|
||||
|
||||
// 视频生成
|
||||
video_gen: {
|
||||
providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video", "runway"],
|
||||
providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video", "runway", "krea"],
|
||||
description: "视频生成",
|
||||
},
|
||||
|
||||
@ -198,25 +198,25 @@ const SUB_INTENT_OVERRIDES: Record<
|
||||
> = {
|
||||
// 抠图:alibaba-image 效果最好
|
||||
remove_background: {
|
||||
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini"],
|
||||
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini"],
|
||||
reason: "抠图首选灵魂4.5",
|
||||
},
|
||||
|
||||
// 换背景:alibaba-image 支持场景生成
|
||||
replace_background: {
|
||||
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini"],
|
||||
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini"],
|
||||
reason: "换背景首选灵魂4.5",
|
||||
},
|
||||
|
||||
// 图片增强
|
||||
enhance: {
|
||||
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "volcano-vision"],
|
||||
providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini", "volcano-vision"],
|
||||
reason: "增强首选灵魂4.5",
|
||||
},
|
||||
|
||||
// 风格迁移:gemini 可能更有创意
|
||||
style_transfer: {
|
||||
providerTypes: ["gemini", "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway"],
|
||||
providerTypes: ["gemini", "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea"],
|
||||
reason: "风格迁移优选Gemini",
|
||||
},
|
||||
};
|
||||
|
||||
@ -21,6 +21,7 @@ const PROVIDER_DEFAULT_BASE_URL: Partial<Record<AiProvider["type"], string>> = {
|
||||
"worldlabs-3d": "https://api.worldlabs.ai",
|
||||
"kittycad-cad": "https://api.zoo.dev",
|
||||
runway: "https://api.dev.runwayml.com",
|
||||
krea: "https://api.krea.ai",
|
||||
};
|
||||
|
||||
export function getProviderDefaultBaseUrl(type: AiProvider["type"]): string | undefined {
|
||||
|
||||
832
lib/ai/providers/krea.ts
Normal file
832
lib/ai/providers/krea.ts
Normal file
@ -0,0 +1,832 @@
|
||||
import "server-only";
|
||||
|
||||
import sharp from "sharp";
|
||||
import {
|
||||
getKreaModelPresetById,
|
||||
inferKreaModelKind,
|
||||
type KreaModelKind,
|
||||
} from "@/lib/ai/krea-models";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
|
||||
interface KreaConfig {
|
||||
apiKey: string;
|
||||
baseURL: string;
|
||||
}
|
||||
|
||||
interface KreaImageParams {
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
imageCount?: number;
|
||||
seed?: number;
|
||||
negativePrompt?: string;
|
||||
promptExtend?: boolean;
|
||||
}
|
||||
|
||||
interface KreaVideoParams {
|
||||
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||||
ratio?: string;
|
||||
duration?: number;
|
||||
resolution?: string;
|
||||
seed?: number;
|
||||
generateAudio?: boolean;
|
||||
}
|
||||
|
||||
interface UploadedImage {
|
||||
thumbnail: string;
|
||||
original: string;
|
||||
}
|
||||
|
||||
const DEFAULT_BASE_URL = "https://api.krea.ai";
|
||||
const IMAGE_POLL_INTERVAL_MS = 2000;
|
||||
const VIDEO_POLL_INTERVAL_MS = 5000;
|
||||
const IMAGE_TIMEOUT_MS = 5 * 60 * 1000;
|
||||
const VIDEO_TIMEOUT_MS = 20 * 60 * 1000;
|
||||
|
||||
function normalizeText(value?: string | null) {
|
||||
return String(value || "").replace(/\u00a0/g, " ").trim().normalize("NFKC");
|
||||
}
|
||||
|
||||
function normalizeBaseURL(baseURL: string) {
|
||||
const normalized = normalizeText(baseURL).replace(/\/+$/, "");
|
||||
return normalized || DEFAULT_BASE_URL;
|
||||
}
|
||||
|
||||
function normalizeKreaApiKey(apiKey: string) {
|
||||
const stripped = normalizeText(apiKey)
|
||||
.replace(/^bearer\s+/i, "")
|
||||
.replace(/^key\s+/i, "")
|
||||
.replace(/^(?:krea[_-]?api[_-]?key|api[_-]?key|token)\s*[::]\s*/i, "");
|
||||
return stripped.trim().replace(/[^\x20-\x7E]/g, "");
|
||||
}
|
||||
|
||||
function buildHeaders(apiKey: string) {
|
||||
return {
|
||||
Authorization: `Bearer ${normalizeKreaApiKey(apiKey)}`,
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json",
|
||||
};
|
||||
}
|
||||
|
||||
function joinEndpoint(baseURL: string, path: string) {
|
||||
const cleanPath = path.startsWith("/") ? path : `/${path}`;
|
||||
return `${normalizeBaseURL(baseURL)}${cleanPath}`;
|
||||
}
|
||||
|
||||
function resolveKreaEndpoint(modelId: string) {
|
||||
const normalized = normalizeText(modelId);
|
||||
if (!normalized) {
|
||||
throw new Error("Krea 模型 ID 不能为空");
|
||||
}
|
||||
if (normalized.startsWith("/generate/")) return normalized;
|
||||
if (normalized.startsWith("generate/")) return `/${normalized}`;
|
||||
if (/^(image|video|enhance)\//i.test(normalized)) return `/generate/${normalized}`;
|
||||
throw new Error("Krea 模型 ID 需要使用官方 /generate/... endpoint");
|
||||
}
|
||||
|
||||
function detectExtensionFromContentType(contentType?: string | null) {
|
||||
const normalized = String(contentType || "").toLowerCase();
|
||||
if (normalized.includes("jpeg") || normalized.includes("jpg")) return "jpg";
|
||||
if (normalized.includes("webp")) return "webp";
|
||||
if (normalized.includes("gif")) return "gif";
|
||||
if (normalized.includes("png")) return "png";
|
||||
if (normalized.includes("mp4")) return "mp4";
|
||||
if (normalized.includes("quicktime")) return "mov";
|
||||
if (normalized.includes("webm")) return "webm";
|
||||
return "bin";
|
||||
}
|
||||
|
||||
function generateFilename(prefix: string, extension: string) {
|
||||
const timestamp = Date.now();
|
||||
const random = Math.random().toString(36).slice(2, 8);
|
||||
return `${prefix}-${timestamp}-${random}.${extension}`;
|
||||
}
|
||||
|
||||
function extractDataUriParts(dataUri: string) {
|
||||
const match = dataUri.match(/^data:([^;]+);base64,(.+)$/);
|
||||
if (!match) {
|
||||
throw new Error("不支持的 Data URI 格式");
|
||||
}
|
||||
return {
|
||||
mimeType: match[1],
|
||||
buffer: Buffer.from(match[2], "base64"),
|
||||
};
|
||||
}
|
||||
|
||||
async function uploadDataUriAsset(dataUri: string, prefix: "krea-input" | "krea-video") {
|
||||
const { mimeType, buffer } = extractDataUriParts(dataUri);
|
||||
const extension = detectExtensionFromContentType(mimeType);
|
||||
const arrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const uploaded = await uploadFile(generateFilename(prefix, extension), arrayBuffer);
|
||||
return uploaded.url;
|
||||
}
|
||||
|
||||
async function normalizeAssetUrl(url?: string | null) {
|
||||
const normalized = normalizeText(url);
|
||||
if (!normalized) return "";
|
||||
if (/^https?:\/\//i.test(normalized)) return normalized;
|
||||
if (/^data:(image|video)\//i.test(normalized)) {
|
||||
return uploadDataUriAsset(
|
||||
normalized,
|
||||
/^data:video\//i.test(normalized) ? "krea-video" : "krea-input"
|
||||
);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function isVideoUrl(url: string, mediaType?: string | null) {
|
||||
const normalizedMediaType = String(mediaType || "").toLowerCase();
|
||||
if (normalizedMediaType.startsWith("video/")) return true;
|
||||
return /\.(mp4|mov|webm|m4v)(\?|#|$)/i.test(url) || /^data:video\//i.test(url);
|
||||
}
|
||||
|
||||
function extractPromptAndAssets(messages: Array<{ role: string; content: any }>) {
|
||||
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
||||
if (!lastUserMessage) {
|
||||
throw new Error("未找到用户输入内容");
|
||||
}
|
||||
|
||||
let prompt = "";
|
||||
const imageUrls: string[] = [];
|
||||
const videoUrls: string[] = [];
|
||||
|
||||
const pushAssetUrl = (value?: string, mediaType?: string | null) => {
|
||||
if (!value) return;
|
||||
const normalized = String(value || "").trim();
|
||||
if (!normalized) return;
|
||||
const target = isVideoUrl(normalized, mediaType) ? videoUrls : imageUrls;
|
||||
if (!target.includes(normalized)) {
|
||||
target.push(normalized);
|
||||
}
|
||||
};
|
||||
|
||||
const readContentPart = (part: any) => {
|
||||
if (!part) return;
|
||||
if (part.type === "text" && typeof part.text === "string") {
|
||||
prompt = `${prompt} ${part.text}`.trim();
|
||||
return;
|
||||
}
|
||||
if (part.type === "image_url" && part.image_url?.url) {
|
||||
pushAssetUrl(part.image_url.url, "image/jpeg");
|
||||
return;
|
||||
}
|
||||
if (part.type === "file") {
|
||||
pushAssetUrl(part.url || part.data, part.mediaType || part.mimeType);
|
||||
return;
|
||||
}
|
||||
if (part.type === "image") {
|
||||
pushAssetUrl(part.image || part.url || part.data, part.mediaType || "image/jpeg");
|
||||
return;
|
||||
}
|
||||
if (part.type === "video") {
|
||||
pushAssetUrl(part.video || part.url || part.data, part.mediaType || "video/mp4");
|
||||
}
|
||||
};
|
||||
|
||||
if (typeof lastUserMessage.content === "string") {
|
||||
prompt = normalizeText(lastUserMessage.content);
|
||||
} else if (Array.isArray(lastUserMessage.content)) {
|
||||
for (const part of lastUserMessage.content) {
|
||||
readContentPart(part);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
prompt: normalizeText(prompt) || "生成内容",
|
||||
imageUrls,
|
||||
videoUrls,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeAspectRatio(value?: string, fallback = "16:9") {
|
||||
const normalized = normalizeText(value || fallback);
|
||||
if (
|
||||
[
|
||||
"21:9",
|
||||
"16:9",
|
||||
"9:16",
|
||||
"4:3",
|
||||
"3:4",
|
||||
"1:1",
|
||||
"9:21",
|
||||
"auto",
|
||||
].includes(normalized)
|
||||
) {
|
||||
return normalized;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
|
||||
function roundToMultiple(value: number, multiple = 16) {
|
||||
return Math.max(multiple, Math.round(value / multiple) * multiple);
|
||||
}
|
||||
|
||||
function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" {
|
||||
const normalized = normalizeText(imageSize).toUpperCase();
|
||||
if (normalized === "1K" || normalized === "2K" || normalized === "4K") {
|
||||
return normalized;
|
||||
}
|
||||
return "2K";
|
||||
}
|
||||
|
||||
function resolveImageDimensions(aspectRatio?: string, imageSize?: string) {
|
||||
const size = normalizeImageSizeLabel(imageSize);
|
||||
const base = size === "4K" ? 2048 : size === "2K" ? 1536 : 1024;
|
||||
const ratio = normalizeAspectRatio(aspectRatio, "1:1");
|
||||
if (ratio === "auto" || !ratio.includes(":")) {
|
||||
return { width: base, height: base, imageSize: size };
|
||||
}
|
||||
|
||||
const [rawW, rawH] = ratio.split(":").map((item) => Number(item));
|
||||
const ratioW = Number.isFinite(rawW) && rawW > 0 ? rawW : 1;
|
||||
const ratioH = Number.isFinite(rawH) && rawH > 0 ? rawH : 1;
|
||||
const landscape = ratioW >= ratioH;
|
||||
const width = landscape ? base : Math.round((base * ratioW) / ratioH);
|
||||
const height = landscape ? Math.round((base * ratioH) / ratioW) : base;
|
||||
return {
|
||||
width: clamp(roundToMultiple(width), 256, 2048),
|
||||
height: clamp(roundToMultiple(height), 256, 2048),
|
||||
imageSize: size,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveVideoDimensions(ratio?: string, resolution?: string) {
|
||||
const normalizedResolution = normalizeText(resolution).toLowerCase();
|
||||
const baseHeight = normalizedResolution === "1080p" ? 1080 : 720;
|
||||
const aspectRatio = normalizeAspectRatio(ratio, "16:9");
|
||||
if (aspectRatio === "9:16") {
|
||||
return { width: baseHeight, height: Math.round((baseHeight * 16) / 9) };
|
||||
}
|
||||
if (aspectRatio === "1:1") {
|
||||
return { width: baseHeight, height: baseHeight };
|
||||
}
|
||||
if (aspectRatio === "4:3") {
|
||||
return { width: Math.round((baseHeight * 4) / 3), height: baseHeight };
|
||||
}
|
||||
if (aspectRatio === "3:4") {
|
||||
return { width: baseHeight, height: Math.round((baseHeight * 4) / 3) };
|
||||
}
|
||||
if (aspectRatio === "21:9") {
|
||||
return { width: Math.round((baseHeight * 21) / 9), height: baseHeight };
|
||||
}
|
||||
return { width: Math.round((baseHeight * 16) / 9), height: baseHeight };
|
||||
}
|
||||
|
||||
function normalizeSeed(seed?: number) {
|
||||
if (!Number.isFinite(Number(seed))) return undefined;
|
||||
return clamp(Math.round(Number(seed)), 0, 4294967295);
|
||||
}
|
||||
|
||||
function normalizeBatchSize(value?: number) {
|
||||
if (!Number.isFinite(Number(value))) return 1;
|
||||
return clamp(Math.round(Number(value)), 1, 4);
|
||||
}
|
||||
|
||||
function normalizeVideoDuration(duration: unknown, endpoint: string) {
|
||||
const value = Number.isFinite(Number(duration)) ? Math.round(Number(duration)) : 5;
|
||||
if (endpoint.includes("/minimax/hailuo-02") || endpoint.includes("/minimax/hailuo-2.3")) {
|
||||
return clamp(value || 6, 6, 10);
|
||||
}
|
||||
if (endpoint.includes("/google/veo")) {
|
||||
return clamp(value || 8, 4, 8);
|
||||
}
|
||||
if (endpoint.includes("/openai/sora-2")) {
|
||||
return clamp(value || 4, 4, 20);
|
||||
}
|
||||
if (endpoint.includes("/kling/")) {
|
||||
return value >= 8 ? 10 : 5;
|
||||
}
|
||||
if (endpoint.includes("/alibaba/wan-2.5")) {
|
||||
return clamp(value || 5, 5, 10);
|
||||
}
|
||||
return value >= 8 ? 10 : 5;
|
||||
}
|
||||
|
||||
function normalizeVideoResolution(resolution: unknown, endpoint: string) {
|
||||
const normalized = normalizeText(String(resolution || "")).toLowerCase();
|
||||
if (endpoint.includes("/minimax/")) {
|
||||
if (normalized === "1080p") return "1080p";
|
||||
if (normalized === "512p" || normalized === "480p") return "512p";
|
||||
return "768p";
|
||||
}
|
||||
if (normalized === "1080p" || normalized === "480p") return normalized;
|
||||
return "720p";
|
||||
}
|
||||
|
||||
function normalizeVideoAspectRatio(ratio: unknown, endpoint: string) {
|
||||
const normalized = normalizeAspectRatio(String(ratio || ""), "16:9");
|
||||
if (endpoint.includes("/openai/sora-2")) {
|
||||
return normalized === "9:16" ? "9:16" : "16:9";
|
||||
}
|
||||
if (endpoint.includes("/google/veo") || endpoint.includes("/kling/")) {
|
||||
return ["16:9", "9:16", "1:1"].includes(normalized) ? normalized : "16:9";
|
||||
}
|
||||
return normalized === "auto" ? "16:9" : normalized;
|
||||
}
|
||||
|
||||
function buildStyleImages(imageUrls: string[]) {
|
||||
return imageUrls.map((url) => ({ url, strength: 1 }));
|
||||
}
|
||||
|
||||
function buildTaggedReferences(imageUrls: string[], tagPrefix = "ref", max = 3) {
|
||||
return imageUrls.slice(0, max).map((url, index) => ({
|
||||
url,
|
||||
tag: `${tagPrefix}_${index + 1}`.replace(/[^a-z0-9_]/gi, "_").toLowerCase(),
|
||||
}));
|
||||
}
|
||||
|
||||
function collectHttpUrls(payload: unknown): string[] {
|
||||
const urls = new Set<string>();
|
||||
const visit = (value: unknown) => {
|
||||
if (typeof value === "string") {
|
||||
const normalized = value.trim();
|
||||
if (/^https?:\/\//i.test(normalized)) {
|
||||
urls.add(normalized);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach(visit);
|
||||
return;
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
Object.values(value as Record<string, unknown>).forEach(visit);
|
||||
}
|
||||
};
|
||||
visit(payload);
|
||||
return Array.from(urls);
|
||||
}
|
||||
|
||||
async function downloadRemoteAsset(url: string) {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 120000);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
cache: "no-store",
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`下载 Krea 输出失败: ${response.status}`);
|
||||
}
|
||||
return {
|
||||
buffer: Buffer.from(await response.arrayBuffer()),
|
||||
contentType: response.headers.get("content-type"),
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadGeneratedBuffer(buffer: Buffer, extension: string): Promise<UploadedImage> {
|
||||
const thumbnailBuffer = await sharp(buffer)
|
||||
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
||||
.jpeg({ quality: 80 })
|
||||
.toBuffer();
|
||||
|
||||
const originalArrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const originalUploaded = await uploadFile(
|
||||
generateFilename("krea-image-orig", extension === "bin" ? "png" : extension),
|
||||
originalArrayBuffer
|
||||
);
|
||||
|
||||
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
||||
thumbnailBuffer.byteOffset,
|
||||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const thumbnailUploaded = await uploadFile(
|
||||
generateFilename("krea-image-thumb", "jpg"),
|
||||
thumbnailArrayBuffer
|
||||
);
|
||||
|
||||
return {
|
||||
thumbnail: thumbnailUploaded.url,
|
||||
original: originalUploaded.url,
|
||||
};
|
||||
}
|
||||
|
||||
async function uploadKreaImages(payload: unknown) {
|
||||
const uploadedImages: UploadedImage[] = [];
|
||||
for (const url of collectHttpUrls(payload)) {
|
||||
try {
|
||||
const { buffer, contentType } = await downloadRemoteAsset(url);
|
||||
if (!String(contentType || "").startsWith("image/")) continue;
|
||||
uploadedImages.push(await uploadGeneratedBuffer(buffer, detectExtensionFromContentType(contentType)));
|
||||
} catch (error) {
|
||||
console.warn("[Krea] Failed to archive image output:", error);
|
||||
}
|
||||
}
|
||||
if (uploadedImages.length === 0) {
|
||||
throw new Error("Krea 未返回可用图片");
|
||||
}
|
||||
return uploadedImages;
|
||||
}
|
||||
|
||||
async function uploadVideoToStorage(remoteVideoUrl: string) {
|
||||
try {
|
||||
const { buffer, contentType } = await downloadRemoteAsset(remoteVideoUrl);
|
||||
const extension = detectExtensionFromContentType(contentType || "video/mp4");
|
||||
const arrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const uploaded = await uploadFile(
|
||||
generateFilename("krea-video", extension === "bin" ? "mp4" : extension),
|
||||
arrayBuffer
|
||||
);
|
||||
return uploaded.url;
|
||||
} catch (error) {
|
||||
console.warn("[Krea] Failed to archive video output, using temporary URL:", error);
|
||||
return remoteVideoUrl;
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveKreaVideoUrl(payload: unknown) {
|
||||
const outputUrls = collectHttpUrls(payload);
|
||||
if (outputUrls.length === 0) {
|
||||
throw new Error("Krea 未返回可用视频地址");
|
||||
}
|
||||
const videoUrl =
|
||||
outputUrls.find((url) => /\.(mp4|mov|webm)(\?|#|$)/i.test(url) || /video/i.test(url)) ||
|
||||
outputUrls[0]!;
|
||||
return uploadVideoToStorage(videoUrl);
|
||||
}
|
||||
|
||||
function formatImageMarkdown(images: UploadedImage[], imageSize: string) {
|
||||
return images
|
||||
.map((image) => ``)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
function formatVideoMarkdown(videoUrl: string) {
|
||||
return `<video src="${videoUrl}" controls style="max-width: 100%; border-radius: 8px; margin-bottom: 12px;"></video>\n\n[下载视频](${videoUrl})`;
|
||||
}
|
||||
|
||||
async function requestKreaJob(
|
||||
baseURL: string,
|
||||
apiKey: string,
|
||||
endpoint: string,
|
||||
body: Record<string, any>
|
||||
) {
|
||||
const response = await fetch(joinEndpoint(baseURL, endpoint), {
|
||||
method: "POST",
|
||||
headers: buildHeaders(apiKey),
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const rawText = await response.text();
|
||||
let payload: any = null;
|
||||
try {
|
||||
payload = rawText ? JSON.parse(rawText) : null;
|
||||
} catch {
|
||||
payload = rawText;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
console.error("[Krea] Job request failed:", {
|
||||
endpoint,
|
||||
status: response.status,
|
||||
requestFields: Object.keys(body),
|
||||
response: payload,
|
||||
});
|
||||
const message =
|
||||
typeof payload?.error === "string"
|
||||
? payload.error
|
||||
: typeof payload?.message === "string"
|
||||
? payload.message
|
||||
: rawText || "Krea 请求失败";
|
||||
throw new Error(`${message} (HTTP ${response.status})`);
|
||||
}
|
||||
|
||||
const jobId = normalizeText(payload?.job_id || payload?.id);
|
||||
if (!jobId && Array.isArray(payload?.result?.urls) && payload.result.urls.length > 0) {
|
||||
return payload;
|
||||
}
|
||||
if (!jobId) {
|
||||
throw new Error("Krea 未返回任务 ID");
|
||||
}
|
||||
return { ...payload, job_id: jobId };
|
||||
}
|
||||
|
||||
async function waitForKreaJob(
|
||||
baseURL: string,
|
||||
apiKey: string,
|
||||
jobId: string,
|
||||
kind: KreaModelKind
|
||||
) {
|
||||
const isVideo = kind === "video";
|
||||
const deadline = Date.now() + (isVideo ? VIDEO_TIMEOUT_MS : IMAGE_TIMEOUT_MS);
|
||||
const pollInterval = isVideo ? VIDEO_POLL_INTERVAL_MS : IMAGE_POLL_INTERVAL_MS;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
const response = await fetch(joinEndpoint(baseURL, `/jobs/${jobId}`), {
|
||||
method: "GET",
|
||||
headers: buildHeaders(apiKey),
|
||||
cache: "no-store",
|
||||
});
|
||||
|
||||
const rawText = await response.text();
|
||||
let payload: any = null;
|
||||
try {
|
||||
payload = rawText ? JSON.parse(rawText) : null;
|
||||
} catch {
|
||||
payload = rawText;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const message =
|
||||
typeof payload?.error === "string"
|
||||
? payload.error
|
||||
: typeof payload?.message === "string"
|
||||
? payload.message
|
||||
: rawText || "Krea 轮询失败";
|
||||
throw new Error(`${message} (HTTP ${response.status})`);
|
||||
}
|
||||
|
||||
const status = normalizeText(payload?.status).toLowerCase();
|
||||
if (status === "completed") {
|
||||
return payload;
|
||||
}
|
||||
if (status === "failed" || status === "cancelled") {
|
||||
throw new Error(String(payload?.error || payload?.message || "Krea 任务失败"));
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
||||
}
|
||||
|
||||
throw new Error("Krea 任务超时,请稍后重试");
|
||||
}
|
||||
|
||||
function getRequiredTopazModel(endpoint: string) {
|
||||
if (endpoint.includes("/standard-enhance")) return "Standard V2";
|
||||
if (endpoint.includes("/bloom-enhance")) return "Reimagine";
|
||||
if (endpoint.includes("/generative-enhance")) return "Redefine";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function getRequiredModelValue(endpoint: string) {
|
||||
if (endpoint.includes("/seedream-3")) return "seedream-3-0-t2i-250415";
|
||||
if (endpoint.includes("/minimax/hailuo")) return "video-01";
|
||||
return getRequiredTopazModel(endpoint);
|
||||
}
|
||||
|
||||
function buildKreaBody(params: {
|
||||
endpoint: string;
|
||||
kind: KreaModelKind;
|
||||
requestFields: string[];
|
||||
requiredFields: string[];
|
||||
prompt: string;
|
||||
imageUrls: string[];
|
||||
videoUrls: string[];
|
||||
imageParams?: KreaImageParams;
|
||||
videoParams?: KreaVideoParams;
|
||||
}) {
|
||||
const {
|
||||
endpoint,
|
||||
kind,
|
||||
requestFields,
|
||||
requiredFields,
|
||||
prompt,
|
||||
imageUrls,
|
||||
videoUrls,
|
||||
imageParams,
|
||||
videoParams,
|
||||
} = params;
|
||||
const fields = new Set(requestFields);
|
||||
const required = new Set(requiredFields);
|
||||
const body: Record<string, any> = {};
|
||||
const isVideo = kind === "video";
|
||||
const ratio = isVideo
|
||||
? normalizeVideoAspectRatio(videoParams?.ratio, endpoint)
|
||||
: normalizeAspectRatio(imageParams?.aspectRatio, "1:1");
|
||||
const videoResolution = normalizeVideoResolution(videoParams?.resolution, endpoint);
|
||||
const imageDimensions = resolveImageDimensions(imageParams?.aspectRatio, imageParams?.imageSize);
|
||||
const videoDimensions = resolveVideoDimensions(ratio, videoResolution);
|
||||
|
||||
if (fields.has("prompt")) body.prompt = prompt;
|
||||
if (fields.has("batchSize")) body.batchSize = normalizeBatchSize(imageParams?.imageCount);
|
||||
|
||||
if (fields.has("width")) body.width = isVideo ? videoDimensions.width : imageDimensions.width;
|
||||
if (fields.has("height")) body.height = isVideo ? videoDimensions.height : imageDimensions.height;
|
||||
if (fields.has("aspectRatio")) body.aspectRatio = ratio === "auto" ? "1:1" : ratio;
|
||||
if (fields.has("ratio")) body.ratio = ratio === "auto" ? "16:9" : ratio;
|
||||
if (fields.has("resolution")) {
|
||||
body.resolution = isVideo ? videoResolution : normalizeImageSizeLabel(imageParams?.imageSize);
|
||||
}
|
||||
|
||||
const seed = normalizeSeed(isVideo ? videoParams?.seed : imageParams?.seed);
|
||||
if (fields.has("seed") && seed !== undefined) body.seed = seed;
|
||||
if (fields.has("negative_prompt") && imageParams?.negativePrompt) {
|
||||
body.negative_prompt = imageParams.negativePrompt;
|
||||
}
|
||||
if (fields.has("skipPromptExpansion") && typeof imageParams?.promptExtend === "boolean") {
|
||||
body.skipPromptExpansion = !imageParams.promptExtend;
|
||||
}
|
||||
if (fields.has("enablePromptExpansion")) {
|
||||
body.enablePromptExpansion = true;
|
||||
}
|
||||
if (fields.has("expandPrompt")) {
|
||||
body.expandPrompt = true;
|
||||
}
|
||||
if (fields.has("duration")) {
|
||||
body.duration = normalizeVideoDuration(videoParams?.duration, endpoint);
|
||||
}
|
||||
if (fields.has("generateAudio") && typeof videoParams?.generateAudio === "boolean") {
|
||||
body.generateAudio = videoParams.generateAudio;
|
||||
}
|
||||
if (fields.has("loop")) {
|
||||
body.loop = false;
|
||||
}
|
||||
if (fields.has("mode")) {
|
||||
body.mode = endpoint.includes("/kling/kling-2") ? "master" : "std";
|
||||
}
|
||||
if (fields.has("model")) {
|
||||
const modelValue = getRequiredModelValue(endpoint);
|
||||
if (modelValue) body.model = modelValue;
|
||||
}
|
||||
if (fields.has("output_format")) body.output_format = "jpg";
|
||||
if (fields.has("backgroundImage")) body.backgroundImage = "auto";
|
||||
|
||||
if (fields.has("imageUrls") && imageUrls.length > 0) {
|
||||
body.imageUrls = imageUrls.slice(0, 8);
|
||||
}
|
||||
if (fields.has("imageUrl") && imageUrls[0]) {
|
||||
body.imageUrl = imageUrls[0];
|
||||
}
|
||||
if (fields.has("image_url") && imageUrls[0]) {
|
||||
body.image_url = imageUrls[0];
|
||||
}
|
||||
if (fields.has("styleImages") && imageUrls.length > 0 && !body.imageUrls && !body.imageUrl) {
|
||||
body.styleImages = buildStyleImages(imageUrls.slice(0, 4));
|
||||
}
|
||||
if (fields.has("imageStyleRefs") && imageUrls.length > 1) {
|
||||
body.imageStyleRefs = buildStyleImages(imageUrls.slice(1, 5));
|
||||
}
|
||||
if (fields.has("characterReferenceImages") && imageUrls.length > 0) {
|
||||
body.characterReferenceImages = imageUrls.slice(0, 4);
|
||||
}
|
||||
|
||||
if (fields.has("referenceImages") && imageUrls.length > 0) {
|
||||
body.referenceImages = endpoint.includes("/image/runway/gen-4-image")
|
||||
? buildTaggedReferences(imageUrls, "ref", 3)
|
||||
: imageUrls.slice(0, endpoint.includes("/seedance-2") ? 9 : 4);
|
||||
}
|
||||
|
||||
if (fields.has("startImage") && imageUrls[0]) body.startImage = imageUrls[0];
|
||||
if (fields.has("endImage") && imageUrls[1]) body.endImage = imageUrls[1];
|
||||
if (fields.has("init_image_url") && imageUrls[0]) body.init_image_url = imageUrls[0];
|
||||
if (fields.has("references") && imageUrls[0]) {
|
||||
body.references = [{ type: "image", uri: imageUrls[0] }];
|
||||
}
|
||||
if (fields.has("subjects") && imageUrls[0] && videoParams?.mode === "reference-images") {
|
||||
body.subjects = [{ type: "character", image: [imageUrls[0]] }];
|
||||
}
|
||||
if (fields.has("styleReferences") && imageUrls.length > 0) {
|
||||
body.styleReferences = buildTaggedReferences(imageUrls, "style", 3);
|
||||
}
|
||||
if (fields.has("elementReferences") && imageUrls.length > 0) {
|
||||
body.elementReferences = [
|
||||
{
|
||||
tag: "element_1",
|
||||
images: imageUrls.slice(0, 3),
|
||||
},
|
||||
];
|
||||
}
|
||||
if (fields.has("videoReference") && videoUrls[0]) {
|
||||
body.videoReference = { url: videoUrls[0], tag: "video_1" };
|
||||
}
|
||||
if (fields.has("startVideo") && videoUrls[0]) body.startVideo = videoUrls[0];
|
||||
if (fields.has("init_video") && videoUrls[0]) body.init_video = videoUrls[0];
|
||||
if (fields.has("referenceVideos") && videoUrls.length > 0) {
|
||||
body.referenceVideos = videoUrls.slice(0, 3);
|
||||
}
|
||||
|
||||
if (required.has("referenceImages") && !body.referenceImages) {
|
||||
throw new Error("该 Krea 模型需要至少 1 张参考图");
|
||||
}
|
||||
if ((required.has("imageUrl") || required.has("image_url")) && !body.imageUrl && !body.image_url) {
|
||||
throw new Error("该 Krea 模型需要上传图片");
|
||||
}
|
||||
if (required.has("init_video") && !body.init_video) {
|
||||
throw new Error("该 Krea 模型需要上传视频");
|
||||
}
|
||||
|
||||
return { body, imageSize: imageDimensions.imageSize };
|
||||
}
|
||||
|
||||
export function createKrea({ apiKey, baseURL }: KreaConfig) {
|
||||
return {
|
||||
provider: "krea",
|
||||
chat: (modelId: string) => ({
|
||||
provider: "krea",
|
||||
doGenerate: async (options: {
|
||||
messages: Array<{ role: string; content: any }>;
|
||||
imageParams?: KreaImageParams;
|
||||
videoParams?: KreaVideoParams;
|
||||
overrideModelId?: string;
|
||||
}) => {
|
||||
try {
|
||||
const activeModelId = normalizeText(options.overrideModelId || modelId);
|
||||
const endpoint = resolveKreaEndpoint(activeModelId);
|
||||
const preset = getKreaModelPresetById(endpoint);
|
||||
const kind =
|
||||
preset?.kind ||
|
||||
inferKreaModelKind(endpoint) ||
|
||||
(options.videoParams ? "video" : "image");
|
||||
const { prompt, imageUrls, videoUrls } = extractPromptAndAssets(options.messages);
|
||||
const normalizedImageUrls = (
|
||||
await Promise.all(imageUrls.map((url) => normalizeAssetUrl(url)))
|
||||
).filter(Boolean);
|
||||
const normalizedVideoUrls = (
|
||||
await Promise.all(videoUrls.map((url) => normalizeAssetUrl(url)))
|
||||
).filter(Boolean);
|
||||
|
||||
const requestFields =
|
||||
preset?.requestFields ||
|
||||
(kind === "video"
|
||||
? ["prompt", "aspectRatio", "duration", "startImage", "endImage", "referenceImages"]
|
||||
: ["prompt", "width", "height", "imageUrls", "imageUrl", "batchSize"]);
|
||||
const requiredFields = preset?.requiredFields || ["prompt"];
|
||||
const { body, imageSize } = buildKreaBody({
|
||||
endpoint,
|
||||
kind,
|
||||
requestFields,
|
||||
requiredFields,
|
||||
prompt,
|
||||
imageUrls: normalizedImageUrls,
|
||||
videoUrls: normalizedVideoUrls,
|
||||
imageParams: options.imageParams,
|
||||
videoParams: options.videoParams,
|
||||
});
|
||||
|
||||
console.log("[Krea] Submitting job:", {
|
||||
endpoint,
|
||||
kind,
|
||||
requestFields: Object.keys(body),
|
||||
imageCount: normalizedImageUrls.length,
|
||||
videoCount: normalizedVideoUrls.length,
|
||||
});
|
||||
|
||||
const job = await requestKreaJob(baseURL, apiKey, endpoint, body);
|
||||
const output = job?.result?.urls?.length
|
||||
? job
|
||||
: await waitForKreaJob(baseURL, apiKey, job.job_id, kind);
|
||||
|
||||
const text =
|
||||
kind === "video"
|
||||
? formatVideoMarkdown(await resolveKreaVideoUrl(output?.result ?? output))
|
||||
: formatImageMarkdown(await uploadKreaImages(output?.result ?? output), imageSize);
|
||||
|
||||
return {
|
||||
text,
|
||||
finishReason: "stop" as const,
|
||||
usage: {
|
||||
promptTokens: 0,
|
||||
completionTokens: text.length,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("[Krea] Generation failed:", error);
|
||||
const kind =
|
||||
inferKreaModelKind(options.overrideModelId || modelId) ||
|
||||
(options.videoParams ? "video" : "image");
|
||||
return {
|
||||
text:
|
||||
error instanceof Error
|
||||
? `${kind === "video" ? "视频" : "图片"}生成失败: ${error.message}`
|
||||
: "Krea 生成失败,请稍后重试",
|
||||
finishReason: "error" as const,
|
||||
usage: {
|
||||
promptTokens: 0,
|
||||
completionTokens: 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
},
|
||||
doStream: async function* (options: {
|
||||
messages: Array<{ role: string; content: any }>;
|
||||
imageParams?: KreaImageParams;
|
||||
videoParams?: KreaVideoParams;
|
||||
overrideModelId?: string;
|
||||
}) {
|
||||
const result = await this.doGenerate(options);
|
||||
yield {
|
||||
type: "text-delta",
|
||||
textDelta: result.text,
|
||||
};
|
||||
},
|
||||
modelId,
|
||||
specificationVersion: "v1" as const,
|
||||
defaultObjectGenerationMode: "json" as const,
|
||||
}),
|
||||
};
|
||||
}
|
||||
53
lib/ai/runtime/agent-media-markdown.ts
Normal file
53
lib/ai/runtime/agent-media-markdown.ts
Normal file
@ -0,0 +1,53 @@
|
||||
export interface AgentToolImageMarkdownItem {
|
||||
originalUrl: string;
|
||||
thumbnailUrl?: string | null;
|
||||
imageSize?: string;
|
||||
}
|
||||
|
||||
export function prependImageModelBadge(
|
||||
text: string,
|
||||
modelDisplayName?: string | null,
|
||||
): string {
|
||||
const normalizedText = String(text || "").trim();
|
||||
if (!normalizedText) return normalizedText;
|
||||
|
||||
const normalizedModelName = String(modelDisplayName || "").trim();
|
||||
if (!normalizedModelName) return normalizedText;
|
||||
|
||||
if (
|
||||
/^\s*[◈◇◆🔹⬡✦•·]\s*[^\n]+/im.test(normalizedText) ||
|
||||
/^\s*(?:Routed to|Using model|使用模型|模型)\s*[::]/im.test(normalizedText)
|
||||
) {
|
||||
return normalizedText;
|
||||
}
|
||||
|
||||
return `◈ ${normalizedModelName}\n\n${normalizedText}`;
|
||||
}
|
||||
|
||||
export function buildAgentToolImageMarkdown(params: AgentToolImageMarkdownItem): string {
|
||||
const originalUrl = String(params.originalUrl || "").trim();
|
||||
const thumbnailUrl = String(params.thumbnailUrl || originalUrl).trim() || originalUrl;
|
||||
const imageSize = String(params.imageSize || "2K").trim() || "2K";
|
||||
return ``;
|
||||
}
|
||||
|
||||
export function buildAgentToolImagesMarkdown(items: AgentToolImageMarkdownItem[]) {
|
||||
return items
|
||||
.map((item) =>
|
||||
buildAgentToolImageMarkdown({
|
||||
originalUrl: item.originalUrl,
|
||||
thumbnailUrl: item.thumbnailUrl,
|
||||
imageSize: item.imageSize,
|
||||
}),
|
||||
)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
export function buildAgentToolVideoMarkdown(videoUrl: string) {
|
||||
const normalizedUrl = String(videoUrl || "").trim();
|
||||
if (!normalizedUrl) return "";
|
||||
return [
|
||||
`<video src="${normalizedUrl}" controls style="max-width: 100%; border-radius: 8px;"></video>`,
|
||||
`[下载视频](${normalizedUrl})`,
|
||||
].join("\n\n");
|
||||
}
|
||||
30
lib/ai/runtime/assistant-message-persistence.ts
Normal file
30
lib/ai/runtime/assistant-message-persistence.ts
Normal file
@ -0,0 +1,30 @@
|
||||
import { saveMessages } from "@/lib/db/queries";
|
||||
|
||||
export async function saveAssistantTextOnlyMessage(params: {
|
||||
chatId: string;
|
||||
messageId: string;
|
||||
text: string;
|
||||
videoStorageStatus?: "temporary" | "permanent" | "failed" | null;
|
||||
temporaryVideoUrl?: string | null;
|
||||
permanentVideoUrl?: string | null;
|
||||
archivedAt?: Date | null;
|
||||
videoMetadata?: Record<string, any> | null;
|
||||
}) {
|
||||
await saveMessages({
|
||||
messages: [
|
||||
{
|
||||
id: params.messageId,
|
||||
chatId: params.chatId,
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: params.text }],
|
||||
attachments: [],
|
||||
createdAt: new Date(),
|
||||
videoStorageStatus: params.videoStorageStatus ?? null,
|
||||
temporaryVideoUrl: params.temporaryVideoUrl ?? null,
|
||||
permanentVideoUrl: params.permanentVideoUrl ?? null,
|
||||
archivedAt: params.archivedAt ?? null,
|
||||
videoMetadata: params.videoMetadata ?? null,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
@ -48,6 +48,12 @@ export interface ResolvedDirectAgentToolExecution {
|
||||
publishToCanvas: boolean;
|
||||
}
|
||||
|
||||
export interface ResolvedDirectAgentToolContext {
|
||||
execution: ResolvedDirectAgentToolExecution;
|
||||
targetImageUrls: string[];
|
||||
targetImageUrl: string | null;
|
||||
}
|
||||
|
||||
export interface DirectAgentWorkingStateSnapshot {
|
||||
lastResolvedTargetImageUrls?: string[] | null;
|
||||
lastGeneratedImageUrls?: string[] | null;
|
||||
@ -162,8 +168,9 @@ export function resolveDirectAgentToolExecution(
|
||||
plan?: AgentExecutionPlan,
|
||||
): ResolvedDirectAgentToolExecution | null {
|
||||
const mergedToolCalls = collectAgentToolCalls(execution, plan);
|
||||
const toolNames = mergedToolCalls.map((tool) => tool.name);
|
||||
const hasToolCall = (toolName: string) => toolNames.includes(toolName);
|
||||
const toolNames = mergedToolCalls.map((tool) => String(tool.name || ""));
|
||||
const toolNameSet = new Set(toolNames);
|
||||
const hasToolCall = (toolName: string) => toolNameSet.has(toolName);
|
||||
const sourceTools = toolNames.filter((toolName): toolName is DirectAgentSourceToolName =>
|
||||
toolName === "useSelectedCanvasImage" ||
|
||||
toolName === "useLatestResultImage" ||
|
||||
@ -288,6 +295,52 @@ export function resolveDirectAgentToolExecution(
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveDirectAgentToolContext(params: {
|
||||
execution?: AgentPreparedExecution;
|
||||
plan?: AgentExecutionPlan;
|
||||
requestResolvedMediaInput?: ResolvedChatMediaInput | null;
|
||||
requestTargetResolution?: ResolvedTargetAssets | null;
|
||||
requestWorkingState?: DirectAgentWorkingStateSnapshot | null;
|
||||
}): ResolvedDirectAgentToolContext | null {
|
||||
const execution = resolveDirectAgentToolExecution(params.execution, params.plan);
|
||||
if (!execution) return null;
|
||||
|
||||
const targetImageUrls = pickAgentSourceImageUrls({
|
||||
sourcePreferences: execution.sourcePreferences,
|
||||
requestResolvedMediaInput: params.requestResolvedMediaInput,
|
||||
requestTargetResolution: params.requestTargetResolution,
|
||||
requestWorkingState: params.requestWorkingState,
|
||||
});
|
||||
|
||||
return {
|
||||
execution,
|
||||
targetImageUrls,
|
||||
targetImageUrl: targetImageUrls[0] || null,
|
||||
};
|
||||
}
|
||||
|
||||
export function isDirectAgentImageEditToolName(
|
||||
toolName?: DirectAgentToolName | null,
|
||||
): toolName is
|
||||
| "editImage"
|
||||
| "enhanceImage"
|
||||
| "removeBackground"
|
||||
| "resizeImage"
|
||||
| "extendImage"
|
||||
| "boxCutout" {
|
||||
switch (toolName) {
|
||||
case "editImage":
|
||||
case "enhanceImage":
|
||||
case "removeBackground":
|
||||
case "resizeImage":
|
||||
case "extendImage":
|
||||
case "boxCutout":
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function getDirectAgentToolLabel(capability: DirectAgentToolCapability): string {
|
||||
switch (capability) {
|
||||
case "analyze":
|
||||
|
||||
444
lib/ai/runtime/direct-agent-image-edit-executor.ts
Normal file
444
lib/ai/runtime/direct-agent-image-edit-executor.ts
Normal file
@ -0,0 +1,444 @@
|
||||
import {
|
||||
BoxCutoutServiceError,
|
||||
executeBoxCutoutJob,
|
||||
} from "@/lib/ai/box-cutout/service";
|
||||
import {
|
||||
executeExtendJob,
|
||||
ExtendServiceError,
|
||||
inferExtendOptionsFromText,
|
||||
} from "@/lib/ai/extend/service";
|
||||
import {
|
||||
executeResizeJob,
|
||||
inferResizeOptionsFromText,
|
||||
ResizeServiceError,
|
||||
} from "@/lib/ai/resize/service";
|
||||
import {
|
||||
executeRmbgJob,
|
||||
fetchImageBufferFromUrl,
|
||||
RmbgServiceError,
|
||||
} from "@/lib/ai/rmbg/service";
|
||||
import { generateCanvasImage } from "@/lib/canvas/generate-image";
|
||||
import { requestLongTaskResult } from "@/lib/client/long-task";
|
||||
import { getMediaAssetsByUserAndUrls } from "@/lib/db/queries";
|
||||
import type {
|
||||
AgentExecutionPlan,
|
||||
AgentPreparedExecution,
|
||||
} from "@/lib/ai/auto-agent-graph";
|
||||
import {
|
||||
findAgentToolCallArgs,
|
||||
type DirectAgentToolCapability,
|
||||
} from "@/lib/ai/runtime/direct-agent-execution";
|
||||
import { buildDirectAgentExecutionPrompt } from "@/lib/ai/runtime/optimized-prompt";
|
||||
import { buildAgentToolImagesMarkdown } from "@/lib/ai/runtime/agent-media-markdown";
|
||||
|
||||
interface DirectAgentImageParamsSnapshot {
|
||||
negativePrompt?: string;
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
}
|
||||
|
||||
type SaveGeneratedImages = (
|
||||
imageMarkdown: string,
|
||||
userId: string,
|
||||
chatId: string,
|
||||
messageId: string,
|
||||
prompt: string,
|
||||
modelId: string,
|
||||
options?: {
|
||||
sourceAssetIds?: string[];
|
||||
sourceImageUrls?: string[];
|
||||
relationType?: "generated_from" | "edited_from";
|
||||
sourceType?: string;
|
||||
projectId?: string | null;
|
||||
userPrompt?: string;
|
||||
optimizedPrompt?: string;
|
||||
requestedImageCount?: number;
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
},
|
||||
) => Promise<void>;
|
||||
|
||||
export interface DirectAgentImageEditExecutionResult {
|
||||
originalUrl: string;
|
||||
thumbnailUrl: string | null;
|
||||
imageSize: string;
|
||||
mediaAssetId: string | null;
|
||||
}
|
||||
|
||||
export interface ExecuteDirectAgentImageEditToolParams {
|
||||
capability: DirectAgentToolCapability;
|
||||
requestUrl: string;
|
||||
requestHeaders: Headers;
|
||||
userId: string;
|
||||
chatId: string;
|
||||
messageId: string;
|
||||
projectId?: string | null;
|
||||
targetAssetIds?: string[] | null;
|
||||
sourceImageUrl: string;
|
||||
sourceImageUrls: string[];
|
||||
requestTextContent: string;
|
||||
optimizedPrompt?: string;
|
||||
hiddenPromptPrefix?: string;
|
||||
finalModelId: string;
|
||||
imageParams?: DirectAgentImageParamsSnapshot | null;
|
||||
defaultImageQualityParams?: DirectAgentImageParamsSnapshot | null;
|
||||
smartAutoExecution?: AgentPreparedExecution;
|
||||
smartAutoPlan?: AgentExecutionPlan;
|
||||
saveGeneratedImages: SaveGeneratedImages;
|
||||
}
|
||||
|
||||
function toFiniteNumber(value: unknown): number | undefined {
|
||||
const normalized = Number(value);
|
||||
return Number.isFinite(normalized) ? normalized : undefined;
|
||||
}
|
||||
|
||||
async function executeDirectAgentEnhanceJob(params: {
|
||||
requestUrl: string;
|
||||
requestHeaders: Headers;
|
||||
imageUrl: string;
|
||||
prompt: string;
|
||||
chatId: string;
|
||||
messageId: string;
|
||||
}) {
|
||||
const endpoint = new URL("/api/enhance", params.requestUrl).toString();
|
||||
const cookieHeader = params.requestHeaders.get("cookie") || "";
|
||||
const result = await requestLongTaskResult({
|
||||
endpoint,
|
||||
payload: {
|
||||
imageUrl: params.imageUrl,
|
||||
prompt: params.prompt,
|
||||
chatId: params.chatId,
|
||||
messageId: params.messageId,
|
||||
},
|
||||
pollingParams: {
|
||||
prompt: params.prompt,
|
||||
chatId: params.chatId,
|
||||
messageId: params.messageId,
|
||||
sourceImageUrl: params.imageUrl,
|
||||
},
|
||||
timeoutMs: 8 * 60 * 1000,
|
||||
headers: cookieHeader
|
||||
? {
|
||||
cookie: cookieHeader,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
|
||||
if (!result?.success || !result?.original_url) {
|
||||
throw new Error(String(result?.error || result?.detail || "图像增强失败"));
|
||||
}
|
||||
|
||||
return {
|
||||
originalUrl: String(result.original_url),
|
||||
thumbnailUrl: String(result.thumbnail_url || result.original_url),
|
||||
imageSize: "2K",
|
||||
};
|
||||
}
|
||||
|
||||
export function isDirectAgentImageEditServiceError(error: unknown) {
|
||||
return (
|
||||
error instanceof RmbgServiceError ||
|
||||
error instanceof ResizeServiceError ||
|
||||
error instanceof ExtendServiceError ||
|
||||
error instanceof BoxCutoutServiceError
|
||||
);
|
||||
}
|
||||
|
||||
async function resolveDirectAgentResultAssetId(params: {
|
||||
userId: string;
|
||||
originalUrl: string;
|
||||
mediaAssetId?: string | null;
|
||||
capability: DirectAgentToolCapability;
|
||||
}) {
|
||||
if (params.mediaAssetId || !params.originalUrl) {
|
||||
return params.mediaAssetId || null;
|
||||
}
|
||||
|
||||
try {
|
||||
const [resultAsset] = await getMediaAssetsByUserAndUrls({
|
||||
userId: params.userId,
|
||||
urls: [params.originalUrl],
|
||||
});
|
||||
return resultAsset?.id || null;
|
||||
} catch (assetLookupError) {
|
||||
console.warn(
|
||||
`[Direct Agent Image Edit] Failed to lookup result asset after ${params.capability}:`,
|
||||
assetLookupError,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function finalizeDirectAgentImageEditResult(params: {
|
||||
userId: string;
|
||||
capability: DirectAgentToolCapability;
|
||||
originalUrl: string;
|
||||
thumbnailUrl?: string | null;
|
||||
imageSize: string;
|
||||
mediaAssetId?: string | null;
|
||||
}): Promise<DirectAgentImageEditExecutionResult> {
|
||||
return {
|
||||
originalUrl: params.originalUrl,
|
||||
thumbnailUrl: params.thumbnailUrl || null,
|
||||
imageSize: params.imageSize,
|
||||
mediaAssetId: await resolveDirectAgentResultAssetId({
|
||||
userId: params.userId,
|
||||
originalUrl: params.originalUrl,
|
||||
mediaAssetId: params.mediaAssetId,
|
||||
capability: params.capability,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function executeDirectAgentImageEditTool(
|
||||
params: ExecuteDirectAgentImageEditToolParams,
|
||||
): Promise<DirectAgentImageEditExecutionResult> {
|
||||
if (params.capability === "generic_edit") {
|
||||
const directEditPrompt =
|
||||
buildDirectAgentExecutionPrompt({
|
||||
originalText: params.requestTextContent || "",
|
||||
optimizedPrompt: params.optimizedPrompt,
|
||||
hiddenPromptPrefix: params.hiddenPromptPrefix,
|
||||
fallback: "在保持主体和整体构图稳定的前提下完成本轮图片编辑。",
|
||||
}) || "在保持主体和整体构图稳定的前提下完成本轮图片编辑。";
|
||||
const aspectRatio =
|
||||
params.imageParams?.aspectRatio || params.defaultImageQualityParams?.aspectRatio;
|
||||
const imageSize =
|
||||
params.imageParams?.imageSize || params.defaultImageQualityParams?.imageSize || "2K";
|
||||
const result = await generateCanvasImage({
|
||||
prompt: directEditPrompt,
|
||||
negativePrompt: params.imageParams?.negativePrompt,
|
||||
model: params.finalModelId,
|
||||
aspectRatio,
|
||||
imageSize,
|
||||
imageCount: 1,
|
||||
referenceImages: params.sourceImageUrls,
|
||||
debugLabel: "agent-direct-edit",
|
||||
debugMeta: {
|
||||
chatId: params.chatId,
|
||||
projectId: params.projectId || null,
|
||||
subIntent: params.smartAutoExecution?.routeSubIntent || null,
|
||||
},
|
||||
});
|
||||
|
||||
const firstImage = result.images[0];
|
||||
if (!firstImage?.originalUrl) {
|
||||
throw new Error("图片编辑完成但未返回结果图");
|
||||
}
|
||||
|
||||
const originalUrl = firstImage.originalUrl;
|
||||
const thumbnailUrl = firstImage.thumbnailUrl || null;
|
||||
const editResultMarkdown = buildAgentToolImagesMarkdown([
|
||||
{
|
||||
originalUrl,
|
||||
thumbnailUrl,
|
||||
imageSize,
|
||||
},
|
||||
]);
|
||||
|
||||
await params.saveGeneratedImages(
|
||||
editResultMarkdown,
|
||||
params.userId,
|
||||
params.chatId,
|
||||
params.messageId,
|
||||
directEditPrompt,
|
||||
result.modelRef || params.finalModelId,
|
||||
{
|
||||
sourceAssetIds: params.targetAssetIds || undefined,
|
||||
sourceImageUrls: params.sourceImageUrls,
|
||||
relationType: "edited_from",
|
||||
sourceType: "edited",
|
||||
projectId: params.projectId || null,
|
||||
userPrompt: params.requestTextContent,
|
||||
optimizedPrompt: directEditPrompt,
|
||||
requestedImageCount: 1,
|
||||
aspectRatio,
|
||||
imageSize,
|
||||
},
|
||||
);
|
||||
|
||||
return finalizeDirectAgentImageEditResult({
|
||||
userId: params.userId,
|
||||
capability: params.capability,
|
||||
originalUrl,
|
||||
thumbnailUrl,
|
||||
imageSize,
|
||||
mediaAssetId: null,
|
||||
});
|
||||
}
|
||||
|
||||
if (params.capability === "enhance") {
|
||||
const result = await executeDirectAgentEnhanceJob({
|
||||
requestUrl: params.requestUrl,
|
||||
requestHeaders: params.requestHeaders,
|
||||
imageUrl: params.sourceImageUrl,
|
||||
prompt: params.requestTextContent || "图像增强处理",
|
||||
chatId: params.chatId,
|
||||
messageId: params.messageId,
|
||||
});
|
||||
|
||||
return finalizeDirectAgentImageEditResult({
|
||||
userId: params.userId,
|
||||
capability: params.capability,
|
||||
originalUrl: result.originalUrl,
|
||||
thumbnailUrl: result.thumbnailUrl,
|
||||
imageSize: result.imageSize,
|
||||
mediaAssetId: null,
|
||||
});
|
||||
}
|
||||
|
||||
if (params.capability === "remove_background") {
|
||||
const sourceBuffer = await fetchImageBufferFromUrl(params.sourceImageUrl);
|
||||
const result = await executeRmbgJob({
|
||||
userId: params.userId,
|
||||
imageBuffer: sourceBuffer,
|
||||
prompt: params.requestTextContent || "抠图处理",
|
||||
chatId: params.chatId,
|
||||
messageId: params.messageId,
|
||||
sourceImageUrl: params.sourceImageUrl,
|
||||
});
|
||||
|
||||
return finalizeDirectAgentImageEditResult({
|
||||
userId: params.userId,
|
||||
capability: params.capability,
|
||||
originalUrl: result.original_url,
|
||||
thumbnailUrl: result.thumbnail_url,
|
||||
imageSize: "2K",
|
||||
mediaAssetId: result.mediaAssetId || null,
|
||||
});
|
||||
}
|
||||
|
||||
if (params.capability === "resize") {
|
||||
const plannedArgs = findAgentToolCallArgs(
|
||||
"resizeImage",
|
||||
params.smartAutoExecution,
|
||||
params.smartAutoPlan,
|
||||
);
|
||||
const inferredOptions = inferResizeOptionsFromText(params.requestTextContent);
|
||||
const canvasWidth = toFiniteNumber(
|
||||
plannedArgs?.canvasWidth ?? plannedArgs?.width ?? inferredOptions.canvasWidth,
|
||||
);
|
||||
const canvasHeight = toFiniteNumber(
|
||||
plannedArgs?.canvasHeight ?? plannedArgs?.height ?? inferredOptions.canvasHeight,
|
||||
);
|
||||
|
||||
if (!canvasWidth || !canvasHeight) {
|
||||
throw new ResizeServiceError(
|
||||
"请告诉我目标尺寸或比例,例如 1024x1024、4:5、16:9。",
|
||||
400,
|
||||
);
|
||||
}
|
||||
|
||||
const transparentBackground =
|
||||
plannedArgs?.transparentBackground === true ||
|
||||
plannedArgs?.transparentBackground === "true" ||
|
||||
inferredOptions.transparentBackground === true;
|
||||
const backgroundColor =
|
||||
typeof plannedArgs?.backgroundColor === "string"
|
||||
? plannedArgs.backgroundColor
|
||||
: inferredOptions.backgroundColor;
|
||||
const mimeType =
|
||||
typeof plannedArgs?.mimeType === "string" ? plannedArgs.mimeType : undefined;
|
||||
const result = await executeResizeJob({
|
||||
userId: params.userId,
|
||||
imageUrl: params.sourceImageUrl,
|
||||
prompt: params.requestTextContent || "尺寸调整处理",
|
||||
chatId: params.chatId,
|
||||
messageId: params.messageId,
|
||||
sourceImageUrl: params.sourceImageUrl,
|
||||
canvasWidth,
|
||||
canvasHeight,
|
||||
transparentBackground,
|
||||
backgroundColor,
|
||||
mimeType,
|
||||
});
|
||||
|
||||
return finalizeDirectAgentImageEditResult({
|
||||
userId: params.userId,
|
||||
capability: params.capability,
|
||||
originalUrl: result.original_url,
|
||||
thumbnailUrl: result.thumbnail_url,
|
||||
imageSize: `${result.canvas_width}x${result.canvas_height}`,
|
||||
mediaAssetId: result.mediaAssetId || null,
|
||||
});
|
||||
}
|
||||
|
||||
if (params.capability === "extend") {
|
||||
const plannedArgs = findAgentToolCallArgs(
|
||||
"extendImage",
|
||||
params.smartAutoExecution,
|
||||
params.smartAutoPlan,
|
||||
);
|
||||
const inferredOptions = inferExtendOptionsFromText(params.requestTextContent);
|
||||
const xScale = toFiniteNumber(plannedArgs?.xScale ?? inferredOptions.xScale);
|
||||
const yScale = toFiniteNumber(plannedArgs?.yScale ?? inferredOptions.yScale);
|
||||
const result = await executeExtendJob({
|
||||
userId: params.userId,
|
||||
imageUrl: params.sourceImageUrl,
|
||||
prompt: params.requestTextContent || "图像拓展处理",
|
||||
chatId: params.chatId,
|
||||
messageId: params.messageId,
|
||||
sourceImageUrl: params.sourceImageUrl,
|
||||
...(xScale ? { xScale } : {}),
|
||||
...(yScale ? { yScale } : {}),
|
||||
});
|
||||
|
||||
return finalizeDirectAgentImageEditResult({
|
||||
userId: params.userId,
|
||||
capability: params.capability,
|
||||
originalUrl: result.original_url,
|
||||
thumbnailUrl: result.thumbnail_url,
|
||||
imageSize: "2K",
|
||||
mediaAssetId: result.mediaAssetId || null,
|
||||
});
|
||||
}
|
||||
|
||||
if (params.capability === "box_cutout") {
|
||||
const plannedArgs = findAgentToolCallArgs(
|
||||
"boxCutout",
|
||||
params.smartAutoExecution,
|
||||
params.smartAutoPlan,
|
||||
);
|
||||
const cropRect =
|
||||
plannedArgs?.cropRect && typeof plannedArgs.cropRect === "object"
|
||||
? (plannedArgs.cropRect as Record<string, unknown>)
|
||||
: undefined;
|
||||
|
||||
if (!cropRect) {
|
||||
throw new BoxCutoutServiceError(
|
||||
"缺少框选区域,暂时无法执行框选抠图。",
|
||||
400,
|
||||
);
|
||||
}
|
||||
|
||||
const result = await executeBoxCutoutJob({
|
||||
userId: params.userId,
|
||||
imageUrl: params.sourceImageUrl,
|
||||
cropRect: {
|
||||
x: toFiniteNumber(cropRect.x),
|
||||
y: toFiniteNumber(cropRect.y),
|
||||
width: toFiniteNumber(cropRect.width),
|
||||
height: toFiniteNumber(cropRect.height),
|
||||
},
|
||||
paddingRatio: toFiniteNumber(plannedArgs?.paddingRatio),
|
||||
prompt: params.requestTextContent || "框选抠图处理",
|
||||
chatId: params.chatId,
|
||||
messageId: params.messageId,
|
||||
sourceNodeId:
|
||||
typeof plannedArgs?.sourceNodeId === "string"
|
||||
? plannedArgs.sourceNodeId
|
||||
: undefined,
|
||||
});
|
||||
|
||||
return finalizeDirectAgentImageEditResult({
|
||||
userId: params.userId,
|
||||
capability: params.capability,
|
||||
originalUrl: result.original_url,
|
||||
thumbnailUrl: result.thumbnail_url || null,
|
||||
imageSize: "2K",
|
||||
mediaAssetId: result.mediaAssetId || null,
|
||||
});
|
||||
}
|
||||
|
||||
throw new Error(`不支持的图片编辑工具能力:${params.capability}`);
|
||||
}
|
||||
238
lib/ai/runtime/direct-agent-media-executors.ts
Normal file
238
lib/ai/runtime/direct-agent-media-executors.ts
Normal file
@ -0,0 +1,238 @@
|
||||
import { generateCanvasImage } from "@/lib/canvas/generate-image";
|
||||
import {
|
||||
executeDirectVideoGeneration,
|
||||
type ExecuteDirectVideoGenerationParams,
|
||||
type ExecuteDirectVideoGenerationResult,
|
||||
} from "@/lib/ai/tools/generate-video";
|
||||
import type {
|
||||
AgentExecutionPlan,
|
||||
AgentPreparedExecution,
|
||||
} from "@/lib/ai/auto-agent-graph";
|
||||
import { findAgentToolCallArgs } from "@/lib/ai/runtime/direct-agent-execution";
|
||||
import { buildDirectAgentImageAnalysisAnswer } from "@/lib/ai/runtime/direct-agent-analysis-answer";
|
||||
import { buildAgentToolImagesMarkdown } from "@/lib/ai/runtime/agent-media-markdown";
|
||||
|
||||
interface DirectAgentImageParamsSnapshot {
|
||||
negativePrompt?: string;
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
imageCount?: number;
|
||||
}
|
||||
|
||||
type DirectAgentVideoMode = ExecuteDirectVideoGenerationParams["mode"];
|
||||
type DirectAgentVideoRatio = NonNullable<ExecuteDirectVideoGenerationParams["ratio"]>;
|
||||
|
||||
export interface DirectAgentImageAnalysisResult {
|
||||
summary: string;
|
||||
answer: string;
|
||||
}
|
||||
|
||||
export interface ExecuteDirectAgentImageAnalysisParams {
|
||||
requestTextContent: string;
|
||||
imageUrls: string[];
|
||||
context?: string;
|
||||
preferredModelId?: string;
|
||||
answerModelId?: string;
|
||||
assetIds?: string[];
|
||||
}
|
||||
|
||||
export interface DirectAgentImageGenerationResult {
|
||||
prompt: string;
|
||||
imageMarkdown: string;
|
||||
images: Array<{
|
||||
originalUrl: string;
|
||||
thumbnailUrl?: string | null;
|
||||
}>;
|
||||
model: string;
|
||||
modelRef?: string | null;
|
||||
imageSize?: string;
|
||||
imageCount: number;
|
||||
}
|
||||
|
||||
export interface ExecuteDirectAgentImageGenerationParams {
|
||||
requestTextContent: string;
|
||||
optimizedPrompt?: string;
|
||||
finalModelId: string;
|
||||
imageParams?: DirectAgentImageParamsSnapshot | null;
|
||||
defaultImageQualityParams?: DirectAgentImageParamsSnapshot | null;
|
||||
referenceImageUrls: string[];
|
||||
}
|
||||
|
||||
export interface DirectAgentVideoGenerationResult {
|
||||
prompt: string;
|
||||
mode: DirectAgentVideoMode;
|
||||
images: string[];
|
||||
videoUrl: string;
|
||||
result: ExecuteDirectVideoGenerationResult;
|
||||
}
|
||||
|
||||
export interface ExecuteDirectAgentVideoGenerationParams {
|
||||
requestTextContent: string;
|
||||
optimizedPrompt?: string;
|
||||
finalModelId: string;
|
||||
sourceImageUrls: string[];
|
||||
videoParams?: {
|
||||
mode?: string;
|
||||
resolution?: ExecuteDirectVideoGenerationParams["resolution"];
|
||||
ratio?: string;
|
||||
duration?: number;
|
||||
} | null;
|
||||
smartAutoExecution?: AgentPreparedExecution;
|
||||
smartAutoPlan?: AgentExecutionPlan;
|
||||
}
|
||||
|
||||
function normalizeDirectAgentVideoMode(value: unknown): DirectAgentVideoMode | undefined {
|
||||
const normalized = String(value || "").trim();
|
||||
switch (normalized) {
|
||||
case "text-to-video":
|
||||
case "first-frame":
|
||||
case "first-last-frame":
|
||||
case "reference-images":
|
||||
return normalized;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeDirectAgentVideoRatio(value: unknown): DirectAgentVideoRatio | undefined {
|
||||
const normalized = String(value || "").trim();
|
||||
switch (normalized) {
|
||||
case "16:9":
|
||||
case "4:3":
|
||||
case "1:1":
|
||||
case "3:4":
|
||||
case "9:16":
|
||||
case "21:9":
|
||||
case "adaptive":
|
||||
return normalized;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function inferDirectAgentVideoMode(imageUrls: string[]): DirectAgentVideoMode {
|
||||
if (imageUrls.length >= 2) return "first-last-frame";
|
||||
if (imageUrls.length >= 1) return "first-frame";
|
||||
return "text-to-video";
|
||||
}
|
||||
|
||||
function pickDirectAgentVideoImages(mode: DirectAgentVideoMode, imageUrls: string[]): string[] {
|
||||
switch (mode) {
|
||||
case "first-last-frame":
|
||||
return imageUrls.slice(0, 2);
|
||||
case "first-frame":
|
||||
return imageUrls.slice(0, 1);
|
||||
case "reference-images":
|
||||
return imageUrls.slice(0, 4);
|
||||
case "text-to-video":
|
||||
default:
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export async function executeDirectAgentImageAnalysis(
|
||||
params: ExecuteDirectAgentImageAnalysisParams,
|
||||
): Promise<DirectAgentImageAnalysisResult> {
|
||||
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
||||
const summary = await summarizeImageForExecutionReport({
|
||||
text: params.requestTextContent,
|
||||
imageUrls: params.imageUrls,
|
||||
context: params.context,
|
||||
preferredModelId: params.preferredModelId,
|
||||
assetIds: params.assetIds,
|
||||
});
|
||||
|
||||
if (!summary?.trim()) {
|
||||
throw new Error("当前图片暂时无法提取稳定的视觉摘要,请稍后再试。");
|
||||
}
|
||||
|
||||
const normalizedSummary = summary.trim();
|
||||
const answer = await buildDirectAgentImageAnalysisAnswer({
|
||||
question: params.requestTextContent,
|
||||
summary: normalizedSummary,
|
||||
modelId: params.answerModelId,
|
||||
});
|
||||
|
||||
return {
|
||||
summary: normalizedSummary,
|
||||
answer,
|
||||
};
|
||||
}
|
||||
|
||||
export async function executeDirectAgentImageGeneration(
|
||||
params: ExecuteDirectAgentImageGenerationParams,
|
||||
): Promise<DirectAgentImageGenerationResult> {
|
||||
const prompt =
|
||||
String(params.optimizedPrompt || params.requestTextContent || "").trim() || "生成图片";
|
||||
const imageSize =
|
||||
params.imageParams?.imageSize || params.defaultImageQualityParams?.imageSize;
|
||||
const imageCount =
|
||||
params.imageParams?.imageCount || params.defaultImageQualityParams?.imageCount;
|
||||
const result = await generateCanvasImage({
|
||||
prompt,
|
||||
negativePrompt: params.imageParams?.negativePrompt,
|
||||
model: params.finalModelId,
|
||||
aspectRatio:
|
||||
params.imageParams?.aspectRatio || params.defaultImageQualityParams?.aspectRatio,
|
||||
imageSize,
|
||||
imageCount,
|
||||
referenceImages: params.referenceImageUrls,
|
||||
});
|
||||
|
||||
const imageMarkdown = buildAgentToolImagesMarkdown(
|
||||
result.images.map((image) => ({
|
||||
originalUrl: image.originalUrl,
|
||||
thumbnailUrl: image.thumbnailUrl,
|
||||
imageSize,
|
||||
})),
|
||||
);
|
||||
|
||||
return {
|
||||
prompt,
|
||||
imageMarkdown,
|
||||
images: result.images,
|
||||
model: result.model,
|
||||
modelRef: result.modelRef || null,
|
||||
imageSize,
|
||||
imageCount: result.images.length,
|
||||
};
|
||||
}
|
||||
|
||||
export async function executeDirectAgentVideoGeneration(
|
||||
params: ExecuteDirectAgentVideoGenerationParams,
|
||||
): Promise<DirectAgentVideoGenerationResult> {
|
||||
const prompt =
|
||||
String(params.optimizedPrompt || params.requestTextContent || "").trim() || "生成视频";
|
||||
const plannedArgs = findAgentToolCallArgs(
|
||||
"generateVideo",
|
||||
params.smartAutoExecution,
|
||||
params.smartAutoPlan,
|
||||
);
|
||||
const mode =
|
||||
normalizeDirectAgentVideoMode(plannedArgs?.mode) ||
|
||||
normalizeDirectAgentVideoMode(params.videoParams?.mode) ||
|
||||
inferDirectAgentVideoMode(params.sourceImageUrls);
|
||||
const images = pickDirectAgentVideoImages(mode, params.sourceImageUrls);
|
||||
const result = await executeDirectVideoGeneration({
|
||||
prompt,
|
||||
mode,
|
||||
images,
|
||||
resolution: params.videoParams?.resolution,
|
||||
ratio: normalizeDirectAgentVideoRatio(params.videoParams?.ratio),
|
||||
duration: params.videoParams?.duration,
|
||||
preferredModelRef: params.finalModelId,
|
||||
});
|
||||
|
||||
const videoUrl = String(result.videoUrl || "").trim();
|
||||
if (!videoUrl) {
|
||||
throw new Error("视频生成完成但未返回视频地址");
|
||||
}
|
||||
|
||||
return {
|
||||
prompt,
|
||||
mode,
|
||||
images,
|
||||
videoUrl,
|
||||
result,
|
||||
};
|
||||
}
|
||||
213
lib/ai/runtime/direct-agent-progress.ts
Normal file
213
lib/ai/runtime/direct-agent-progress.ts
Normal file
@ -0,0 +1,213 @@
|
||||
import type {
|
||||
DirectAgentToolCapability,
|
||||
DirectAgentToolName,
|
||||
} from "@/lib/ai/runtime/direct-agent-execution";
|
||||
|
||||
type DirectAgentProgressStage =
|
||||
| "planning"
|
||||
| "clarify"
|
||||
| "executing"
|
||||
| "completed"
|
||||
| "failed";
|
||||
|
||||
type DirectAgentProgressStep =
|
||||
| "lock_target"
|
||||
| "analyze"
|
||||
| "select_tool"
|
||||
| "execute"
|
||||
| "finalize";
|
||||
|
||||
export interface DirectAgentProgressPayload {
|
||||
stage: DirectAgentProgressStage;
|
||||
currentStep: DirectAgentProgressStep;
|
||||
title?: string;
|
||||
message?: string;
|
||||
toolName?: DirectAgentToolName;
|
||||
capability?: DirectAgentToolCapability;
|
||||
}
|
||||
|
||||
function buildDirectAgentProgressPayload(params: DirectAgentProgressPayload) {
|
||||
return params;
|
||||
}
|
||||
|
||||
export function buildDirectAgentAnalyzePlanningProgress() {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "planning",
|
||||
currentStep: "analyze",
|
||||
title: "正在分析目标图片",
|
||||
message: "我会先提取稳定的视觉摘要,再整理成更直接的分析结论。",
|
||||
toolName: "analyzeImage",
|
||||
capability: "analyze",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentAnalyzeCompletedProgress() {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "completed",
|
||||
currentStep: "finalize",
|
||||
title: "图片分析完成",
|
||||
message: "我已经整理好可直接阅读的分析结论。",
|
||||
toolName: "analyzeImage",
|
||||
capability: "analyze",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentAnalyzeFailedProgress(errorMessage: string) {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "failed",
|
||||
currentStep: "analyze",
|
||||
title: "图片分析失败",
|
||||
message: errorMessage,
|
||||
toolName: "analyzeImage",
|
||||
capability: "analyze",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentImageGenerateSelectProgress() {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "planning",
|
||||
currentStep: "select_tool",
|
||||
title: "已选中图片生成工具",
|
||||
message: "我正在整理参考图和生图参数,准备开始生成。",
|
||||
toolName: "generateImage",
|
||||
capability: "generate_image",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentImageGenerateExecutingProgress() {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "executing",
|
||||
currentStep: "execute",
|
||||
title: "正在生成图片",
|
||||
message: "我会结合当前提示词和参考图生成新的画面。",
|
||||
toolName: "generateImage",
|
||||
capability: "generate_image",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentImageGenerateCompletedProgress() {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "completed",
|
||||
currentStep: "finalize",
|
||||
title: "图片生成完成",
|
||||
message: "结果已经回写到对话和画布。",
|
||||
toolName: "generateImage",
|
||||
capability: "generate_image",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentImageGenerateFailedProgress(errorMessage: string) {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "failed",
|
||||
currentStep: "execute",
|
||||
title: "图片生成失败",
|
||||
message: errorMessage,
|
||||
toolName: "generateImage",
|
||||
capability: "generate_image",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentVideoGenerateSelectProgress() {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "planning",
|
||||
currentStep: "select_tool",
|
||||
title: "已选中视频生成工具",
|
||||
message: "我正在整理模式、参考图和视频参数。",
|
||||
toolName: "generateVideo",
|
||||
capability: "generate_video",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentVideoGenerateExecutingProgress() {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "executing",
|
||||
currentStep: "execute",
|
||||
title: "正在生成视频",
|
||||
message: "我正在按当前模式执行视频生成。",
|
||||
toolName: "generateVideo",
|
||||
capability: "generate_video",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentVideoGenerateCompletedProgress() {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "completed",
|
||||
currentStep: "finalize",
|
||||
title: "视频生成完成",
|
||||
message: "结果已经写回对话和画布。",
|
||||
toolName: "generateVideo",
|
||||
capability: "generate_video",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentVideoGenerateFailedProgress(errorMessage: string) {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "failed",
|
||||
currentStep: "execute",
|
||||
title: "视频生成失败",
|
||||
message: errorMessage,
|
||||
toolName: "generateVideo",
|
||||
capability: "generate_video",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentImageEditSelectProgress(params: {
|
||||
toolName: DirectAgentToolName;
|
||||
capability: DirectAgentToolCapability;
|
||||
label: string;
|
||||
}) {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "planning",
|
||||
currentStep: "select_tool",
|
||||
title: `已选中${params.label}`,
|
||||
message: "我正在整理目标图和当前参数,准备开始执行。",
|
||||
toolName: params.toolName,
|
||||
capability: params.capability,
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentImageEditExecutingProgress(params: {
|
||||
toolName: DirectAgentToolName;
|
||||
capability: DirectAgentToolCapability;
|
||||
label: string;
|
||||
}) {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "executing",
|
||||
currentStep: "execute",
|
||||
title: `正在执行${params.label}`,
|
||||
message: "执行完成后我会把结果直接回写到当前对话和画布。",
|
||||
toolName: params.toolName,
|
||||
capability: params.capability,
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentImageEditCompletedProgress(params: {
|
||||
toolName: DirectAgentToolName;
|
||||
capability: DirectAgentToolCapability;
|
||||
label: string;
|
||||
}) {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "completed",
|
||||
currentStep: "finalize",
|
||||
title: `${params.label}完成`,
|
||||
message: "结果已经回写到当前对话和画布。",
|
||||
toolName: params.toolName,
|
||||
capability: params.capability,
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDirectAgentImageEditFailedProgress(params: {
|
||||
toolName: DirectAgentToolName;
|
||||
capability: DirectAgentToolCapability;
|
||||
label: string;
|
||||
errorMessage: string;
|
||||
}) {
|
||||
return buildDirectAgentProgressPayload({
|
||||
stage: "failed",
|
||||
currentStep: "execute",
|
||||
title: `${params.label}失败`,
|
||||
message: params.errorMessage,
|
||||
toolName: params.toolName,
|
||||
capability: params.capability,
|
||||
});
|
||||
}
|
||||
78
lib/ai/runtime/direct-agent-response-builders.ts
Normal file
78
lib/ai/runtime/direct-agent-response-builders.ts
Normal file
@ -0,0 +1,78 @@
|
||||
import {
|
||||
buildAgentToolImageMarkdown,
|
||||
buildAgentToolVideoMarkdown,
|
||||
prependImageModelBadge,
|
||||
} from "@/lib/ai/runtime/agent-media-markdown";
|
||||
import {
|
||||
getDirectAgentToolLabel,
|
||||
getDirectAgentToolSuccessMessage,
|
||||
type DirectAgentToolCapability,
|
||||
} from "@/lib/ai/runtime/direct-agent-execution";
|
||||
|
||||
export function buildDirectAgentAnalysisAssistantText(params: {
|
||||
summary: string;
|
||||
answer?: string | null;
|
||||
}) {
|
||||
const answer = String(params.answer || "").trim();
|
||||
return prependImageModelBadge(
|
||||
[
|
||||
getDirectAgentToolSuccessMessage("analyze"),
|
||||
`图像观察:\n${String(params.summary || "").trim()}`,
|
||||
answer ? `分析结论:\n${answer}` : "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n"),
|
||||
getDirectAgentToolLabel("analyze"),
|
||||
);
|
||||
}
|
||||
|
||||
export function buildDirectAgentImageGenerationAssistantText(params: {
|
||||
imageMarkdown: string;
|
||||
modelDisplayName?: string | null;
|
||||
}) {
|
||||
return prependImageModelBadge(
|
||||
[
|
||||
getDirectAgentToolSuccessMessage("generate_image"),
|
||||
params.imageMarkdown,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n"),
|
||||
params.modelDisplayName,
|
||||
);
|
||||
}
|
||||
|
||||
export function buildDirectAgentVideoGenerationAssistantText(params: {
|
||||
videoUrl: string;
|
||||
modelDisplayName?: string | null;
|
||||
}) {
|
||||
return prependImageModelBadge(
|
||||
[
|
||||
getDirectAgentToolSuccessMessage("generate_video"),
|
||||
buildAgentToolVideoMarkdown(params.videoUrl),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n"),
|
||||
params.modelDisplayName,
|
||||
);
|
||||
}
|
||||
|
||||
export function buildDirectAgentImageEditAssistantText(params: {
|
||||
capability: DirectAgentToolCapability;
|
||||
originalUrl: string;
|
||||
thumbnailUrl?: string | null;
|
||||
imageSize?: string;
|
||||
}) {
|
||||
return prependImageModelBadge(
|
||||
[
|
||||
getDirectAgentToolSuccessMessage(params.capability),
|
||||
buildAgentToolImageMarkdown({
|
||||
originalUrl: params.originalUrl,
|
||||
thumbnailUrl: params.thumbnailUrl,
|
||||
imageSize: params.imageSize,
|
||||
}),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n"),
|
||||
getDirectAgentToolLabel(params.capability),
|
||||
);
|
||||
}
|
||||
144
lib/ai/runtime/direct-agent-working-state.ts
Normal file
144
lib/ai/runtime/direct-agent-working-state.ts
Normal file
@ -0,0 +1,144 @@
|
||||
import {
|
||||
persistImageWorkflowTurn,
|
||||
persistVideoGenerationCompletedTurn,
|
||||
persistVideoGenerationFailedTurn,
|
||||
} from "@/lib/ai/conversation-state-protocol";
|
||||
import type { DirectAgentToolCapability } from "@/lib/ai/runtime/direct-agent-execution";
|
||||
|
||||
export async function safePersistDirectAgentAnalyzeTurn(params: {
|
||||
userId: string;
|
||||
chatId: string;
|
||||
projectId?: string | null;
|
||||
currentSubIntent?: string | null;
|
||||
targetAssetIds?: string[] | null;
|
||||
targetImageUrls: string[];
|
||||
messageId: string;
|
||||
requestTextContent: string;
|
||||
}) {
|
||||
try {
|
||||
const targetImageUrls = params.targetImageUrls.length > 0 ? params.targetImageUrls : null;
|
||||
await persistImageWorkflowTurn({
|
||||
userId: params.userId,
|
||||
chatId: params.chatId,
|
||||
projectId: params.projectId || null,
|
||||
currentIntent: "vision",
|
||||
currentSubIntent: params.currentSubIntent || "analyze",
|
||||
currentFocusAssetIds: params.targetAssetIds || null,
|
||||
currentFocusImageUrls: targetImageUrls,
|
||||
lastResolvedTargetAssetIds: params.targetAssetIds || null,
|
||||
lastResolvedTargetImageUrls: targetImageUrls,
|
||||
lastMessageId: params.messageId,
|
||||
lastUserPrompt: params.requestTextContent,
|
||||
lastSourceImageUrls: targetImageUrls,
|
||||
});
|
||||
} catch (memoryError) {
|
||||
console.warn("[Direct Agent Working State] Failed to persist analyze turn:", memoryError);
|
||||
}
|
||||
}
|
||||
|
||||
export async function safePersistDirectAgentImageEditTurn(params: {
|
||||
userId: string;
|
||||
chatId: string;
|
||||
projectId?: string | null;
|
||||
capability: DirectAgentToolCapability;
|
||||
currentSubIntent?: string | null;
|
||||
resultAssetId?: string | null;
|
||||
resultOriginalUrl?: string | null;
|
||||
resultImageSize?: string | null;
|
||||
sourceImageUrls: string[];
|
||||
messageId: string;
|
||||
requestTextContent: string;
|
||||
optimizedPrompt?: string | null;
|
||||
}) {
|
||||
try {
|
||||
const resultAssetIds = params.resultAssetId ? [params.resultAssetId] : null;
|
||||
const resultImageUrls = params.resultOriginalUrl ? [params.resultOriginalUrl] : null;
|
||||
await persistImageWorkflowTurn({
|
||||
userId: params.userId,
|
||||
chatId: params.chatId,
|
||||
projectId: params.projectId || null,
|
||||
currentIntent: "image_edit",
|
||||
currentSubIntent: params.currentSubIntent || params.capability,
|
||||
currentFocusAssetIds: resultAssetIds,
|
||||
currentFocusImageUrls: resultImageUrls,
|
||||
lastGeneratedAssetIds: resultAssetIds,
|
||||
lastGeneratedImageUrls: resultImageUrls,
|
||||
lastResolvedTargetAssetIds: resultAssetIds,
|
||||
lastResolvedTargetImageUrls: resultImageUrls,
|
||||
lastMessageId: params.messageId,
|
||||
lastUserPrompt: params.requestTextContent,
|
||||
lastOptimizedPrompt: params.optimizedPrompt || undefined,
|
||||
lastRequestedImageCount: resultImageUrls?.length || 1,
|
||||
lastImageSize: params.resultImageSize || undefined,
|
||||
lastSourceImageUrls: params.sourceImageUrls,
|
||||
lastResultImageUrls: resultImageUrls,
|
||||
});
|
||||
} catch (memoryError) {
|
||||
console.warn(
|
||||
`[Direct Agent Working State] Failed to persist image edit turn after ${params.capability}:`,
|
||||
memoryError,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function safePersistDirectAgentVideoCompletedTurn(params: {
|
||||
userId: string;
|
||||
chatId: string;
|
||||
projectId?: string | null;
|
||||
messageId: string;
|
||||
requestTextContent: string;
|
||||
modelRef: string;
|
||||
mode: string;
|
||||
temporaryVideoUrl: string;
|
||||
}) {
|
||||
try {
|
||||
await persistVideoGenerationCompletedTurn({
|
||||
userId: params.userId,
|
||||
chatId: params.chatId,
|
||||
projectId: params.projectId || null,
|
||||
messageId: params.messageId,
|
||||
text: params.requestTextContent,
|
||||
source: "agent_direct_generate_video",
|
||||
providerType: "direct-video",
|
||||
modelRef: params.modelRef,
|
||||
mode: params.mode as any,
|
||||
status: "completed",
|
||||
temporaryVideoUrl: params.temporaryVideoUrl,
|
||||
});
|
||||
} catch (memoryError) {
|
||||
console.warn(
|
||||
"[Direct Agent Working State] Failed to persist direct video generation:",
|
||||
memoryError,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function safePersistDirectAgentVideoFailedTurn(params: {
|
||||
userId: string;
|
||||
chatId: string;
|
||||
projectId?: string | null;
|
||||
messageId: string;
|
||||
requestTextContent: string;
|
||||
modelRef: string;
|
||||
error: unknown;
|
||||
}) {
|
||||
try {
|
||||
await persistVideoGenerationFailedTurn({
|
||||
userId: params.userId,
|
||||
chatId: params.chatId,
|
||||
projectId: params.projectId || null,
|
||||
messageId: params.messageId,
|
||||
text: params.requestTextContent,
|
||||
source: "agent_direct_generate_video",
|
||||
providerType: "direct-video",
|
||||
modelRef: params.modelRef,
|
||||
status: "failed",
|
||||
error: params.error instanceof Error ? params.error.message : String(params.error),
|
||||
});
|
||||
} catch (videoMemoryError) {
|
||||
console.warn(
|
||||
"[Direct Agent Working State] Failed to persist direct video failure:",
|
||||
videoMemoryError,
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -8,6 +8,7 @@ import { createFalImage } from "@/lib/ai/providers/fal-image";
|
||||
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
|
||||
import { createMidjourney } from "@/lib/ai/providers/midjourney";
|
||||
import { createRunway } from "@/lib/ai/providers/runway";
|
||||
import { createKrea } from "@/lib/ai/providers/krea";
|
||||
import { filterReachableRemoteUrls } from "@/lib/net/url-access";
|
||||
|
||||
export interface CanvasGenerateImageRequest {
|
||||
@ -43,6 +44,7 @@ const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
|
||||
"openai-image",
|
||||
"midjourney",
|
||||
"runway",
|
||||
"krea",
|
||||
]);
|
||||
|
||||
function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
|
||||
@ -82,6 +84,8 @@ function getProviderBatchLimit(providerType: string) {
|
||||
case "midjourney":
|
||||
case "runway":
|
||||
return 1;
|
||||
case "krea":
|
||||
return 4;
|
||||
case "gemini":
|
||||
return 2;
|
||||
case "alibaba-image":
|
||||
@ -162,7 +166,7 @@ export async function generateCanvasImage(
|
||||
}
|
||||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(fallbackProvider.type)) {
|
||||
throw new Error(
|
||||
"当前图片模型类型暂不支持画布生成,请在后台选择 Gemini/阿里/RUXA/FAL/OpenAI/Runway 图片模型。"
|
||||
"当前图片模型类型暂不支持画布生成,请在后台选择 Gemini/阿里/RUXA/FAL/OpenAI/Runway/Krea 图片模型。"
|
||||
);
|
||||
}
|
||||
if (fallbackModel.model.id !== selectedModel.model.id) {
|
||||
@ -349,9 +353,25 @@ export async function generateCanvasImage(
|
||||
});
|
||||
return result.text || "";
|
||||
};
|
||||
} else if (provider.type === "krea") {
|
||||
const kreaProvider = createKrea({
|
||||
apiKey: provider.apiKey,
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
const modelInstance = kreaProvider.chat(selectedModel.model.modelId);
|
||||
executeGenerateBatch = async (batchImageCount) => {
|
||||
const result = await modelInstance.doGenerate({
|
||||
messages,
|
||||
imageParams: {
|
||||
...imageParams,
|
||||
imageCount: batchImageCount,
|
||||
},
|
||||
});
|
||||
return result.text || "";
|
||||
};
|
||||
} else {
|
||||
throw new Error(
|
||||
`不支持的图片生成模型类型: ${provider.type || "unknown"},请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney/Runway 图片模型`
|
||||
`不支持的图片生成模型类型: ${provider.type || "unknown"},请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney/Runway/Krea 图片模型`
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@ -36,6 +36,7 @@ const providerBaseUrlDefaults = [
|
||||
{ type: "worldlabs-3d", baseUrl: "https://api.worldlabs.ai" },
|
||||
{ type: "kittycad-cad", baseUrl: "https://api.zoo.dev" },
|
||||
{ type: "runway", baseUrl: "https://api.dev.runwayml.com" },
|
||||
{ type: "krea", baseUrl: "https://api.krea.ai" },
|
||||
] as const;
|
||||
|
||||
function assertMigrationJournalIsConsistent() {
|
||||
@ -95,7 +96,7 @@ const runMigrate = async () => {
|
||||
ADD CONSTRAINT "AiProvider_type_check"
|
||||
CHECK (type IN (
|
||||
'alibaba', 'openai', 'custom', 'local', 'gemini',
|
||||
'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'bfl-image', 'ruxa-image', 'fal-image', 'openai-image', 'midjourney', 'fal-video', 'zhipu', 'tripo-3d', 'worldlabs-3d', 'kittycad-cad', 'runway'
|
||||
'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'bfl-image', 'ruxa-image', 'fal-image', 'openai-image', 'midjourney', 'fal-video', 'zhipu', 'tripo-3d', 'worldlabs-3d', 'kittycad-cad', 'runway', 'krea'
|
||||
));
|
||||
`);
|
||||
await db.execute(sql`
|
||||
|
||||
5
lib/db/migrations/0071_allow_group_world_shares.sql
Normal file
5
lib/db/migrations/0071_allow_group_world_shares.sql
Normal file
@ -0,0 +1,5 @@
|
||||
DROP INDEX IF EXISTS "Model3DAsset_shareToken_idx";
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "Model3DAsset_shareToken_lookup_idx"
|
||||
ON "Model3DAsset" ("shareToken")
|
||||
WHERE "shareToken" IS NOT NULL;
|
||||
33
lib/db/migrations/0072_add_krea_provider_type.sql
Normal file
33
lib/db/migrations/0072_add_krea_provider_type.sql
Normal file
@ -0,0 +1,33 @@
|
||||
ALTER TABLE "AiProvider"
|
||||
DROP CONSTRAINT IF EXISTS "AiProvider_type_check";
|
||||
|
||||
ALTER TABLE "AiProvider"
|
||||
ADD CONSTRAINT "AiProvider_type_check"
|
||||
CHECK (
|
||||
"type" IN (
|
||||
'alibaba',
|
||||
'openai',
|
||||
'custom',
|
||||
'local',
|
||||
'gemini',
|
||||
'huanke-gemini',
|
||||
'qianwen-video',
|
||||
'doubao-video',
|
||||
'ruxa-video',
|
||||
'volcano-vision',
|
||||
'volcano-video',
|
||||
'alibaba-image',
|
||||
'bfl-image',
|
||||
'ruxa-image',
|
||||
'fal-image',
|
||||
'openai-image',
|
||||
'midjourney',
|
||||
'fal-video',
|
||||
'zhipu',
|
||||
'tripo-3d',
|
||||
'worldlabs-3d',
|
||||
'kittycad-cad',
|
||||
'runway',
|
||||
'krea'
|
||||
)
|
||||
);
|
||||
@ -400,6 +400,20 @@
|
||||
"when": 1775908800000,
|
||||
"tag": "0070_flow_board_cloud_files",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 57,
|
||||
"version": "7",
|
||||
"when": 1775955600000,
|
||||
"tag": "0071_allow_group_world_shares",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 58,
|
||||
"version": "7",
|
||||
"when": 1776042000000,
|
||||
"tag": "0072_add_krea_provider_type",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -3750,7 +3750,7 @@ export async function createAiProvider({
|
||||
updatedBy,
|
||||
}: {
|
||||
name: string;
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway";
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway" | "krea";
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
isEnabled?: boolean;
|
||||
@ -3788,7 +3788,7 @@ export async function updateAiProvider({
|
||||
}: {
|
||||
id: string;
|
||||
name?: string;
|
||||
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway";
|
||||
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway" | "krea";
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
isEnabled?: boolean;
|
||||
|
||||
@ -264,7 +264,7 @@ export type ToolConfig = InferSelectModel<typeof toolConfig>;
|
||||
export const aiProvider = pgTable("AiProvider", {
|
||||
id: uuid("id").primaryKey().notNull().defaultRandom(),
|
||||
name: varchar("name", { length: 128 }).notNull(), // 提供商名称,如 "阿里云通义千问"
|
||||
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "fal-video", "zhipu", "tripo-3d", "worldlabs-3d", "kittycad-cad", "runway", "bfl-image"] })
|
||||
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "fal-video", "zhipu", "tripo-3d", "worldlabs-3d", "kittycad-cad", "runway", "krea", "bfl-image"] })
|
||||
.notNull()
|
||||
.default("custom"), // 提供商类型
|
||||
baseUrl: text("baseUrl").notNull(), // API 基础 URL
|
||||
|
||||
File diff suppressed because one or more lines are too long
Loading…
x
Reference in New Issue
Block a user