476 lines
16 KiB
TypeScript
476 lines
16 KiB
TypeScript
import {
|
|
executeBoxCutoutJob,
|
|
} from "@/lib/ai/box-cutout/service";
|
|
import { executeExtendJob } from "@/lib/ai/extend/service";
|
|
import { executeLayerSplitJob } from "@/lib/ai/layer-split/service";
|
|
import { executeResizeJob } from "@/lib/ai/resize/service";
|
|
import {
|
|
executeRmbgJob,
|
|
fetchImageBufferFromUrl,
|
|
} from "@/lib/ai/rmbg/service";
|
|
import type { CanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
|
import type { CanvasCapabilityTarget } from "@/lib/canvas/capabilities/context";
|
|
import { resolveCanvasRuntimeEndpoint } from "@/lib/canvas/capabilities/internal-endpoint";
|
|
import { generateCanvasImage } from "@/lib/canvas/generate-image";
|
|
import { requestLongTaskResult } from "@/lib/client/long-task";
|
|
import {
|
|
findStyleProImageModelByRef,
|
|
getStyleProImageModelEntries,
|
|
sortStyleProImageModelEntries,
|
|
} from "@/lib/style-pro/model-options";
|
|
|
|
export type CanvasCapabilityMediaResult = {
|
|
assetId?: string;
|
|
height?: number;
|
|
metadata?: Record<string, unknown>;
|
|
nodeType?: "image" | "model3d" | "world3d";
|
|
prompt?: string;
|
|
sourceNodeId: string;
|
|
title: string;
|
|
url: string;
|
|
width?: number;
|
|
};
|
|
|
|
type HeadlessExecutorParams = {
|
|
command: CanvasCapabilityClientCommand;
|
|
requestHeaders: Headers;
|
|
requestUrl: string;
|
|
targets: CanvasCapabilityTarget[];
|
|
userId: string;
|
|
};
|
|
|
|
function asFiniteNumber(value: unknown) {
|
|
const number = Number(value);
|
|
return Number.isFinite(number) ? number : undefined;
|
|
}
|
|
|
|
function getCookieHeaders(headers: Headers) {
|
|
const cookie = headers.get("cookie") || "";
|
|
return cookie ? { cookie } : undefined;
|
|
}
|
|
|
|
async function postRuntimeEndpoint(params: {
|
|
body: Record<string, unknown>;
|
|
path: string;
|
|
requestHeaders: Headers;
|
|
requestUrl: string;
|
|
timeoutMs?: number;
|
|
}) {
|
|
const response = await fetch(resolveCanvasRuntimeEndpoint(params.path, params.requestUrl), {
|
|
method: "POST",
|
|
headers: {
|
|
"Content-Type": "application/json",
|
|
...getCookieHeaders(params.requestHeaders),
|
|
},
|
|
body: JSON.stringify(params.body),
|
|
signal: AbortSignal.timeout(params.timeoutMs || 20 * 60 * 1000),
|
|
});
|
|
const payload = await response.json().catch(() => null);
|
|
if (!response.ok || payload?.success === false || payload?.error) {
|
|
throw new Error(
|
|
String(payload?.error || payload?.detail || `能力服务返回 ${response.status}`)
|
|
);
|
|
}
|
|
return payload as Record<string, any>;
|
|
}
|
|
|
|
async function postRuntimeFormData(params: {
|
|
formData: FormData;
|
|
path: string;
|
|
requestHeaders: Headers;
|
|
requestUrl: string;
|
|
timeoutMs?: number;
|
|
}) {
|
|
const response = await fetch(resolveCanvasRuntimeEndpoint(params.path, params.requestUrl), {
|
|
method: "POST",
|
|
headers: getCookieHeaders(params.requestHeaders),
|
|
body: params.formData,
|
|
signal: AbortSignal.timeout(params.timeoutMs || 20 * 60 * 1000),
|
|
});
|
|
const payload = await response.json().catch(() => null);
|
|
if (!response.ok || payload?.success === false || payload?.error) {
|
|
throw new Error(
|
|
String(payload?.error || payload?.detail || `能力服务返回 ${response.status}`)
|
|
);
|
|
}
|
|
return payload as Record<string, any>;
|
|
}
|
|
|
|
function inferTripoImageType(imageUrl: string) {
|
|
try {
|
|
const extension = new URL(imageUrl).pathname.split(".").pop()?.toLowerCase();
|
|
if (extension === "png" || extension === "webp") return extension;
|
|
} catch {}
|
|
return "jpg";
|
|
}
|
|
|
|
async function resolveImageModel(modelRef?: unknown) {
|
|
const entries = sortStyleProImageModelEntries(
|
|
await getStyleProImageModelEntries({ includeHidden: false })
|
|
);
|
|
const requested = findStyleProImageModelByRef(entries, String(modelRef || ""));
|
|
const selected = requested || entries.find((entry) => entry.model.isDefault) || entries[0];
|
|
if (!selected) throw new Error("系统未配置可用的图片生成模型");
|
|
return selected;
|
|
}
|
|
|
|
async function executeForTarget(
|
|
params: HeadlessExecutorParams,
|
|
target: CanvasCapabilityTarget
|
|
): Promise<CanvasCapabilityMediaResult[]> {
|
|
const { command } = params;
|
|
const prompt = String(command.args.prompt || "").trim();
|
|
const common = {
|
|
sourceNodeId: target.nodeId,
|
|
prompt: prompt || undefined,
|
|
};
|
|
|
|
if (command.capabilityId === "image.remove_background") {
|
|
const imageBuffer = await fetchImageBufferFromUrl(target.sourceImageUrl);
|
|
const result = await executeRmbgJob({
|
|
userId: params.userId,
|
|
imageBuffer,
|
|
sourceImageUrl: target.sourceImageUrl,
|
|
prompt: prompt || "抠图处理",
|
|
skipGallerySave: false,
|
|
});
|
|
return [{
|
|
...common,
|
|
assetId: result.mediaAssetId || undefined,
|
|
title: `${target.title}-抠图`,
|
|
url: result.original_url,
|
|
}];
|
|
}
|
|
|
|
if (command.capabilityId === "image.extend") {
|
|
const result = await executeExtendJob({
|
|
userId: params.userId,
|
|
imageUrl: target.sourceImageUrl,
|
|
sourceImageUrl: target.sourceImageUrl,
|
|
prompt: prompt || "图像拓展处理",
|
|
xScale: asFiniteNumber(command.args.xScale),
|
|
yScale: asFiniteNumber(command.args.yScale),
|
|
requestId: command.requestId,
|
|
});
|
|
return [{
|
|
...common,
|
|
assetId: result.mediaAssetId || undefined,
|
|
title: `${target.title}-扩图`,
|
|
url: result.original_url,
|
|
}];
|
|
}
|
|
|
|
if (command.capabilityId === "image.resize") {
|
|
const width = asFiniteNumber(command.args.width);
|
|
const height = asFiniteNumber(command.args.height);
|
|
if (!width || !height) throw new Error("请提供完整的目标宽度和高度");
|
|
const result = await executeResizeJob({
|
|
userId: params.userId,
|
|
imageUrl: target.sourceImageUrl,
|
|
sourceImageUrl: target.sourceImageUrl,
|
|
canvasWidth: Math.round(width),
|
|
canvasHeight: Math.round(height),
|
|
backgroundColor: String(command.args.backgroundColor || "").trim() || undefined,
|
|
transparentBackground: command.args.transparentBackground === true,
|
|
mimeType: String(command.args.mimeType || "").trim() || undefined,
|
|
});
|
|
return [{
|
|
...common,
|
|
assetId: result.mediaAssetId || undefined,
|
|
height: result.canvas_height,
|
|
title: `${target.title}-${result.canvas_width}x${result.canvas_height}`,
|
|
url: result.original_url,
|
|
width: result.canvas_width,
|
|
}];
|
|
}
|
|
|
|
if (command.capabilityId === "image.box_cutout") {
|
|
const result = await executeBoxCutoutJob({
|
|
userId: params.userId,
|
|
imageUrl: target.sourceImageUrl,
|
|
sourceNodeId: target.nodeId,
|
|
cropRect: {
|
|
x: asFiniteNumber(command.args.x),
|
|
y: asFiniteNumber(command.args.y),
|
|
width: asFiniteNumber(command.args.width),
|
|
height: asFiniteNumber(command.args.height),
|
|
},
|
|
paddingRatio: asFiniteNumber(command.args.paddingRatio),
|
|
prompt: prompt || "框选抠图",
|
|
});
|
|
return [{
|
|
...common,
|
|
assetId: result.mediaAssetId || undefined,
|
|
metadata: {
|
|
cropRectApplied: result.cropRectApplied,
|
|
paddedCropRectApplied: result.paddedCropRectApplied,
|
|
},
|
|
title: `${target.title}-框选抠图`,
|
|
url: result.original_url,
|
|
}];
|
|
}
|
|
|
|
if (command.capabilityId === "image.layer_split") {
|
|
const result = await executeLayerSplitJob({
|
|
userId: params.userId,
|
|
imageUrl: target.sourceImageUrl,
|
|
sourceNodeId: target.nodeId,
|
|
prompt: prompt || undefined,
|
|
});
|
|
return result.layers.map((layer) => ({
|
|
...common,
|
|
assetId: layer.mediaAssetId || undefined,
|
|
metadata: {
|
|
bounds: layer.bounds || null,
|
|
layerIndex: layer.layerIndex,
|
|
model: result.model,
|
|
provider: result.provider,
|
|
totalLayers: layer.totalLayers,
|
|
},
|
|
title: layer.title || `${target.title}-图层${layer.layerIndex + 1}`,
|
|
url: layer.originalUrl,
|
|
}));
|
|
}
|
|
|
|
if (command.capabilityId === "image.enhance") {
|
|
const result = await requestLongTaskResult({
|
|
endpoint: resolveCanvasRuntimeEndpoint("/api/enhance", params.requestUrl),
|
|
payload: {
|
|
imageUrl: target.sourceImageUrl,
|
|
prompt: prompt || "图像增强处理",
|
|
scaleFactor: asFiniteNumber(command.args.scaleFactor),
|
|
},
|
|
pollingParams: {
|
|
prompt: prompt || "图像增强处理",
|
|
sourceImageUrl: target.sourceImageUrl,
|
|
},
|
|
timeoutMs: 10 * 60 * 1000,
|
|
headers: getCookieHeaders(params.requestHeaders),
|
|
});
|
|
if (!result?.success || !result.original_url) {
|
|
throw new Error(String(result?.error || result?.detail || "图像增强失败"));
|
|
}
|
|
return [{
|
|
...common,
|
|
title: `${target.title}-高清增强`,
|
|
url: String(result.original_url),
|
|
}];
|
|
}
|
|
|
|
if (command.capabilityId === "image.remove_watermark") {
|
|
const result = await requestLongTaskResult({
|
|
endpoint: resolveCanvasRuntimeEndpoint(
|
|
"/api/watermark-remove",
|
|
params.requestUrl
|
|
),
|
|
payload: { image_url: target.sourceImageUrl },
|
|
timeoutMs: 20 * 60 * 1000,
|
|
headers: getCookieHeaders(params.requestHeaders),
|
|
});
|
|
if (!result?.success || !result.original_url) {
|
|
throw new Error(String(result?.error || result?.detail || "去水印失败"));
|
|
}
|
|
return [{
|
|
...common,
|
|
title: `${target.title}-去水印`,
|
|
url: String(result.original_url),
|
|
}];
|
|
}
|
|
|
|
if (command.capabilityId === "image.multi_angle") {
|
|
const result = await postRuntimeEndpoint({
|
|
path: "/api/angle-edit",
|
|
requestUrl: params.requestUrl,
|
|
requestHeaders: params.requestHeaders,
|
|
body: {
|
|
image_url: target.sourceImageUrl,
|
|
horizontal_angle: asFiniteNumber(command.args.horizontalAngle) ?? 30,
|
|
vertical_angle: asFiniteNumber(command.args.verticalAngle) ?? 0,
|
|
zoom: asFiniteNumber(command.args.zoom) ?? 0,
|
|
},
|
|
});
|
|
const url = String(result.image_url || result.original_url || "").trim();
|
|
if (!url) throw new Error("多视角服务未返回图片");
|
|
return [{ ...common, title: `${target.title}-新视角`, url }];
|
|
}
|
|
|
|
if (command.capabilityId === "image.generate_3d_model") {
|
|
const modelVersion =
|
|
String(command.args.modelId || command.args.modelVersion || "").trim() ||
|
|
"tripo-standard";
|
|
const promptText = prompt || target.title || "图生 3D 模型";
|
|
const response = await postRuntimeEndpoint({
|
|
path: "/api/canvas/3d/tasks",
|
|
requestUrl: params.requestUrl,
|
|
requestHeaders: params.requestHeaders,
|
|
timeoutMs: 180_000,
|
|
body: {
|
|
taskType: "image_to_model",
|
|
imageUrl: target.sourceImageUrl,
|
|
imageType: inferTripoImageType(target.sourceImageUrl),
|
|
prompt: promptText,
|
|
modelVersion,
|
|
faceLimit: 120_000,
|
|
texture: true,
|
|
pbr: true,
|
|
autoSize: false,
|
|
quad: false,
|
|
generateParts: false,
|
|
smartLowPoly: false,
|
|
geometryQuality: "standard",
|
|
textureQuality: "standard",
|
|
textureAlignment: "original_image",
|
|
orientation: "default",
|
|
},
|
|
});
|
|
const task = response.data as Record<string, unknown> | undefined;
|
|
const taskId = String(task?.taskId || "").trim();
|
|
if (!taskId) throw new Error("3D 模型服务未返回任务 ID");
|
|
const providerId = String(task?.providerId || "").trim() || null;
|
|
const providerName = String(task?.providerName || "").trim() || null;
|
|
return [
|
|
{
|
|
...common,
|
|
nodeType: "model3d",
|
|
title: `${target.title}-3D模型`,
|
|
url: target.sourceImageUrl,
|
|
metadata: {
|
|
taskId,
|
|
provider: "tripo-3d",
|
|
mode: "image",
|
|
status: "generating",
|
|
sourceImageUrl: target.sourceImageUrl,
|
|
providerId,
|
|
providerName,
|
|
modelVersion,
|
|
generationStartedAt: new Date().toISOString(),
|
|
assetPayload: {
|
|
providerId,
|
|
providerName,
|
|
sourceImageUrl: target.sourceImageUrl,
|
|
},
|
|
},
|
|
},
|
|
];
|
|
}
|
|
|
|
if (command.capabilityId === "image.generate_3d_world") {
|
|
const modelVersion =
|
|
String(command.args.modelId || command.args.modelVersion || "").trim() ||
|
|
"world-standard";
|
|
const promptText = prompt || target.title || "图生 3D 世界";
|
|
const formData = new FormData();
|
|
formData.append("sourceUrl", target.sourceImageUrl);
|
|
formData.append("kind", "image");
|
|
const uploadResponse = await postRuntimeFormData({
|
|
formData,
|
|
path: "/api/canvas/world/upload",
|
|
requestUrl: params.requestUrl,
|
|
requestHeaders: params.requestHeaders,
|
|
timeoutMs: 180_000,
|
|
});
|
|
const upload = uploadResponse.data as Record<string, unknown> | undefined;
|
|
const mediaAssetId = String(upload?.mediaAssetId || "").trim();
|
|
if (!mediaAssetId) throw new Error("3D 世界服务未返回媒体资源 ID");
|
|
const providerId = String(upload?.providerId || "").trim() || null;
|
|
const providerName = String(upload?.providerName || "").trim() || null;
|
|
const taskResponse = await postRuntimeEndpoint({
|
|
path: "/api/canvas/world/tasks",
|
|
requestUrl: params.requestUrl,
|
|
requestHeaders: params.requestHeaders,
|
|
timeoutMs: 240_000,
|
|
body: {
|
|
mode: "image",
|
|
mediaAssetId,
|
|
providerId,
|
|
prompt: promptText,
|
|
displayName: target.title || promptText,
|
|
model: modelVersion,
|
|
disableRecaption: false,
|
|
makePublic: false,
|
|
},
|
|
});
|
|
const task = taskResponse.data as Record<string, unknown> | undefined;
|
|
const taskId = String(task?.taskId || "").trim();
|
|
if (!taskId) throw new Error("3D 世界服务未返回任务 ID");
|
|
const taskAssetPayload =
|
|
task?.assetPayload && typeof task.assetPayload === "object"
|
|
? (task.assetPayload as Record<string, unknown>)
|
|
: {};
|
|
return [
|
|
{
|
|
...common,
|
|
nodeType: "world3d",
|
|
title: `${target.title}-3D世界`,
|
|
url: target.sourceImageUrl,
|
|
metadata: {
|
|
taskId,
|
|
provider: "worldlabs-3d",
|
|
mode: "world-image",
|
|
status: "generating",
|
|
sourceImageUrl: target.sourceImageUrl,
|
|
providerId,
|
|
providerName,
|
|
modelVersion,
|
|
generationStartedAt: new Date().toISOString(),
|
|
assetPayload: {
|
|
...taskAssetPayload,
|
|
providerId,
|
|
providerName,
|
|
sourceImageUrl: target.sourceImageUrl,
|
|
},
|
|
},
|
|
},
|
|
];
|
|
}
|
|
|
|
if (
|
|
command.capabilityId === "image.prompt_regenerate" ||
|
|
command.capabilityId === "image.ai_wash"
|
|
) {
|
|
if (!prompt) throw new Error("请提供重绘提示词");
|
|
const model = await resolveImageModel(command.args.modelId);
|
|
const result = await generateCanvasImage({
|
|
prompt,
|
|
model: model.model.id,
|
|
ownerKey: params.userId,
|
|
imageCount: 1,
|
|
imageSize: String(command.args.imageSize || "2K"),
|
|
aspectRatio: String(command.args.aspectRatio || "auto"),
|
|
promptExtend: command.args.promptExtend === true,
|
|
referenceImage: target.sourceImageUrl,
|
|
debugLabel:
|
|
command.capabilityId === "image.ai_wash"
|
|
? "canvas-capability-ai-wash"
|
|
: "canvas-capability-regenerate",
|
|
debugMeta: {
|
|
capabilityId: command.capabilityId,
|
|
requestId: command.requestId,
|
|
sourceNodeId: target.nodeId,
|
|
},
|
|
});
|
|
const image = result.images[0];
|
|
if (!image?.originalUrl) throw new Error("图片模型未返回有效结果");
|
|
return [{
|
|
...common,
|
|
title:
|
|
command.capabilityId === "image.ai_wash"
|
|
? `${target.title}-AI洗图`
|
|
: `${target.title}-重绘`,
|
|
url: image.originalUrl,
|
|
}];
|
|
}
|
|
|
|
throw new Error(`尚未实现后台执行能力：${command.capabilityId}`);
|
|
}
|
|
|
|
export async function executeCanvasCapabilityHeadlessly(
|
|
params: HeadlessExecutorParams
|
|
) {
|
|
const results: CanvasCapabilityMediaResult[] = [];
|
|
for (const target of params.targets) {
|
|
results.push(...(await executeForTarget(params, target)));
|
|
}
|
|
return results;
|
|
}
|