feat: align scene fusion output params with model capabilities
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-03-07 11:09:51 +08:00
parent d104efd7e9
commit 21497ae57c
5 changed files with 500 additions and 90 deletions

View File

@ -6,7 +6,14 @@
import { NextResponse } from "next/server";
import { auth } from "@/app/(auth)/auth";
import { getSystemSetting } from "@/lib/db/queries";
import { getEnabledAiModelsWithProvider, getSystemSetting } from "@/lib/db/queries";
import {
buildSceneFusionCapabilitySummary,
canProcessSceneFusionProviderType,
normalizeSceneFusionProviderType,
type SceneFusionCapabilityCandidate,
type SceneFusionImageGenConfig,
} from "@/lib/scene-fusion/output-capabilities";
// 默认提示词
const FALLBACK_PROMPT = "将图1的产品替换到图2的场景中,保持图1产品的外观、颜色和细节不变,自然融合到图2的环境中。";
@ -25,21 +32,47 @@ export async function GET() {
// 获取配置
const setting = await getSystemSetting({ key: "scene_fusion_config" });
const config = setting?.value as
| {
enabled?: boolean;
defaultPrompt?: string;
maxScenes?: number;
preRmbgEnabled?: boolean;
modelId?: string;
}
| undefined;
const allModels = await getEnabledAiModelsWithProvider();
const capabilityCandidates: SceneFusionCapabilityCandidate[] = allModels
.filter((entry) => {
if (!entry?.model?.isEnabled) return false;
if (!entry?.model?.supportsImageGen) return false;
if (entry?.model?.isSubModel) return false;
return canProcessSceneFusionProviderType(String(entry.provider?.type || ""));
})
.map((entry) => ({
model: {
id: String(entry.model.id),
displayName: String(entry.model.displayName || entry.model.modelId || entry.model.id),
imageGenConfig: (entry.model.imageGenConfig || null) as SceneFusionImageGenConfig | null,
},
provider: {
type: normalizeSceneFusionProviderType(String(entry.provider.type || "")),
},
}));
const outputCapabilities = buildSceneFusionCapabilitySummary(
capabilityCandidates,
config?.modelId || null
);
if (setting?.value) {
const config = setting.value as {
enabled?: boolean;
defaultPrompt?: string;
maxScenes?: number;
preRmbgEnabled?: boolean;
};
if (setting?.value && config) {
return NextResponse.json({
enabled: config.enabled ?? true,
preRmbgEnabled: config.preRmbgEnabled ?? true,
defaultPrompt: config.defaultPrompt || FALLBACK_PROMPT,
maxScenes: Math.min(config.maxScenes ?? SCENE_FUSION_MAX_ITEMS, SCENE_FUSION_MAX_ITEMS),
mode: "diversity",
preferredModelId: config.modelId || null,
outputCapabilities,
});
}
@ -50,6 +83,8 @@ export async function GET() {
defaultPrompt: FALLBACK_PROMPT,
maxScenes: SCENE_FUSION_MAX_ITEMS,
mode: "diversity",
preferredModelId: null,
outputCapabilities,
});
} catch (error) {
console.error("[SceneFusion Config] Error:", error);

View File

@ -25,6 +25,12 @@ import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
import { createFalImage } from "@/lib/ai/providers/fal-image";
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
import {
canProcessSceneFusionProviderType,
normalizeSceneFusionOutputParamsForCandidate,
normalizeSceneFusionProviderType,
type SceneFusionImageGenConfig,
} from "@/lib/scene-fusion/output-capabilities";
// 请求体 Schema
const sceneFusionRequestSchema = z.object({
@ -67,6 +73,7 @@ interface SceneFusionModelCandidate {
modelId: string;
displayName: string;
supportsVision: boolean;
imageGenConfig?: SceneFusionImageGenConfig | null;
};
provider: {
type: string;
@ -89,25 +96,6 @@ interface SceneFusionAgentMeta {
}>;
}
function normalizeSceneFusionProviderType(providerType: string): string {
return providerType === "gemini-image" ? "gemini" : providerType;
}
function canProcessSceneFusionProviderType(providerType: string): boolean {
const normalized = normalizeSceneFusionProviderType(providerType);
switch (normalized) {
case "alibaba-image":
case "ruxa-image":
case "fal-image":
case "openai-image":
case "gemini":
case "volcano-vision":
return true;
default:
return false;
}
}
function pruneSceneFusionCooldown(now: number = Date.now()) {
for (const [key, expireAt] of sceneFusionModelCooldown.entries()) {
if (expireAt <= now) {
@ -676,6 +664,7 @@ function buildSceneFusionCandidatePool(
modelId: String(entry.model.modelId || ""),
displayName: String(entry.model.displayName || entry.model.modelId || entry.model.id),
supportsVision: Boolean(entry.model.supportsVision),
imageGenConfig: (entry.model.imageGenConfig || null) as SceneFusionImageGenConfig | null,
},
provider: {
type: normalizeSceneFusionProviderType(String(entry.provider.type || "")),
@ -1622,6 +1611,16 @@ async function processSceneWithDiversity(
for (let attemptIndex = 0; attemptIndex < attemptCandidates.length; attemptIndex += 1) {
const candidate = attemptCandidates[attemptIndex]!;
try {
const normalizedOutputParams = normalizeSceneFusionOutputParamsForCandidate(candidate, {
aspectRatio: params.aspectRatio,
imageSize: params.imageSize,
});
if (normalizedOutputParams.changed) {
console.log(
`[SceneFusion][${params.requestId}] Candidate ${candidate.model.displayName} output params normalized: ${normalizedOutputParams.changes.join(", ")}`
);
}
const result = await processSceneWithProvider(
params.productImageUrl,
params.sceneImageUrl,
@ -1636,8 +1635,8 @@ async function processSceneWithDiversity(
apiKey: candidate.provider.apiKey,
},
candidate.model.modelId,
params.aspectRatio,
params.imageSize
normalizedOutputParams.aspectRatio,
normalizedOutputParams.imageSize
);
const echoCheck = await detectEchoedProductResult({

View File

@ -19,6 +19,11 @@ import {
import { toast } from "sonner";
import { requestLongTaskResult } from "@/lib/client/long-task";
import { cn } from "@/lib/utils";
import {
SCENE_FUSION_ASPECT_RATIO_OPTIONS,
SCENE_FUSION_IMAGE_SIZE_OPTIONS,
type SceneFusionCapabilitySummary,
} from "@/lib/scene-fusion/output-capabilities";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
@ -54,21 +59,6 @@ interface SceneFusionViewProps {
onBack?: () => void;
}
const aspectRatioOptions = [
{ label: "自动", value: "auto" },
{ label: "1:1", value: "1:1" },
{ label: "4:3", value: "4:3" },
{ label: "3:4", value: "3:4" },
{ label: "16:9", value: "16:9" },
{ label: "9:16", value: "9:16" },
];
const imageSizeOptions = [
{ label: "标准 (1K)", value: "1K" },
{ label: "高清 (2K)", value: "2K" },
{ label: "超清 (4K)", value: "4K" },
];
const randomBackgroundStyles = [
"现代简约客厅",
"北欧卧室",
@ -131,14 +121,40 @@ export function SceneFusionView({
enabled?: boolean;
defaultPrompt?: string;
maxScenes?: number;
outputCapabilities?: SceneFusionCapabilitySummary;
} | undefined;
const maxScenes = Math.min(config?.maxScenes ?? MAX_BATCH_ITEMS, MAX_BATCH_ITEMS);
const outputCapabilities = config?.outputCapabilities;
const availableAspectRatioOptions = useMemo(
() =>
SCENE_FUSION_ASPECT_RATIO_OPTIONS.filter((option) =>
outputCapabilities?.aspectRatio?.enabled === false
? false
: !outputCapabilities?.aspectRatio?.options?.length ||
outputCapabilities.aspectRatio.options.includes(option.value)
),
[outputCapabilities?.aspectRatio?.enabled, outputCapabilities?.aspectRatio?.options]
);
const availableImageSizeOptions = useMemo(
() =>
SCENE_FUSION_IMAGE_SIZE_OPTIONS.filter((option) =>
outputCapabilities?.imageSize?.enabled === false
? false
: !outputCapabilities?.imageSize?.options?.length ||
outputCapabilities.imageSize.options.includes(option.value)
),
[outputCapabilities?.imageSize?.enabled, outputCapabilities?.imageSize?.options]
);
const showAspectRatioSetting = availableAspectRatioOptions.length > 0;
const showImageSizeSetting = availableImageSizeOptions.length > 0;
const [mode, setMode] = useState<BatchMode>("scene-fission");
const [isProcessing, setIsProcessing] = useState(false);
const [aspectRatio, setAspectRatio] = useState("auto");
const [imageSize, setImageSize] = useState("2K");
const showOutputSettings =
mode === "scene-fission" || mode === "random-bg" || mode === "multi-product-scene";
const [productImage, setProductImage] = useState<string | null>(null);
const [productImagePreview, setProductImagePreview] = useState<string | null>(null);
@ -187,6 +203,13 @@ export function SceneFusionView({
() => resultItems.length > 0 && resultItems.every((item) => item.status !== "processing" && item.status !== "pending"),
[resultItems]
);
const sceneFusionOutputPayload = useMemo(
() => ({
aspectRatio: showAspectRatioSetting ? aspectRatio : undefined,
imageSize: showImageSizeSetting ? imageSize : undefined,
}),
[aspectRatio, imageSize, showAspectRatioSetting, showImageSizeSetting]
);
const updateProductPreview = (previewUrl: string | null, isObjectUrl = false) => {
if (productPreviewObjectUrlRef.current) {
@ -219,6 +242,52 @@ export function SceneFusionView({
};
}, []);
useEffect(() => {
if (showAspectRatioSetting) {
const nextAspectRatio =
availableAspectRatioOptions.find((option) => option.value === aspectRatio)?.value ||
outputCapabilities?.aspectRatio?.defaultValue ||
availableAspectRatioOptions[0]?.value ||
"auto";
if (nextAspectRatio !== aspectRatio) {
setAspectRatio(nextAspectRatio);
}
return;
}
if (aspectRatio !== "auto") {
setAspectRatio("auto");
}
}, [
aspectRatio,
availableAspectRatioOptions,
outputCapabilities?.aspectRatio?.defaultValue,
showAspectRatioSetting,
]);
useEffect(() => {
if (showImageSizeSetting) {
const nextImageSize =
availableImageSizeOptions.find((option) => option.value === imageSize)?.value ||
outputCapabilities?.imageSize?.defaultValue ||
availableImageSizeOptions[0]?.value ||
"2K";
if (nextImageSize !== imageSize) {
setImageSize(nextImageSize);
}
return;
}
if (imageSize !== "2K") {
setImageSize("2K");
}
}, [
availableImageSizeOptions,
imageSize,
outputCapabilities?.imageSize?.defaultValue,
showImageSizeSetting,
]);
const handleProductUpload = (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0];
if (!file) return;
@ -419,8 +488,7 @@ export function SceneFusionView({
productImage,
productName: productName || undefined,
sceneMode: mode,
aspectRatio,
imageSize,
...sceneFusionOutputPayload,
scenes: scenesPayload,
}),
});
@ -635,8 +703,7 @@ export function SceneFusionView({
productImage: params.productImageInput,
productName: productName || undefined,
sceneMode: mode,
aspectRatio,
imageSize,
...sceneFusionOutputPayload,
scenes: [
{
id: params.requestId,
@ -1246,49 +1313,77 @@ export function SceneFusionView({
</div>
)}
{(mode === "scene-fission" || mode === "random-bg" || mode === "multi-product-scene") && (
{showOutputSettings && (
<div className="space-y-3">
<Label className="text-base font-medium">输出设置</Label>
<div className="space-y-1.5">
<label className="text-xs text-muted-foreground">图片比例</label>
<div className="flex flex-wrap gap-1.5">
{aspectRatioOptions.map((option) => (
<button
key={option.value}
onClick={() => setAspectRatio(option.value)}
disabled={isProcessing}
className={cn(
"px-2.5 py-1 text-xs rounded-md border transition-colors",
aspectRatio === option.value
? "bg-primary text-primary-foreground border-primary"
: "bg-background hover:bg-muted border-input"
)}
>
{option.label}
</button>
))}
</div>
<div className="space-y-1">
<Label className="text-base font-medium">输出设置</Label>
{outputCapabilities?.strategyLabel ? (
<p className="text-xs text-muted-foreground">{outputCapabilities.strategyLabel}</p>
) : null}
{outputCapabilities?.note ? (
<p className="text-xs text-muted-foreground">{outputCapabilities.note}</p>
) : null}
</div>
<div className="space-y-1.5">
<label className="text-xs text-muted-foreground">输出分辨率</label>
<div className="flex flex-wrap gap-1.5">
{imageSizeOptions.map((option) => (
<button
key={option.value}
onClick={() => setImageSize(option.value)}
disabled={isProcessing}
className={cn(
"px-2.5 py-1 text-xs rounded-md border transition-colors",
imageSize === option.value
? "bg-primary text-primary-foreground border-primary"
: "bg-background hover:bg-muted border-input"
)}
>
{option.label}
</button>
))}
{showAspectRatioSetting ? (
<div className="space-y-1.5">
<label className="text-xs text-muted-foreground">图片比例</label>
<div className="flex flex-wrap gap-1.5">
{availableAspectRatioOptions.map((option) => (
<button
key={option.value}
onClick={() => setAspectRatio(option.value)}
disabled={isProcessing}
className={cn(
"px-2.5 py-1 text-xs rounded-md border transition-colors",
aspectRatio === option.value
? "bg-primary text-primary-foreground border-primary"
: "bg-background hover:bg-muted border-input"
)}
>
{option.label}
</button>
))}
</div>
</div>
</div>
) : (
<div className="space-y-1">
<label className="text-xs text-muted-foreground">图片比例</label>
<p className="text-xs text-muted-foreground">
当前模型链路不支持自定义图片比例,将按模型默认比例生成。
</p>
</div>
)}
{showImageSizeSetting ? (
<div className="space-y-1.5">
<label className="text-xs text-muted-foreground">输出分辨率</label>
<div className="flex flex-wrap gap-1.5">
{availableImageSizeOptions.map((option) => (
<button
key={option.value}
onClick={() => setImageSize(option.value)}
disabled={isProcessing}
className={cn(
"px-2.5 py-1 text-xs rounded-md border transition-colors",
imageSize === option.value
? "bg-primary text-primary-foreground border-primary"
: "bg-background hover:bg-muted border-input"
)}
>
{option.label}
</button>
))}
</div>
</div>
) : (
<div className="space-y-1">
<label className="text-xs text-muted-foreground">输出分辨率</label>
<p className="text-xs text-muted-foreground">
当前模型链路不支持自定义分辨率,将按模型默认清晰度生成。
</p>
</div>
)}
</div>
)}

View File

@ -0,0 +1,281 @@
export const SCENE_FUSION_ASPECT_RATIO_OPTIONS = [
{ label: "自动", value: "auto" },
{ label: "1:1", value: "1:1" },
{ label: "4:3", value: "4:3" },
{ label: "3:4", value: "3:4" },
{ label: "16:9", value: "16:9" },
{ label: "9:16", value: "9:16" },
] as const;
export const SCENE_FUSION_IMAGE_SIZE_OPTIONS = [
{ label: "标准 (1K)", value: "1K" },
{ label: "高清 (2K)", value: "2K" },
{ label: "超清 (4K)", value: "4K" },
] as const;
export type SceneFusionAspectRatioValue =
(typeof SCENE_FUSION_ASPECT_RATIO_OPTIONS)[number]["value"];
export type SceneFusionImageSizeValue =
(typeof SCENE_FUSION_IMAGE_SIZE_OPTIONS)[number]["value"];
export type SceneFusionImageGenConfig = {
supportsSize?: boolean;
supports1K?: boolean;
supports2K?: boolean;
supports4K?: boolean;
};
export type SceneFusionCapabilityCandidate = {
model: {
id: string;
displayName: string;
imageGenConfig?: SceneFusionImageGenConfig | null;
};
provider: {
type: string;
};
};
export type SceneFusionCapabilitySummary = {
strategy: "preferred-model" | "candidate-intersection" | "fallback-default";
strategyLabel: string;
preferredModelId: string | null;
preferredModelName: string | null;
candidateCount: number;
note: string | null;
aspectRatio: {
enabled: boolean;
options: SceneFusionAspectRatioValue[];
defaultValue: SceneFusionAspectRatioValue;
};
imageSize: {
enabled: boolean;
options: SceneFusionImageSizeValue[];
defaultValue: SceneFusionImageSizeValue;
};
};
const DEFAULT_ASPECT_RATIO: SceneFusionAspectRatioValue = "auto";
const DEFAULT_IMAGE_SIZE: SceneFusionImageSizeValue = "2K";
const ALL_ASPECT_RATIO_VALUES = SCENE_FUSION_ASPECT_RATIO_OPTIONS.map(
(option) => option.value
);
const ALL_IMAGE_SIZE_VALUES = SCENE_FUSION_IMAGE_SIZE_OPTIONS.map((option) => option.value);
export function normalizeSceneFusionProviderType(providerType: string): string {
return providerType === "gemini-image" ? "gemini" : providerType;
}
export function canProcessSceneFusionProviderType(providerType: string): boolean {
const normalized = normalizeSceneFusionProviderType(providerType);
switch (normalized) {
case "alibaba-image":
case "ruxa-image":
case "fal-image":
case "openai-image":
case "gemini":
case "volcano-vision":
return true;
default:
return false;
}
}
function orderedIntersect<T extends string>(base: readonly T[], groups: T[][]): T[] {
if (groups.length === 0) return [...base];
return base.filter((item) => groups.every((group) => group.includes(item)));
}
function getSupportedAspectRatiosByProvider(providerType: string): SceneFusionAspectRatioValue[] {
switch (normalizeSceneFusionProviderType(providerType)) {
case "volcano-vision":
return [];
case "ruxa-image":
return ["1:1", "4:3", "3:4", "16:9", "9:16"];
case "alibaba-image":
case "fal-image":
case "openai-image":
case "gemini":
default:
return [...ALL_ASPECT_RATIO_VALUES];
}
}
function getSupportedImageSizesByConfig(
imageGenConfig?: SceneFusionImageGenConfig | null
): SceneFusionImageSizeValue[] {
if (imageGenConfig?.supportsSize === false) {
return [];
}
return ALL_IMAGE_SIZE_VALUES.filter((value) => {
if (value === "1K") return imageGenConfig?.supports1K !== false;
if (value === "2K") return imageGenConfig?.supports2K !== false;
if (value === "4K") return imageGenConfig?.supports4K !== false;
return true;
});
}
function pickDefaultAspectRatio(
options: SceneFusionAspectRatioValue[]
): SceneFusionAspectRatioValue {
return (
options.find((value) => value === DEFAULT_ASPECT_RATIO) ||
options.find((value) => value === "1:1") ||
options[0] ||
DEFAULT_ASPECT_RATIO
);
}
function pickDefaultImageSize(options: SceneFusionImageSizeValue[]): SceneFusionImageSizeValue {
return (
options.find((value) => value === DEFAULT_IMAGE_SIZE) ||
options.find((value) => value === "1K") ||
options.find((value) => value === "4K") ||
options[0] ||
DEFAULT_IMAGE_SIZE
);
}
export function getSceneFusionCandidateCapability(candidate: SceneFusionCapabilityCandidate) {
const aspectRatioOptions = getSupportedAspectRatiosByProvider(candidate.provider.type);
const imageSizeOptions = getSupportedImageSizesByConfig(candidate.model.imageGenConfig);
return {
aspectRatio: {
enabled: aspectRatioOptions.length > 0,
options: aspectRatioOptions,
defaultValue: pickDefaultAspectRatio(aspectRatioOptions),
},
imageSize: {
enabled: imageSizeOptions.length > 0,
options: imageSizeOptions,
defaultValue: pickDefaultImageSize(imageSizeOptions),
},
};
}
export function buildSceneFusionCapabilitySummary(
candidates: SceneFusionCapabilityCandidate[],
preferredModelId?: string | null
): SceneFusionCapabilitySummary {
const supportedCandidates = candidates.filter((candidate) =>
canProcessSceneFusionProviderType(candidate.provider.type)
);
if (supportedCandidates.length === 0) {
return {
strategy: "fallback-default",
strategyLabel: "未检测到可用模型,使用默认参数",
preferredModelId: null,
preferredModelName: null,
candidateCount: 0,
note: "后台当前没有可用于批量处理的生图模型,以下参数仅作为默认值展示。",
aspectRatio: {
enabled: true,
options: [...ALL_ASPECT_RATIO_VALUES],
defaultValue: DEFAULT_ASPECT_RATIO,
},
imageSize: {
enabled: true,
options: [...ALL_IMAGE_SIZE_VALUES],
defaultValue: DEFAULT_IMAGE_SIZE,
},
};
}
const preferredCandidate =
preferredModelId
? supportedCandidates.find((candidate) => candidate.model.id === preferredModelId) || null
: null;
const sourceCandidates = preferredCandidate ? [preferredCandidate] : supportedCandidates;
const strategy = preferredCandidate ? "preferred-model" : "candidate-intersection";
const aspectRatioOptions = orderedIntersect(
ALL_ASPECT_RATIO_VALUES,
sourceCandidates.map((candidate) => getSceneFusionCandidateCapability(candidate).aspectRatio.options)
);
const imageSizeOptions = orderedIntersect(
ALL_IMAGE_SIZE_VALUES,
sourceCandidates.map((candidate) => getSceneFusionCandidateCapability(candidate).imageSize.options)
);
return {
strategy,
strategyLabel: preferredCandidate
? `按首选模型 ${preferredCandidate.model.displayName} 的能力展示`
: `未指定首选模型,按 ${supportedCandidates.length} 个候选模型交集展示`,
preferredModelId: preferredCandidate?.model.id || null,
preferredModelName: preferredCandidate?.model.displayName || null,
candidateCount: supportedCandidates.length,
note: preferredCandidate
? supportedCandidates.length > 1
? "当首选模型失败回退到其他模型时,系统会自动按目标模型能力降级不支持的参数。"
: "当前仅使用一个模型,参数会严格按该模型能力生效。"
: "多模型轮换时,不支持的参数已在前端按交集隐藏,后端也会继续做能力兜底。",
aspectRatio: {
enabled: aspectRatioOptions.length > 0,
options: aspectRatioOptions,
defaultValue: pickDefaultAspectRatio(aspectRatioOptions),
},
imageSize: {
enabled: imageSizeOptions.length > 0,
options: imageSizeOptions,
defaultValue: pickDefaultImageSize(imageSizeOptions),
},
};
}
export function normalizeSceneFusionOutputParamsForCandidate(
candidate: SceneFusionCapabilityCandidate,
requested: {
aspectRatio?: string | null;
imageSize?: string | null;
}
) {
const capability = getSceneFusionCandidateCapability(candidate);
const requestedAspectRatio = String(requested.aspectRatio || "").trim() as SceneFusionAspectRatioValue;
const requestedImageSize = String(requested.imageSize || "").trim().toUpperCase() as SceneFusionImageSizeValue;
const aspectRatio =
!capability.aspectRatio.enabled
? undefined
: capability.aspectRatio.options.includes(requestedAspectRatio)
? requestedAspectRatio
: requestedAspectRatio
? capability.aspectRatio.defaultValue
: undefined;
const imageSize =
!capability.imageSize.enabled
? undefined
: capability.imageSize.options.includes(requestedImageSize)
? requestedImageSize
: requestedImageSize
? capability.imageSize.defaultValue
: undefined;
const changes: string[] = [];
if (requestedAspectRatio && aspectRatio !== requestedAspectRatio) {
changes.push(
aspectRatio
? `aspectRatio:${requestedAspectRatio}->${aspectRatio}`
: `aspectRatio:${requestedAspectRatio}->default`
);
}
if (requestedImageSize && imageSize !== requestedImageSize) {
changes.push(
imageSize
? `imageSize:${requestedImageSize}->${imageSize}`
: `imageSize:${requestedImageSize}->default`
);
}
return {
aspectRatio,
imageSize,
changed: changes.length > 0,
changes,
};
}

File diff suppressed because one or more lines are too long