fix: reduce scale adjust false failures
This commit is contained in:
parent
7959dd6b3a
commit
e5bcf6f403
@ -111,6 +111,34 @@ export type ImageFallbackCandidate = {
|
|||||||
sameProvider: boolean;
|
sameProvider: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function messagePartHasImageInput(part: any) {
|
||||||
|
if (!part || typeof part !== "object") return false;
|
||||||
|
if (part.type === "file" || part.type === "image" || part.type === "image_url") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (typeof part.url === "string" && /^https?:\/\//i.test(part.url)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (typeof part.image === "string" && part.image.trim()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (typeof part.image_url?.url === "string" && part.image_url.url.trim()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function chatMessagesHaveImageInput(messages: ChatMessage[]) {
|
||||||
|
return messages.some((message: any) => {
|
||||||
|
const content = message?.content;
|
||||||
|
if (Array.isArray(content) && content.some(messagePartHasImageInput)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const parts = message?.parts;
|
||||||
|
return Array.isArray(parts) && parts.some(messagePartHasImageInput);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export function getImageFailoverConvertProvider(providerType?: string | null) {
|
export function getImageFailoverConvertProvider(providerType?: string | null) {
|
||||||
switch (providerType) {
|
switch (providerType) {
|
||||||
case "bfl-image":
|
case "bfl-image":
|
||||||
@ -219,6 +247,11 @@ export async function generateImageWithFailoverCandidate(params: {
|
|||||||
const { candidate, messages, imageParams, fallbackImageParams } = params;
|
const { candidate, messages, imageParams, fallbackImageParams } = params;
|
||||||
const providerType = String(candidate.provider?.type || "");
|
const providerType = String(candidate.provider?.type || "");
|
||||||
const candidateRuntimeModelId = resolveImageModelRuntimeId(candidate.model);
|
const candidateRuntimeModelId = resolveImageModelRuntimeId(candidate.model);
|
||||||
|
if (chatMessagesHaveImageInput(messages) && candidate.model?.supportsVision === false) {
|
||||||
|
throw new Error(
|
||||||
|
`备用模型不支持参考图/图生图输入:${candidate.model.displayName || candidate.model.id}`
|
||||||
|
);
|
||||||
|
}
|
||||||
if (
|
if (
|
||||||
providerType === "fal-image" &&
|
providerType === "fal-image" &&
|
||||||
isFalGptImage2EditModelId(candidateRuntimeModelId)
|
isFalGptImage2EditModelId(candidateRuntimeModelId)
|
||||||
@ -237,9 +270,18 @@ export async function generateImageWithFailoverCandidate(params: {
|
|||||||
messages,
|
messages,
|
||||||
getImageFailoverConvertProvider(providerType)
|
getImageFailoverConvertProvider(providerType)
|
||||||
);
|
);
|
||||||
|
const effectiveImageParams = imageParams || fallbackImageParams;
|
||||||
|
chatApiLogger.info("[Chat API] Image failover candidate params:", {
|
||||||
|
model: candidate.model?.modelId || candidate.model?.id,
|
||||||
|
providerType,
|
||||||
|
aspectRatio: effectiveImageParams?.aspectRatio,
|
||||||
|
imageSize: effectiveImageParams?.imageSize,
|
||||||
|
imageCount: effectiveImageParams?.imageCount,
|
||||||
|
hasImageInput: chatMessagesHaveImageInput(messages),
|
||||||
|
});
|
||||||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||||||
messages: candidateMessages,
|
messages: candidateMessages,
|
||||||
imageParams: imageParams || fallbackImageParams,
|
imageParams: effectiveImageParams,
|
||||||
imageGenConfig: candidate.model.imageGenConfig || null,
|
imageGenConfig: candidate.model.imageGenConfig || null,
|
||||||
modelDisplayName: candidate.model.displayName,
|
modelDisplayName: candidate.model.displayName,
|
||||||
overrideModelId: candidateRuntimeModelId || candidate.model.modelId,
|
overrideModelId: candidateRuntimeModelId || candidate.model.modelId,
|
||||||
|
|||||||
@ -892,6 +892,12 @@ export async function POST(request: NextRequest) {
|
|||||||
? siteSettingsForModel.canvasDetailDefaultModelId.trim()
|
? siteSettingsForModel.canvasDetailDefaultModelId.trim()
|
||||||
: "";
|
: "";
|
||||||
const resolvedModel = detailFixedModel || requestedModel;
|
const resolvedModel = detailFixedModel || requestedModel;
|
||||||
|
const scaleAdjustCompositeRegions = isScaleAdjustOperation
|
||||||
|
? buildScaleAdjustCompositeRegions(
|
||||||
|
regions,
|
||||||
|
scalePercent as ScaleAdjustPercent
|
||||||
|
)
|
||||||
|
: [];
|
||||||
|
|
||||||
const imageRequest: CanvasGenerateImageRequest = {
|
const imageRequest: CanvasGenerateImageRequest = {
|
||||||
aspectRatio: resolvedAspectRatio,
|
aspectRatio: resolvedAspectRatio,
|
||||||
@ -937,6 +943,7 @@ export async function POST(request: NextRequest) {
|
|||||||
referenceImages,
|
referenceImages,
|
||||||
scaleAdjustVerification: isScaleAdjustOperation
|
scaleAdjustVerification: isScaleAdjustOperation
|
||||||
? {
|
? {
|
||||||
|
protectedRegions: scaleAdjustCompositeRegions,
|
||||||
sourceImageUrl,
|
sourceImageUrl,
|
||||||
regions: regions.map((region) => region.rect),
|
regions: regions.map((region) => region.rect),
|
||||||
scalePercent: scalePercent as ScaleAdjustPercent,
|
scalePercent: scalePercent as ScaleAdjustPercent,
|
||||||
@ -945,10 +952,7 @@ export async function POST(request: NextRequest) {
|
|||||||
localEditComposite: isScaleAdjustOperation
|
localEditComposite: isScaleAdjustOperation
|
||||||
? {
|
? {
|
||||||
sourceImageUrl,
|
sourceImageUrl,
|
||||||
regions: buildScaleAdjustCompositeRegions(
|
regions: scaleAdjustCompositeRegions,
|
||||||
regions,
|
|
||||||
scalePercent as ScaleAdjustPercent
|
|
||||||
),
|
|
||||||
expandRegions: false,
|
expandRegions: false,
|
||||||
maskMode: "diff",
|
maskMode: "diff",
|
||||||
featherMinPx: 4,
|
featherMinPx: 4,
|
||||||
|
|||||||
@ -51,6 +51,7 @@ export interface CanvasLocalEditCompositeConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface CanvasScaleAdjustVerificationConfig {
|
export interface CanvasScaleAdjustVerificationConfig {
|
||||||
|
protectedRegions?: CanvasLocalEditCompositeRegion[];
|
||||||
sourceImageUrl: string;
|
sourceImageUrl: string;
|
||||||
regions: CanvasLocalEditCompositeRegion[];
|
regions: CanvasLocalEditCompositeRegion[];
|
||||||
scalePercent: number;
|
scalePercent: number;
|
||||||
@ -334,8 +335,22 @@ function normalizeScaleAdjustVerificationConfig(
|
|||||||
const regions = value.regions
|
const regions = value.regions
|
||||||
.map((region) => normalizeCanvasRegion(region))
|
.map((region) => normalizeCanvasRegion(region))
|
||||||
.filter((region): region is CanvasLocalEditCompositeRegion => Boolean(region));
|
.filter((region): region is CanvasLocalEditCompositeRegion => Boolean(region));
|
||||||
|
const protectedRegions = Array.isArray(value.protectedRegions)
|
||||||
|
? value.protectedRegions
|
||||||
|
.map((region) => normalizeCanvasRegion(region))
|
||||||
|
.filter((region): region is CanvasLocalEditCompositeRegion =>
|
||||||
|
Boolean(region)
|
||||||
|
)
|
||||||
|
: [];
|
||||||
|
|
||||||
return regions.length > 0 ? { sourceImageUrl, regions, scalePercent } : null;
|
return regions.length > 0
|
||||||
|
? {
|
||||||
|
sourceImageUrl,
|
||||||
|
regions,
|
||||||
|
scalePercent,
|
||||||
|
...(protectedRegions.length > 0 ? { protectedRegions } : {}),
|
||||||
|
}
|
||||||
|
: null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function scaleRegionAroundCenter(
|
function scaleRegionAroundCenter(
|
||||||
@ -402,6 +417,11 @@ function canvasRegionsIntersect(
|
|||||||
function getScaleAdjustProtectedRegions(
|
function getScaleAdjustProtectedRegions(
|
||||||
verification: CanvasScaleAdjustVerificationConfig
|
verification: CanvasScaleAdjustVerificationConfig
|
||||||
) {
|
) {
|
||||||
|
if (verification.protectedRegions?.length) {
|
||||||
|
return verification.protectedRegions.map((region) =>
|
||||||
|
expandCanvasRegion(region, SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN)
|
||||||
|
);
|
||||||
|
}
|
||||||
return verification.regions.flatMap((region) => [
|
return verification.regions.flatMap((region) => [
|
||||||
expandCanvasRegion(region, SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN),
|
expandCanvasRegion(region, SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN),
|
||||||
expandCanvasRegion(
|
expandCanvasRegion(
|
||||||
@ -682,6 +702,7 @@ async function applyScaleAdjustVerificationToImages(params: {
|
|||||||
outsideDiffScores: outsideDiffScores.map(
|
outsideDiffScores: outsideDiffScores.map(
|
||||||
(score) => Math.round(score * 10) / 10
|
(score) => Math.round(score * 10) / 10
|
||||||
),
|
),
|
||||||
|
protectedRegionCount: verification.protectedRegions?.length || 0,
|
||||||
scalePercent: verification.scalePercent,
|
scalePercent: verification.scalePercent,
|
||||||
});
|
});
|
||||||
checkedImages.push(image);
|
checkedImages.push(image);
|
||||||
@ -700,6 +721,7 @@ async function applyScaleAdjustVerificationToImages(params: {
|
|||||||
outsideDiffScores: outsideDiffScores.map(
|
outsideDiffScores: outsideDiffScores.map(
|
||||||
(score) => Math.round(score * 10) / 10
|
(score) => Math.round(score * 10) / 10
|
||||||
),
|
),
|
||||||
|
protectedRegionCount: verification.protectedRegions?.length || 0,
|
||||||
reason: rejection.reason,
|
reason: rejection.reason,
|
||||||
scalePercent: verification.scalePercent,
|
scalePercent: verification.scalePercent,
|
||||||
});
|
});
|
||||||
@ -1202,6 +1224,7 @@ function findCanvasFailoverModel(params: {
|
|||||||
selectedModel: CanvasModelWithProvider;
|
selectedModel: CanvasModelWithProvider;
|
||||||
imageModels: CanvasModelWithProvider[];
|
imageModels: CanvasModelWithProvider[];
|
||||||
failedModelRefs: string[];
|
failedModelRefs: string[];
|
||||||
|
requiresImageInput?: boolean;
|
||||||
}) {
|
}) {
|
||||||
const failedRefs = new Set(
|
const failedRefs = new Set(
|
||||||
params.failedModelRefs.map((item) => String(item || "").trim()).filter(Boolean)
|
params.failedModelRefs.map((item) => String(item || "").trim()).filter(Boolean)
|
||||||
@ -1218,6 +1241,9 @@ function findCanvasFailoverModel(params: {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!item.model.supportsImageGen) return false;
|
if (!item.model.supportsImageGen) return false;
|
||||||
|
if (params.requiresImageInput && item.model.supportsVision === false) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(item.provider.type)) return false;
|
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(item.provider.type)) return false;
|
||||||
return getCanvasModelRefKeys(item).every((key) => !failedRefs.has(key));
|
return getCanvasModelRefKeys(item).every((key) => !failedRefs.has(key));
|
||||||
};
|
};
|
||||||
@ -1319,8 +1345,49 @@ export async function generateCanvasImage(
|
|||||||
selectedModel = imageModels[0];
|
selectedModel = imageModels[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const normalizedReferenceImages = Array.from(
|
||||||
|
new Set(
|
||||||
|
[
|
||||||
|
...referenceImages,
|
||||||
|
...(referenceImage ? [referenceImage] : []),
|
||||||
|
]
|
||||||
|
.map((item) => String(item || "").trim())
|
||||||
|
.filter((url) => /^https?:\/\//i.test(url))
|
||||||
|
)
|
||||||
|
).slice(0, 12);
|
||||||
|
const requiresImageInput = normalizedReferenceImages.length > 0;
|
||||||
|
|
||||||
let provider = selectedModel.provider;
|
let provider = selectedModel.provider;
|
||||||
|
|
||||||
|
if (requiresImageInput && selectedModel.model.supportsVision === false) {
|
||||||
|
if (request.allowFailover === false) {
|
||||||
|
throw new Error("当前图片模型不支持参考图/图生图,请选择支持图像输入的生图模型。");
|
||||||
|
}
|
||||||
|
|
||||||
|
const imageInputFallback = findCanvasFailoverModel({
|
||||||
|
selectedModel,
|
||||||
|
imageModels,
|
||||||
|
failedModelRefs: getCanvasModelRefKeys(selectedModel),
|
||||||
|
requiresImageInput,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!imageInputFallback) {
|
||||||
|
throw new Error("当前任务需要参考图输入,但暂无可用的图生图模型。");
|
||||||
|
}
|
||||||
|
|
||||||
|
console.warn("[Canvas Generate Image] Fallback to reference-image capable model:", {
|
||||||
|
debugLabel,
|
||||||
|
requestedModel: selectedModel.model.modelId || selectedModel.model.id,
|
||||||
|
fallbackModel: imageInputFallback.model.modelId || imageInputFallback.model.id,
|
||||||
|
requestedProviderType: provider.type,
|
||||||
|
fallbackProviderType: imageInputFallback.provider.type,
|
||||||
|
reason: "selected-model-lacks-reference-image-support",
|
||||||
|
});
|
||||||
|
|
||||||
|
selectedModel = imageInputFallback;
|
||||||
|
provider = selectedModel.provider;
|
||||||
|
}
|
||||||
|
|
||||||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(provider.type)) {
|
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(provider.type)) {
|
||||||
if (request.allowFailover === false) {
|
if (request.allowFailover === false) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
@ -1333,6 +1400,7 @@ export async function generateCanvasImage(
|
|||||||
for (const candidate of imageModels) {
|
for (const candidate of imageModels) {
|
||||||
const candidateProvider = candidate.provider;
|
const candidateProvider = candidate.provider;
|
||||||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(candidateProvider.type)) continue;
|
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(candidateProvider.type)) continue;
|
||||||
|
if (requiresImageInput && candidate.model.supportsVision === false) continue;
|
||||||
fallbackModel = candidate;
|
fallbackModel = candidate;
|
||||||
fallbackProvider = candidateProvider;
|
fallbackProvider = candidateProvider;
|
||||||
break;
|
break;
|
||||||
@ -1374,16 +1442,6 @@ export async function generateCanvasImage(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const normalizedReferenceImages = Array.from(
|
|
||||||
new Set(
|
|
||||||
[
|
|
||||||
...referenceImages,
|
|
||||||
...(referenceImage ? [referenceImage] : []),
|
|
||||||
]
|
|
||||||
.map((item) => String(item || "").trim())
|
|
||||||
.filter((url) => /^https?:\/\//i.test(url))
|
|
||||||
)
|
|
||||||
).slice(0, 12);
|
|
||||||
const { kept: reachableReferenceImages, dropped: unreachableReferenceImages } =
|
const { kept: reachableReferenceImages, dropped: unreachableReferenceImages } =
|
||||||
await filterReachableRemoteUrls(normalizedReferenceImages, { timeoutMs: 3000 });
|
await filterReachableRemoteUrls(normalizedReferenceImages, { timeoutMs: 3000 });
|
||||||
const effectiveReferenceImages =
|
const effectiveReferenceImages =
|
||||||
@ -1850,6 +1908,7 @@ export async function generateCanvasImage(
|
|||||||
selectedModel,
|
selectedModel,
|
||||||
imageModels,
|
imageModels,
|
||||||
failedModelRefs,
|
failedModelRefs,
|
||||||
|
requiresImageInput: normalizedReferenceImages.length > 0,
|
||||||
})
|
})
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user