fix: stabilize detail replacement generation

This commit is contained in:
134 2026-09-11 11:10:20 +08:00
parent dcff190c62
commit a2a9e7d8b2
21 changed files with 1167 additions and 120 deletions

View File

@ -20,6 +20,7 @@ import {
type ReplacementProductIdentityProfile,
} from "@/lib/ai/replacement-product-identity";
import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime";
import { resolveDetailReplacementVisionRuntime } from "@/lib/ai/detail-replacement-vision-runtime";
import { inlineVisionImages } from "@/lib/ai/vision-inline";
import {
getEnabledAiModelsWithProvider,
@ -594,6 +595,7 @@ async function resolveReplacementProductIdentityProfile(params: {
timeoutMs: 12_000,
retries: 2,
loggerPrefix: "[Chat API] Replacement product identity",
trimTransparentPadding: true,
});
const effectiveImages =
inlineDataUrls.length === productImageUrls.length
@ -2449,6 +2451,16 @@ export async function analyzeReplacementReferenceImage(params: {
export async function resolveReplacementScenePlanningRuntime(params: {
models?: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>;
}): Promise<ReplacementScenePlanningRuntime> {
try {
const dedicatedVisionRuntime = await resolveDetailReplacementVisionRuntime();
if (dedicatedVisionRuntime) return dedicatedVisionRuntime;
} catch (error) {
chatApiLogger.warn(
"[Chat API] Failed to resolve dedicated detail replacement vision model:",
error,
);
}
try {
const siteSettings = await getSiteSettings().catch(() => null);
const localDetailAgentResolved = await resolveDetailAgentLocalMoonshotModel({

View File

@ -364,6 +364,7 @@ import {
resolveReplacementReferenceGeometry,
resolveReplacementScenePlanningRuntime,
} from "./_lib/detail-replacement";
import { selectDetailReplacementGenerationInputs } from "@/lib/ai/detail-replacement-inputs";
import {
normalizeAgentReportForDisplay,
sanitizeAgentOutputSettingsSection,
@ -5938,16 +5939,14 @@ export async function POST(request: Request) {
replacementExecutionPlan.scenePlan,
) ||
selectedProductContextImageUrls.length === 0);
const sourceImageUrls = Array.from(
new Set([
...selectedProductContextImageUrls,
...(shouldIncludeModelImage && modelInputImageUrl
? [modelInputImageUrl]
: []),
...rerunReferenceImageUrls,
reference.imageUrl,
]),
);
const generationInputs = selectDetailReplacementGenerationInputs({
productImageUrls: selectedProductContextImageUrls,
modelImageUrl: modelInputImageUrl,
includeModelImage: shouldIncludeModelImage,
rerunReferenceImageUrls,
referenceImageUrl: reference.imageUrl,
});
const sourceImageUrls = generationInputs.imageUrls;
chatApiLogger.info("[Chat API] Direct detail replacement image generation started:", {
referenceIndex: referenceIndex + 1,
totalReferences: replacementReferences.length,
@ -5955,10 +5954,10 @@ export async function POST(request: Request) {
pageKind: replacementExecutionPlan.scenePlan.pageKind,
generationRoute: replacementExecutionPlan.scenePlan.generationRoute,
packPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy,
selectedProductImages: selectedProductContextImageUrls.length,
selectedProductImages: generationInputs.productImageUrls.length,
sourceProductImages: productInputImageUrls.length,
includeModelImage: shouldIncludeModelImage,
rerunReferenceImages: rerunReferenceImageUrls.length,
includeModelImage: Boolean(generationInputs.modelImageUrl),
rerunReferenceImages: generationInputs.rerunImageUrls.length,
sourceImageCount: sourceImageUrls.length,
whiteLayout: replacementExecutionPlan.scenePlan.whiteBackgroundLikely,
copyCoupled: replacementExecutionPlan.scenePlan.copyImageCoupled,
@ -5973,10 +5972,10 @@ export async function POST(request: Request) {
referenceLabel: reference.label,
referenceIndex,
totalReferences: replacementReferences.length,
hasModelImage: shouldIncludeModelImage,
productImageCount: selectedProductContextImageUrls.length,
hasModelImage: Boolean(generationInputs.modelImageUrl),
productImageCount: generationInputs.productImageUrls.length,
originalProductImageCount: productInputImageUrls.length,
rerunReferenceCount: rerunReferenceImageUrls.length,
rerunReferenceCount: generationInputs.rerunImageUrls.length,
templateReferenceCount: 0,
scenePlan: replacementExecutionPlan.scenePlan,
referenceGeometry,
@ -6008,6 +6007,10 @@ export async function POST(request: Request) {
imageSize: resolvedImageSize,
imageCount: 1,
referenceImages: sourceImageUrls,
promptExtend: false,
providerImageParams: {
inputImageRoleByUrl: generationInputs.inputImageRoleByUrl,
},
debugLabel: "agent-detail-replacement",
debugMeta: {
referenceLabel: reference.label,
@ -6830,19 +6833,31 @@ export async function POST(request: Request) {
) ||
(isDetailReplacementPreset &&
selectedProductContextImageUrls.length === 0));
const sourceImageUrls = Array.from(
new Set(
[
...selectedProductContextImageUrls,
...(shouldIncludeModelImage && modelInputImageUrl
? [modelInputImageUrl]
: []),
...replacementTemplateReferenceImageUrls,
...rerunReferenceImageUrls,
reference.imageUrl,
].filter(Boolean),
),
);
const detailGenerationInputs = isDetailReplacementPreset
? selectDetailReplacementGenerationInputs({
productImageUrls: selectedProductContextImageUrls,
modelImageUrl: modelInputImageUrl,
includeModelImage: shouldIncludeModelImage,
templateReferenceImageUrls: replacementTemplateReferenceImageUrls,
rerunReferenceImageUrls,
referenceImageUrl: reference.imageUrl,
})
: null;
const sourceImageUrls = detailGenerationInputs
? detailGenerationInputs.imageUrls
: Array.from(
new Set(
[
...selectedProductContextImageUrls,
...(shouldIncludeModelImage && modelInputImageUrl
? [modelInputImageUrl]
: []),
...replacementTemplateReferenceImageUrls,
...rerunReferenceImageUrls,
reference.imageUrl,
].filter(Boolean),
),
);
const replacementPrompt = buildAgentReplacementPrompt({
basePrompt:
baseReplacementPrompt ||
@ -6859,11 +6874,19 @@ export async function POST(request: Request) {
referenceLabel: reference.label,
referenceIndex,
totalReferences: replacementReferences.length,
hasModelImage: shouldIncludeModelImage,
productImageCount: selectedProductContextImageUrls.length,
hasModelImage: detailGenerationInputs
? Boolean(detailGenerationInputs.modelImageUrl)
: shouldIncludeModelImage,
productImageCount: detailGenerationInputs
? detailGenerationInputs.productImageUrls.length
: selectedProductContextImageUrls.length,
originalProductImageCount: productInputImageUrls.length,
rerunReferenceCount: rerunReferenceImageUrls.length,
templateReferenceCount: replacementTemplateReferenceImageUrls.length,
rerunReferenceCount: detailGenerationInputs
? detailGenerationInputs.rerunImageUrls.length
: rerunReferenceImageUrls.length,
templateReferenceCount: detailGenerationInputs
? detailGenerationInputs.templateImageUrls.length
: replacementTemplateReferenceImageUrls.length,
pairedProductIndex,
scenePlan: replacementExecutionPlan.scenePlan,
referenceGeometry,
@ -6884,11 +6907,20 @@ export async function POST(request: Request) {
sourceImageCount: sourceImageUrls.length,
generationAspectRatio,
imageOrder: {
productImages: selectedProductContextImageUrls.length,
productImages:
detailGenerationInputs !== null
? detailGenerationInputs.productImageUrls.length
: selectedProductContextImageUrls.length,
originalProductImages: productInputImageUrls.length,
hasModelImage: shouldIncludeModelImage,
templateReferenceImages: replacementTemplateReferenceImageUrls.length,
rerunReferenceImages: rerunReferenceImageUrls.length,
hasModelImage: detailGenerationInputs
? Boolean(detailGenerationInputs.modelImageUrl)
: shouldIncludeModelImage,
templateReferenceImages: detailGenerationInputs
? detailGenerationInputs.templateImageUrls.length
: replacementTemplateReferenceImageUrls.length,
rerunReferenceImages: detailGenerationInputs
? detailGenerationInputs.rerunImageUrls.length
: rerunReferenceImageUrls.length,
finalReferenceImageIndex: sourceImageUrls.length,
},
});
@ -6906,6 +6938,17 @@ export async function POST(request: Request) {
imageSize: resolvedImageSize,
imageCount: 1,
referenceImages: sourceImageUrls,
promptExtend: isDetailReplacementPreset
? false
: typeof mergedImageParams.promptExtend === "boolean"
? mergedImageParams.promptExtend
: undefined,
providerImageParams: detailGenerationInputs
? {
inputImageRoleByUrl:
detailGenerationInputs.inputImageRoleByUrl,
}
: undefined,
debugLabel: `agent-${replacementSourceType}`,
debugMeta: {
referenceLabel: reference.label,

View File

@ -25,6 +25,36 @@ export type DetailAgentLocalMoonshotConfig = {
displayName: string;
};
const ALIBABA_DASHSCOPE_HOST = "dashscope.aliyuncs.com";
export function isAlibabaDashScopeBaseUrl(value: unknown) {
try {
return new URL(String(value || "").trim()).hostname === ALIBABA_DASHSCOPE_HOST;
} catch {
return false;
}
}
export function resolveDetailAgentApiKey(params: {
configuredApiKey: string;
configuredBaseUrl: string;
enabledAlibabaProviders: Array<{
apiKey?: string | null;
baseUrl?: string | null;
}>;
}) {
if (isAlibabaDashScopeBaseUrl(params.configuredBaseUrl)) {
const provider = params.enabledAlibabaProviders.find(
(item) =>
isAlibabaDashScopeBaseUrl(item.baseUrl) &&
Boolean(String(item.apiKey || "").trim()),
);
if (provider?.apiKey) return provider.apiKey.trim();
}
return String(params.configuredApiKey || "").trim();
}
export function isDetailAgentLocalMoonshotOption(value: unknown) {
return String(value || "").trim() === DETAIL_AGENT_LOCAL_MOONSHOT_OPTION_ID;
}

View File

@ -2,13 +2,14 @@ import "server-only";
import { createOpenAI } from "@ai-sdk/openai";
import type { LanguageModel } from "ai";
import { getSystemSetting } from "@/lib/db/queries";
import { getAiProvidersByType, getSystemSetting } from "@/lib/db/queries";
import {
DETAIL_AGENT_LOCAL_MOONSHOT_BASE_URL,
DETAIL_AGENT_LOCAL_MOONSHOT_OPTION_ID,
DETAIL_AGENT_LOCAL_MOONSHOT_SETTING_KEY,
isDetailAgentLocalMoonshotOption,
normalizeDetailAgentLocalMoonshotConfig,
resolveDetailAgentApiKey,
} from "@/lib/ai/detail-agent-model-config";
import { createMoonshotKimiFetch } from "@/lib/ai/detail-agent-kimi-profile";
@ -23,13 +24,22 @@ export async function resolveDetailAgentLocalMoonshotModel(params: {
key: DETAIL_AGENT_LOCAL_MOONSHOT_SETTING_KEY,
}).catch(() => null);
const config = normalizeDetailAgentLocalMoonshotConfig(setting?.value);
const enabledAlibabaProviders = await getAiProvidersByType({
type: "alibaba",
enabledOnly: true,
}).catch(() => []);
const apiKey = resolveDetailAgentApiKey({
configuredApiKey: config.apiKey,
configuredBaseUrl: config.baseUrl,
enabledAlibabaProviders,
});
if (!config.enabled || !config.apiKey.trim()) {
if (!config.enabled || !apiKey) {
return null;
}
const provider = createOpenAI({
apiKey: config.apiKey,
apiKey,
baseURL: config.baseUrl || DETAIL_AGENT_LOCAL_MOONSHOT_BASE_URL,
fetch: createMoonshotKimiFetch(),
});

View File

@ -1,7 +1,7 @@
export const DETAIL_REPLACEMENT_IMAGE_MODEL_CONFIG_KEY =
"detail_replacement_image_model_config";
export const DEFAULT_DETAIL_REPLACEMENT_IMAGE_MODEL_ID = "qwen-image-3.0";
export const DEFAULT_DETAIL_REPLACEMENT_IMAGE_MODEL_ID = "gpt-image-2.5-flare";
export type DetailReplacementImageModelConfig = {
modelId: string;

View File

@ -0,0 +1,146 @@
export type DetailReplacementGenerationInputRole =
| "product"
| "model"
| "supplement"
| "layout";
export function selectDetailReplacementGenerationInputs(params: {
productImageUrls: string[];
modelImageUrl?: string | null;
includeModelImage?: boolean;
templateReferenceImageUrls?: string[];
rerunReferenceImageUrls?: string[];
referenceImageUrl: string;
maxImages?: number;
}) {
const normalizeUrls = (values: Array<string | null | undefined>) =>
Array.from(
new Set(
values
.map((value) => String(value || "").trim())
.filter((value) => /^https?:\/\//i.test(value)),
),
);
const referenceImageUrl = String(params.referenceImageUrl || "").trim();
const productImageUrls = normalizeUrls(params.productImageUrls).filter(
(url) => url !== referenceImageUrl,
);
const modelImageUrl = params.includeModelImage
? normalizeUrls([params.modelImageUrl]).find((url) => url !== referenceImageUrl) || null
: null;
const rerunReferenceImageUrls = normalizeUrls(
params.rerunReferenceImageUrls || [],
).filter((url) => url !== referenceImageUrl);
const templateReferenceImageUrls = normalizeUrls(
params.templateReferenceImageUrls || [],
).filter((url) => url !== referenceImageUrl);
const maxImages = Math.max(2, Math.floor(params.maxImages || 3));
const contentCapacity = maxImages - 1;
const selectedProductImageUrls: string[] = [];
const selectedTemplateImageUrls: string[] = [];
const selectedRerunImageUrls: string[] = [];
let selectedModelImageUrl: string | null = null;
if (productImageUrls[0]) selectedProductImageUrls.push(productImageUrls[0]);
if (modelImageUrl && selectedProductImageUrls.length < contentCapacity) {
selectedModelImageUrl = modelImageUrl;
}
const usedCount = () =>
selectedProductImageUrls.length +
selectedTemplateImageUrls.length +
selectedRerunImageUrls.length +
(selectedModelImageUrl ? 1 : 0);
const addSupplement = (url: string | undefined, kind: "template" | "rerun") => {
if (!url || usedCount() >= contentCapacity) return;
const selectedSupplementImageUrls = [
...selectedTemplateImageUrls,
...selectedRerunImageUrls,
];
if (
selectedProductImageUrls.includes(url) ||
selectedSupplementImageUrls.includes(url) ||
selectedModelImageUrl === url
) {
return;
}
if (kind === "template") selectedTemplateImageUrls.push(url);
else selectedRerunImageUrls.push(url);
};
if (!selectedModelImageUrl) {
addSupplement(rerunReferenceImageUrls[0], "rerun");
if (usedCount() < contentCapacity && productImageUrls[1]) {
selectedProductImageUrls.push(productImageUrls[1]);
}
addSupplement(templateReferenceImageUrls[0], "template");
} else if (selectedProductImageUrls.length === 0) {
addSupplement(rerunReferenceImageUrls[0], "rerun");
addSupplement(templateReferenceImageUrls[0], "template");
}
while (usedCount() < contentCapacity) {
const nextProduct = productImageUrls.find(
(url) => !selectedProductImageUrls.includes(url),
);
if (nextProduct) {
selectedProductImageUrls.push(nextProduct);
continue;
}
const nextRerun = rerunReferenceImageUrls.find(
(url) =>
!selectedRerunImageUrls.includes(url) &&
!selectedTemplateImageUrls.includes(url) &&
!selectedProductImageUrls.includes(url) &&
selectedModelImageUrl !== url,
);
if (nextRerun) {
selectedRerunImageUrls.push(nextRerun);
continue;
}
const nextTemplate = templateReferenceImageUrls.find(
(url) =>
!selectedTemplateImageUrls.includes(url) &&
!selectedRerunImageUrls.includes(url) &&
!selectedProductImageUrls.includes(url) &&
selectedModelImageUrl !== url,
);
if (!nextTemplate) break;
selectedTemplateImageUrls.push(nextTemplate);
}
const entries: Array<{
url: string;
role: DetailReplacementGenerationInputRole;
}> = [
...selectedProductImageUrls.map((url) => ({ url, role: "product" as const })),
...(selectedModelImageUrl
? [{ url: selectedModelImageUrl, role: "model" as const }]
: []),
...selectedTemplateImageUrls.map((url) => ({
url,
role: "supplement" as const,
})),
...selectedRerunImageUrls.map((url) => ({
url,
role: "supplement" as const,
})),
{ url: referenceImageUrl, role: "layout" as const },
].filter((entry) => /^https?:\/\//i.test(entry.url));
return {
entries,
imageUrls: entries.map((entry) => entry.url),
inputImageRoleByUrl: Object.fromEntries(
entries.map((entry) => [entry.url, entry.role]),
) as Record<string, DetailReplacementGenerationInputRole>,
productImageUrls: selectedProductImageUrls,
modelImageUrl: selectedModelImageUrl,
supplementImageUrls: [
...selectedTemplateImageUrls,
...selectedRerunImageUrls,
],
templateImageUrls: selectedTemplateImageUrls,
rerunImageUrls: selectedRerunImageUrls,
};
}

View File

@ -0,0 +1,16 @@
export const DETAIL_REPLACEMENT_VISION_MODEL_CONFIG_KEY =
"detail_replacement_vision_model_config";
export const DEFAULT_DETAIL_REPLACEMENT_VISION_MODEL_ID = "qwen3.7-plus";
export function normalizeDetailReplacementVisionModelConfig(value: unknown) {
const record =
value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
return {
modelId:
String(record.modelId || "").trim() ||
DEFAULT_DETAIL_REPLACEMENT_VISION_MODEL_ID,
};
}

View File

@ -0,0 +1,56 @@
import "server-only";
import { createOpenAI } from "@ai-sdk/openai";
import { isAlibabaDashScopeBaseUrl } from "@/lib/ai/detail-agent-model-config";
import {
DETAIL_REPLACEMENT_VISION_MODEL_CONFIG_KEY,
normalizeDetailReplacementVisionModelConfig,
} from "@/lib/ai/detail-replacement-vision-model-config";
import { getAiProvidersByType, getSystemSetting } from "@/lib/db/queries";
const ALIBABA_COMPATIBLE_BASE_URL =
"https://dashscope.aliyuncs.com/compatible-mode/v1";
function createNonThinkingVisionFetch(baseFetch: typeof fetch = fetch): typeof fetch {
return async (input, init) => {
if (typeof init?.body !== "string") return baseFetch(input, init);
try {
const body = JSON.parse(init.body) as Record<string, unknown>;
return baseFetch(input, {
...init,
body: JSON.stringify({ ...body, enable_thinking: false }),
});
} catch {
return baseFetch(input, init);
}
};
}
export async function resolveDetailReplacementVisionRuntime() {
const [setting, providers] = await Promise.all([
getSystemSetting({ key: DETAIL_REPLACEMENT_VISION_MODEL_CONFIG_KEY }).catch(
() => null,
),
getAiProvidersByType({ type: "alibaba", enabledOnly: true }).catch(() => []),
]);
const provider = providers.find(
(item) =>
Boolean(item.apiKey?.trim()) &&
(!item.baseUrl?.trim() || isAlibabaDashScopeBaseUrl(item.baseUrl)),
);
if (!provider?.apiKey) return null;
const config = normalizeDetailReplacementVisionModelConfig(setting?.value);
const runtime = createOpenAI({
apiKey: provider.apiKey.trim(),
baseURL: provider.baseUrl?.trim() || ALIBABA_COMPATIBLE_BASE_URL,
fetch: createNonThinkingVisionFetch(),
});
return {
kind: "configured_vision" as const,
model: runtime.chat(config.modelId),
modelName: config.modelId,
};
}

View File

@ -0,0 +1,164 @@
import sharp from "sharp";
export type AlibabaImageInputRole =
| "product"
| "model"
| "supplement"
| "layout";
export type PreparedAlibabaImageInput = {
buffer: Buffer;
extension: "png" | "jpg";
transformed: boolean;
transparentProductTrimmed: boolean;
};
function clamp(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, value));
}
export function selectAlibabaInputImageUrls(
imageUrls: string[],
maxImages = 3,
) {
const normalized = Array.from(
new Set(
imageUrls
.map((url) => String(url || "").trim())
.filter((url) => /^https?:\/\//i.test(url)),
),
);
const limit = Math.max(1, Math.floor(maxImages));
if (normalized.length <= limit) return normalized;
return [...normalized.slice(0, limit - 1), normalized.at(-1)!];
}
async function trimTransparentProduct(
buffer: Buffer,
maxSide: number,
jpegQuality: number,
) {
const trimmed = await sharp(buffer, {
animated: false,
failOn: "none",
})
.rotate()
.trim({
background: { r: 0, g: 0, b: 0, alpha: 0 },
threshold: 8,
})
.toBuffer({ resolveWithObject: true });
const padding = clamp(
Math.round(Math.max(trimmed.info.width, trimmed.info.height) * 0.04),
24,
160,
);
return sharp(trimmed.data, { failOn: "none" })
.flatten({ background: "#ffffff" })
.extend({
top: padding,
bottom: padding,
left: padding,
right: padding,
background: "#ffffff",
})
.resize(maxSide, maxSide, {
fit: "inside",
withoutEnlargement: true,
})
.jpeg({ quality: jpegQuality, mozjpeg: true })
.toBuffer();
}
export async function prepareAlibabaImageInputBuffer(params: {
buffer: Buffer;
contentType?: string | null;
imageUrl?: string | null;
inputRole?: AlibabaImageInputRole | null;
maxInputBytes: number;
maxSide: number;
modelId: string;
}): Promise<PreparedAlibabaImageInput> {
const metadata = await sharp(params.buffer, { failOn: "none" }).metadata();
const format = String(metadata.format || "").toLowerCase();
const contentType = String(params.contentType || "").toLowerCase();
const cleanUrl = String(params.imageUrl || "")
.split("?")[0]
?.toLowerCase();
const isGif =
format === "gif" || contentType.includes("gif") || cleanUrl.endsWith(".gif");
const isTooLarge = params.buffer.length > params.maxInputBytes;
const isTooWide = Number(metadata.width || 0) > params.maxSide;
const isTooTall = Number(metadata.height || 0) > params.maxSide;
const isQwenImage = /qwen-image/i.test(params.modelId);
const shouldTrimTransparentProduct = Boolean(
isQwenImage && params.inputRole === "product" && metadata.hasAlpha,
);
if (shouldTrimTransparentProduct) {
const prepared = await trimTransparentProduct(params.buffer, params.maxSide, 90);
if (prepared.length > params.maxInputBytes) {
const compressed = await sharp(prepared, { failOn: "none" })
.resize(3072, 3072, { fit: "inside", withoutEnlargement: true })
.jpeg({ quality: 82, mozjpeg: true })
.toBuffer();
return {
buffer: compressed,
extension: "jpg",
transformed: true,
transparentProductTrimmed: true,
};
}
return {
buffer: prepared,
extension: "jpg",
transformed: true,
transparentProductTrimmed: true,
};
}
if (!isGif && !isTooLarge && !isTooWide && !isTooTall) {
return {
buffer: params.buffer,
extension: format === "jpeg" || format === "jpg" ? "jpg" : "png",
transformed: false,
transparentProductTrimmed: false,
};
}
const resizeTo = async (side: number, quality?: number) => {
const pipeline = sharp(params.buffer, {
animated: false,
failOn: "none",
})
.rotate()
.resize(side, side, { fit: "inside", withoutEnlargement: true });
return quality
? pipeline.jpeg({ quality, mozjpeg: true }).toBuffer()
: pipeline.png({ compressionLevel: 8 }).toBuffer();
};
let prepared = await resizeTo(params.maxSide);
let extension: "png" | "jpg" = "png";
if (prepared.length > params.maxInputBytes) {
prepared = await resizeTo(params.maxSide, 88);
extension = "jpg";
}
if (prepared.length > params.maxInputBytes) {
prepared = await resizeTo(3072, 82);
extension = "jpg";
}
if (prepared.length > params.maxInputBytes) {
prepared = await resizeTo(2048, 76);
extension = "jpg";
}
return {
buffer: prepared,
extension,
transformed: true,
transparentProductTrimmed: false,
};
}

View File

@ -19,6 +19,11 @@ import {
resolveAlibabaImageAsyncEndpoint,
resolveAlibabaImageTaskEndpoint,
} from "./alibaba-image-api";
import {
prepareAlibabaImageInputBuffer,
selectAlibabaInputImageUrls,
type AlibabaImageInputRole,
} from "./alibaba-image-input";
interface AlibabaImageConfig {
apiKey: string;
@ -33,6 +38,7 @@ export interface AlibabaImageParams {
promptExtend?: boolean; // 是否开启提示词智能优化
watermark?: boolean; // 是否添加水印
seed?: number; // 随机种子
inputImageRoleByUrl?: Record<string, AlibabaImageInputRole>;
}
// 阿里生图 API 响应
@ -268,7 +274,8 @@ async function uploadPreparedAlibabaInputImage(
async function prepareAlibabaInputImageUrl(
imageUrl: string,
modelId: string
modelId: string,
inputRole?: AlibabaImageInputRole,
): Promise<string> {
const normalizedUrl = String(imageUrl || "").trim();
if (!/^https?:\/\//i.test(normalizedUrl)) return normalizedUrl;
@ -286,25 +293,25 @@ async function prepareAlibabaInputImageUrl(
);
}
const format = String(metadata.format || "").toLowerCase();
const type = String(contentType || "").toLowerCase();
const cleanUrl = normalizedUrl.split("?")[0]?.toLowerCase() || "";
const isGif = format === "gif" || type.includes("gif") || cleanUrl.endsWith(".gif");
const isTooLarge = buffer.length > maxInputBytes;
const isTooWide = Number(metadata.width || 0) > ALIBABA_MAX_INPUT_SIDE;
const isTooTall = Number(metadata.height || 0) > ALIBABA_MAX_INPUT_SIDE;
if (!isGif && !isTooLarge && !isTooWide && !isTooTall) {
const prepared = await prepareAlibabaImageInputBuffer({
buffer,
contentType,
imageUrl: normalizedUrl,
inputRole,
maxInputBytes,
maxSide: ALIBABA_MAX_INPUT_SIDE,
modelId,
});
if (!prepared.transformed) {
return normalizedUrl;
}
console.warn("[Alibaba Image] Normalizing input image before provider request:", {
bytes: buffer.length,
contentType,
format,
format: metadata.format,
height: metadata.height,
isGif,
isTooLarge,
inputRole: inputRole || null,
maxInputBytes,
modelId,
sourceHost: (() => {
@ -314,64 +321,16 @@ async function prepareAlibabaInputImageUrl(
return "";
}
})(),
transparentProductTrimmed: prepared.transparentProductTrimmed,
width: metadata.width,
});
const basePipeline = sharp(buffer, {
animated: false,
failOn: "none",
})
.rotate()
.resize(ALIBABA_MAX_INPUT_SIDE, ALIBABA_MAX_INPUT_SIDE, {
fit: "inside",
withoutEnlargement: true,
});
let prepared = await basePipeline.png({ compressionLevel: 8 }).toBuffer();
let extension: "png" | "jpg" = "png";
if (prepared.length > maxInputBytes) {
prepared = await sharp(buffer, { animated: false, failOn: "none" })
.rotate()
.resize(ALIBABA_MAX_INPUT_SIDE, ALIBABA_MAX_INPUT_SIDE, {
fit: "inside",
withoutEnlargement: true,
})
.jpeg({ quality: 88, mozjpeg: true })
.toBuffer();
extension = "jpg";
}
if (prepared.length > maxInputBytes) {
prepared = await sharp(buffer, { animated: false, failOn: "none" })
.rotate()
.resize(3072, 3072, {
fit: "inside",
withoutEnlargement: true,
})
.jpeg({ quality: 82, mozjpeg: true })
.toBuffer();
extension = "jpg";
}
if (prepared.length > maxInputBytes) {
prepared = await sharp(buffer, { animated: false, failOn: "none" })
.rotate()
.resize(2048, 2048, {
fit: "inside",
withoutEnlargement: true,
})
.jpeg({ quality: 76, mozjpeg: true })
.toBuffer();
extension = "jpg";
}
if (prepared.length > maxInputBytes) {
if (prepared.buffer.length > maxInputBytes) {
const limitMb = Math.floor(maxInputBytes / 1024 / 1024);
throw new Error(`输入图片压缩后仍超过 ${limitMb}MB,请先压缩图片后重试`);
}
return uploadPreparedAlibabaInputImage(prepared, extension);
return uploadPreparedAlibabaInputImage(prepared.buffer, prepared.extension);
}
/**
@ -809,9 +768,16 @@ export function createAlibabaImage({ apiKey, baseURL }: AlibabaImageConfig) {
throw new Error("No prompt found in message");
}
const selectedImageUrls = selectAlibabaInputImageUrls(imageUrls, 3);
const preparedImageUrls: string[] = [];
for (const imageUrl of Array.from(new Set(imageUrls)).slice(0, 3)) {
preparedImageUrls.push(await prepareAlibabaInputImageUrl(imageUrl, activeModelId));
for (const imageUrl of selectedImageUrls) {
preparedImageUrls.push(
await prepareAlibabaInputImageUrl(
imageUrl,
activeModelId,
options.imageParams?.inputImageRoleByUrl?.[imageUrl],
),
);
}
// qwen-image-3.0 同时支持文生图与图生图;旧编辑模型仍必须上传参考图。

View File

@ -11,6 +11,7 @@ import sharp from "sharp";
import { getDefaultStorageConfig, getSystemSetting } from "@/lib/db/queries";
import {
bindVip98kImages,
hasVerifiedVip98kOpenAiImageArrayContract,
normalizeVip98kInvocationHeaders,
resolveVip98kImageTaskSize,
selectVip98kTaskInvocation,
@ -1478,6 +1479,9 @@ function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvocation |
aspectRatio,
fallbackSize: body.size,
preserveResolutionBucket,
useGptImage25AspectSizing:
preserveResolutionBucket &&
hasVerifiedVip98kOpenAiImageArrayContract(params.modelId),
});
if (resolvedTaskSize) {
body.size = resolvedTaskSize;
@ -1485,7 +1489,7 @@ function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvocation |
hasNativeAspectRatio =
Boolean(aspectRatio) &&
resolvedTaskSize !== "auto" &&
!preserveResolutionBucket;
(!preserveResolutionBucket || resolvedTaskSize.includes("x"));
}
}
}

View File

@ -134,7 +134,9 @@ export function buildReplacementProductIdentityAnalysisPrompt(productImageCount:
"只分析商品本身,不分析透明背景、白边、阴影或后期抠图瑕疵。",
"identitySummary:概括商品类别、整体轮廓、材质、颜色和稳定识别特征。",
"structureSummary:列出不可改变的几何拓扑,包括构件数量、左右对应关系、腿/支架/横梁/侧框/层板/把手/转轴的形状、粗细、连接点、开口方向、排列和比例。不要把不同视角下的同一构件误判为不同设计。",
"先区分透视投影产生的斜线和真实斜向构件:只有实际存在连接两个节点的斜杆时才能判定为 K/X/A/V 型;矩形侧框向画面深处延伸形成的斜边仍是正交方管,不是斜撑。structureSummary 必须明确真实斜撑是否存在。",
"structuralRisks:列出最容易被生成模型误画的结构,以及必须禁止的新增、删除、弯曲、分叉、熔合、悬空、错接、镜像和左右不一致。",
"structuralRisks 只写生成时的误画风险,不做承重、抗扭、强度或长期使用等工程安全推测。",
"viewSummaries:逐图说明拍摄角度、可见面和最清楚展示的结构,imageIndex 必须使用 1 开始的原始图片编号。",
"对于桌、椅、柜架、置物架等家具,必须明确腿型和框架拓扑,例如口字型、K/X/A/V 型或直腿;不得含糊写成‘保持结构一致’。",
"只返回严格 JSON,不要解释。",

View File

@ -1,4 +1,8 @@
import { normalizeVip98kInvocationHeaders } from "@/lib/ai/vip98k-task-invocation";
import {
hasVerifiedVip98kGptImage25ResolutionTiers,
hasVerifiedVip98kOpenAiImageArrayContract,
normalizeVip98kInvocationHeaders,
} from "@/lib/ai/vip98k-task-invocation";
export const VIP98K_BASE_URL = "https://api.vip98k.com";
@ -415,7 +419,10 @@ function hasMediaReferenceInput(
// catalog metadata can advertise broad multimodal capability even when the
// published request body only accepts a text prompt.
if (type === "image" && getInvocationCandidates(model).length > 0) {
return hasMediaRequestField;
return (
hasMediaRequestField ||
hasVerifiedVip98kOpenAiImageArrayContract(modelId)
);
}
return (
@ -427,6 +434,11 @@ function hasMediaReferenceInput(
}
function hasMultiImageReferenceInput(model: Vip98kCatalogModel) {
if (
hasVerifiedVip98kOpenAiImageArrayContract(getModelId(model).toLowerCase())
) {
return true;
}
const multiImageKeys = new Set([
"file-urls",
"images",
@ -535,6 +547,9 @@ function getSupportedResolutions(model: Vip98kCatalogModel) {
}
function getSupportedImageSizes(model: Vip98kCatalogModel) {
if (hasVerifiedVip98kGptImage25ResolutionTiers(getModelId(model))) {
return ["1K", "2K", "4K"];
}
const schemaSizes = getInvocationCandidates(model).flatMap((invocation) => {
const properties =
toRecord(toRecord(invocation.input_schema)?.properties) || {};
@ -701,17 +716,19 @@ function createImagePreset(
const invocationKeys = new Set(getInvocationRequestKeys(model));
const supportsReferenceImages = hasMediaReferenceInput(model, "image");
const supportsMultiImageReference = hasMultiImageReferenceInput(model);
const hasVerifiedResolutionTiers =
hasVerifiedVip98kGptImage25ResolutionTiers(modelId);
const supportsNativeSize = [
"size",
"image-size",
"resolution",
"width",
"height",
].some((key) => invocationKeys.has(key));
].some((key) => invocationKeys.has(key)) || hasVerifiedResolutionTiers;
const supportsNativeAspectRatio = [
"ratio",
"aspect-ratio",
].some((key) => invocationKeys.has(key));
].some((key) => invocationKeys.has(key)) || hasVerifiedResolutionTiers;
const supportsQuality = invocationKeys.has("quality");
const supportedModes = [
...(capabilities.includes("text-to-image") || capabilities.length === 0

View File

@ -18,6 +18,12 @@ const VIP98K_IMAGE_SIZE_LONGEST_EDGE: Record<string, number> = {
"4K": 4096,
};
const GPT_IMAGE_25_TARGET_PIXELS: Record<string, number> = {
"1K": 1024 * 1024,
"2K": 2048 * 2048,
"4K": 2880 * 2880,
};
function normalizeVip98kPixelSize(value: unknown) {
const match = String(value || "")
.trim()
@ -45,6 +51,7 @@ export function resolveVip98kImageTaskSize(params: {
aspectRatio?: unknown;
fallbackSize?: unknown;
preserveResolutionBucket?: boolean;
useGptImage25AspectSizing?: boolean;
}) {
const explicitSize = normalizeVip98kPixelSize(params.imageSize);
if (explicitSize) return explicitSize;
@ -54,6 +61,35 @@ export function resolveVip98kImageTaskSize(params: {
.match(/^(\d+(?:\.\d+)?)\s*:\s*(\d+(?:\.\d+)?)$/);
const sizeBucket = String(params.imageSize || "").trim().toUpperCase();
const longestEdge = VIP98K_IMAGE_SIZE_LONGEST_EDGE[sizeBucket];
if (
params.useGptImage25AspectSizing &&
ratioMatch?.[1] &&
ratioMatch[2] &&
sizeBucket in GPT_IMAGE_25_TARGET_PIXELS
) {
const ratioWidth = Number(ratioMatch[1]);
const ratioHeight = Number(ratioMatch[2]);
if (ratioWidth > 0 && ratioHeight > 0 && ratioWidth !== ratioHeight) {
const ratio = ratioWidth / ratioHeight;
const targetPixels = GPT_IMAGE_25_TARGET_PIXELS[sizeBucket] || 0;
let width = Math.sqrt(targetPixels * ratio);
let height = Math.sqrt(targetPixels / ratio);
const maxEdgeScale = Math.min(1, 3840 / Math.max(width, height));
width *= maxEdgeScale;
height *= maxEdgeScale;
const clampDimension = (value: number) =>
Math.max(16, Math.min(3840, value));
const roundToSixteen = (value: number) =>
clampDimension(Math.round(value / 16) * 16);
const floorToSixteen = (value: number) =>
clampDimension(Math.floor(value / 16) * 16);
const shouldFloorWidth = sizeBucket !== "4K" && width < height;
const shouldFloorHeight = sizeBucket !== "4K" && height < width;
return `${shouldFloorWidth ? floorToSixteen(width) : roundToSixteen(width)}x${
shouldFloorHeight ? floorToSixteen(height) : roundToSixteen(height)
}`;
}
}
if (longestEdge && params.preserveResolutionBucket) return sizeBucket;
if (ratioMatch?.[1] && ratioMatch[2] && longestEdge) {
const ratioWidth = Number(ratioMatch[1]);
@ -99,6 +135,27 @@ const KNOWN_IMAGE_TOOL_INPUTS: Record<string, KnownImageInput> = {
"openai/gpt-image-2/edit": "images",
};
const VERIFIED_OPENAI_IMAGE_ARRAY_MODEL_IDS = new Set([
"gpt-image-2.5",
"gpt-image-2.5-flare",
]);
const VERIFIED_GPT_IMAGE_25_RESOLUTION_MODEL_IDS = new Set([
"gpt-image-2.5-flare",
]);
export function hasVerifiedVip98kOpenAiImageArrayContract(modelId: string) {
return VERIFIED_OPENAI_IMAGE_ARRAY_MODEL_IDS.has(
modelId.trim().toLowerCase(),
);
}
export function hasVerifiedVip98kGptImage25ResolutionTiers(modelId: string) {
return VERIFIED_GPT_IMAGE_25_RESOLUTION_MODEL_IDS.has(
modelId.trim().toLowerCase(),
);
}
const KNOWN_IMAGE_PROCESSING_TOOL_REQUESTS: Record<
string,
Record<string, unknown>
@ -296,6 +353,68 @@ function createKnownProcessingToolInvocation(
};
}
function createVerifiedOpenAiImageInvocation(modelId: string) {
return {
kind: "tool",
method: "POST",
path: "/v1/images/generations",
contentType: "application/json",
requestExample: {
image: [],
model: modelId,
n: 1,
prompt: "",
quality: "auto",
size: "1024x1024",
stream: false,
},
inputSchema: {
properties: {
aspect_ratio: { type: "string" },
},
type: "object",
},
resultMethod: "GET",
resultPath: "",
} satisfies Vip98kTaskInvocation;
}
function addVerifiedOpenAiImageFields(
invocation: Vip98kTaskInvocation,
modelId: string,
): Vip98kTaskInvocation {
const inputSchema = invocation.inputSchema || {};
const properties =
inputSchema.properties &&
typeof inputSchema.properties === "object" &&
!Array.isArray(inputSchema.properties)
? (inputSchema.properties as Record<string, unknown>)
: {};
return {
...invocation,
requestExample: {
...invocation.requestExample,
image: [],
},
inputSchema: {
...inputSchema,
properties: {
...properties,
image: { items: { type: "string" }, type: "array" },
aspect_ratio: { type: "string" },
...(hasVerifiedVip98kGptImage25ResolutionTiers(modelId)
? {
size: {
enum: ["1024x1024", "2048x2048", "2880x2880"],
type: "string",
},
}
: {}),
},
},
};
}
/**
* A frontend alias may resolve to a different VIP98K tool at runtime. In that
* case the selected model's invocation cannot be reused because its endpoint
@ -306,6 +425,14 @@ export function resolveVip98kTaskInvocation(
configuredInvocation: Vip98kTaskInvocation | null
) {
const normalizedModelId = modelId.trim().toLowerCase();
if (hasVerifiedVip98kOpenAiImageArrayContract(normalizedModelId)) {
const invocation =
configuredInvocation &&
invocationTargetsModel(configuredInvocation, normalizedModelId)
? configuredInvocation
: createVerifiedOpenAiImageInvocation(modelId);
return addVerifiedOpenAiImageFields(invocation, normalizedModelId);
}
const processingRequest = KNOWN_IMAGE_PROCESSING_TOOL_REQUESTS[normalizedModelId];
if (processingRequest) {
if (invocationTargetsModel(configuredInvocation, normalizedModelId)) {

View File

@ -7,6 +7,7 @@ type InlineVisionOptions = {
retries?: number;
loggerPrefix?: string;
jpegQuality?: number;
trimTransparentPadding?: boolean;
};
type InlineVisionResult = {
@ -51,15 +52,51 @@ async function fetchImageWithRetry(url: string, timeoutMs: number, retries: numb
throw lastError instanceof Error ? lastError : new Error("图片拉取失败");
}
async function downscaleForVision(
export async function downscaleForVision(
inputBuffer: Buffer,
maxSide: number,
jpegQuality: number,
trimTransparentPadding = false,
) {
const image = sharp(inputBuffer, { failOn: "none" });
const metadata = await image.metadata();
const hasAlpha = Boolean(metadata.hasAlpha);
if (hasAlpha && trimTransparentPadding) {
const trimmed = await sharp(inputBuffer, { failOn: "none" })
.rotate()
.trim({
background: { r: 0, g: 0, b: 0, alpha: 0 },
threshold: 8,
})
.toBuffer({ resolveWithObject: true });
const padding = Math.max(
12,
Math.min(
48,
Math.round(Math.max(trimmed.info.width, trimmed.info.height) * 0.04),
),
);
const output = await sharp(trimmed.data, { failOn: "none" })
.flatten({ background: "#ffffff" })
.extend({
top: padding,
bottom: padding,
left: padding,
right: padding,
background: "#ffffff",
})
.resize({
width: maxSide,
height: maxSide,
fit: "inside",
withoutEnlargement: true,
})
.jpeg({ quality: jpegQuality, mozjpeg: true })
.toBuffer();
return { mime: "image/jpeg", buffer: output };
}
const resized = image
.rotate()
.resize({
@ -100,7 +137,12 @@ export async function inlineVisionImages(
for (const url of candidates) {
try {
const buffer = await fetchImageWithRetry(url, timeoutMs, retries);
const processed = await downscaleForVision(buffer, maxSide, jpegQuality);
const processed = await downscaleForVision(
buffer,
maxSide,
jpegQuality,
options.trimTransparentPadding === true,
);
const dataUrl = `data:${processed.mime};base64,${processed.buffer.toString("base64")}`;
inlineDataUrls.push(dataUrl);
} catch (error) {

View File

@ -0,0 +1,100 @@
WITH preferred_provider AS (
SELECT p."id"
FROM "AiProvider" p
LEFT JOIN "AiModel" m
ON m."providerId" = p."id"
AND m."modelId" = 'gpt-image-2.5-flare'
WHERE p."type" = 'vip98k'
ORDER BY
CASE WHEN m."id" IS NOT NULL THEN 0 ELSE 1 END,
CASE WHEN p."isEnabled" THEN 0 ELSE 1 END,
p."createdAt"
LIMIT 1
)
INSERT INTO "AiModel" (
"providerId",
"modelId",
"displayName",
"description",
"supportsVision",
"supportsImageGen",
"isEnabled",
"isHidden",
"isDefault",
"sortOrder",
"capabilities",
"defaultParams",
"imageGenConfig",
"updatedAt"
)
SELECT
preferred_provider."id",
'gpt-image-2.5-flare',
'gpt-image-2.5-flare',
'VIP98K 多图编辑模型,支持 1K、2K 和 4K',
true,
true,
true,
false,
false,
103,
'{"provider":"vip98k","kind":"image","mediaType":"image","modelType":"image","outputType":"image","supportedModes":["text-to-image","image-to-image","reference-images"],"supportedImageSizes":["1K","2K","4K"],"supportedAspectRatios":["auto","1:1","4:3","3:4","16:9","9:16","3:2","2:3","4:5","5:4"],"supportsMultiImageReference":true,"maxReferenceImages":8}'::jsonb,
'{"provider":"vip98k","kind":"image","endpoint":"gpt-image-2.5-flare","apiFormat":"vip98k-external-task","taskApi":"external-task","supportedModes":["text-to-image","image-to-image","reference-images"],"imageSize":"2K","quality":"auto","aspectRatio":"auto","imageCount":1}'::jsonb,
'{"maxImages":1,"maxReferenceImages":8,"supportsSize":true,"supportsQuality":true,"supports1K":true,"supports2K":true,"supports4K":true,"supportedImageSizes":["1K","2K","4K"],"supportedAspectRatios":["auto","1:1","4:3","3:4","16:9","9:16","3:2","2:3","4:5","5:4"],"supportsPromptExtend":false,"supportsNegativePrompt":false,"supportsWatermark":false,"supportsSeed":false,"creditCostPerImage":0,"creditCostMultiplierBySize":{"1K":1,"2K":1,"4K":1},"enableLoadingPlaceholder":true,"loadingColor":"yellow"}'::jsonb,
now()
FROM preferred_provider
WHERE NOT EXISTS (
SELECT 1
FROM "AiModel" existing
WHERE existing."providerId" = preferred_provider."id"
AND existing."modelId" = 'gpt-image-2.5-flare'
);
--> statement-breakpoint
WITH preferred_provider AS (
SELECT p."id"
FROM "AiProvider" p
LEFT JOIN "AiModel" m
ON m."providerId" = p."id"
AND m."modelId" = 'gpt-image-2.5-flare'
WHERE p."type" = 'vip98k'
ORDER BY
CASE WHEN m."id" IS NOT NULL THEN 0 ELSE 1 END,
CASE WHEN p."isEnabled" THEN 0 ELSE 1 END,
p."createdAt"
LIMIT 1
)
UPDATE "AiModel" target
SET
"supportsVision" = true,
"supportsImageGen" = true,
"isEnabled" = true,
"isHidden" = false,
"capabilities" = COALESCE(target."capabilities", '{}'::jsonb) ||
'{"provider":"vip98k","kind":"image","mediaType":"image","modelType":"image","outputType":"image","supportedModes":["text-to-image","image-to-image","reference-images"],"supportedImageSizes":["1K","2K","4K"],"supportedAspectRatios":["auto","1:1","4:3","3:4","16:9","9:16","3:2","2:3","4:5","5:4"],"supportsMultiImageReference":true,"maxReferenceImages":8}'::jsonb,
"defaultParams" = COALESCE(target."defaultParams", '{}'::jsonb) ||
'{"provider":"vip98k","kind":"image","endpoint":"gpt-image-2.5-flare","apiFormat":"vip98k-external-task","taskApi":"external-task","supportedModes":["text-to-image","image-to-image","reference-images"],"imageSize":"2K","quality":"auto","aspectRatio":"auto","imageCount":1}'::jsonb,
"imageGenConfig" = COALESCE(target."imageGenConfig", '{}'::jsonb) ||
'{"maxImages":1,"maxReferenceImages":8,"supportsSize":true,"supportsQuality":true,"supports1K":true,"supports2K":true,"supports4K":true,"supportedImageSizes":["1K","2K","4K"],"supportedAspectRatios":["auto","1:1","4:3","3:4","16:9","9:16","3:2","2:3","4:5","5:4"],"supportsPromptExtend":false,"supportsNegativePrompt":false,"supportsWatermark":false,"supportsSeed":false,"creditCostPerImage":0,"creditCostMultiplierBySize":{"1K":1,"2K":1,"4K":1},"enableLoadingPlaceholder":true,"loadingColor":"yellow"}'::jsonb,
"updatedAt" = now()
FROM preferred_provider
WHERE target."providerId" = preferred_provider."id"
AND target."modelId" = 'gpt-image-2.5-flare';
--> statement-breakpoint
INSERT INTO "SystemSettings" ("key", "value", "updatedAt")
VALUES (
'detail_replacement_image_model_config',
'{"modelId":"gpt-image-2.5-flare"}'::jsonb,
now()
)
ON CONFLICT ("key") DO UPDATE
SET
"value" = EXCLUDED."value",
"updatedAt" = EXCLUDED."updatedAt";
--> statement-breakpoint
INSERT INTO "SystemSettings" ("key", "value", "updatedAt")
VALUES (
'detail_replacement_vision_model_config',
'{"modelId":"qwen3.7-plus"}'::jsonb,
now()
)
ON CONFLICT ("key") DO NOTHING;

View File

@ -603,6 +603,13 @@
"when": 1788220800000,
"tag": "0099_set_detail_replacement_qwen_image3",
"breakpoints": true
},
{
"idx": 84,
"version": "7",
"when": 1788998400000,
"tag": "0100_add_gpt_image25_detail_replacement",
"breakpoints": true
}
]
}

View File

@ -18,6 +18,7 @@ export default defineConfig({
/routes\/model-classification\.test\.ts/,
/routes\/replacement-layout-lock\.test\.ts/,
/routes\/replacement-product-identity\.test\.ts/,
/routes\/alibaba-image-input\.test\.ts/,
/routes\/model3d-style-pro-regressions\.test\.ts/,
/routes\/clawd-request-lifecycle\.test\.ts/,
/routes\/canvas-skill-tag-order\.test\.ts/,

View File

@ -0,0 +1,142 @@
import { expect, test } from "@playwright/test";
import sharp from "sharp";
import {
prepareAlibabaImageInputBuffer,
selectAlibabaInputImageUrls,
} from "@/lib/ai/providers/alibaba-image-input";
import {
resolveDetailAgentApiKey,
} from "@/lib/ai/detail-agent-model-config";
import {
DEFAULT_DETAIL_REPLACEMENT_VISION_MODEL_ID,
normalizeDetailReplacementVisionModelConfig,
} from "@/lib/ai/detail-replacement-vision-model-config";
import { selectDetailReplacementGenerationInputs } from "@/lib/ai/detail-replacement-inputs";
test("preserves the final layout image when Alibaba accepts only three inputs", () => {
expect(
selectAlibabaInputImageUrls([
"https://assets.test/product.png",
"https://assets.test/model.png",
"https://assets.test/supplement.png",
"https://assets.test/layout.png",
]),
).toEqual([
"https://assets.test/product.png",
"https://assets.test/model.png",
"https://assets.test/layout.png",
]);
});
test("packs product, required model identity, and final layout into the three-image contract", () => {
const selected = selectDetailReplacementGenerationInputs({
productImageUrls: [
"https://assets.test/product-main.png",
"https://assets.test/product-side.png",
],
modelImageUrl: "https://assets.test/person.png",
includeModelImage: true,
rerunReferenceImageUrls: ["https://assets.test/old-result.png"],
referenceImageUrl: "https://assets.test/layout.png",
});
expect(selected.imageUrls).toEqual([
"https://assets.test/product-main.png",
"https://assets.test/person.png",
"https://assets.test/layout.png",
]);
expect(selected.inputImageRoleByUrl).toEqual({
"https://assets.test/product-main.png": "product",
"https://assets.test/person.png": "model",
"https://assets.test/layout.png": "layout",
});
});
test("uses a rerun result as the second input without dropping the final layout", () => {
const selected = selectDetailReplacementGenerationInputs({
productImageUrls: ["https://assets.test/product.png"],
includeModelImage: false,
rerunReferenceImageUrls: ["https://assets.test/old-result.png"],
referenceImageUrl: "https://assets.test/layout.png",
});
expect(selected.imageUrls).toEqual([
"https://assets.test/product.png",
"https://assets.test/old-result.png",
"https://assets.test/layout.png",
]);
});
test("crops transparent product padding and flattens it onto white for Qwen", async () => {
const product = await sharp({
create: {
width: 1000,
height: 1000,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 },
},
})
.composite([
{
input: await sharp({
create: {
width: 600,
height: 240,
channels: 4,
background: { r: 25, g: 25, b: 25, alpha: 1 },
},
})
.png()
.toBuffer(),
left: 200,
top: 380,
},
])
.png()
.toBuffer();
const prepared = await prepareAlibabaImageInputBuffer({
buffer: product,
contentType: "image/png",
inputRole: "product",
maxInputBytes: 9.5 * 1024 * 1024,
maxSide: 4096,
modelId: "qwen-image-3.0-pro",
});
const metadata = await sharp(prepared.buffer).metadata();
const corner = await sharp(prepared.buffer)
.extract({ left: 0, top: 0, width: 1, height: 1 })
.raw()
.toBuffer();
expect(prepared.transformed).toBe(true);
expect(prepared.transparentProductTrimmed).toBe(true);
expect(prepared.extension).toBe("jpg");
expect(metadata.width).toBeLessThan(800);
expect(metadata.height).toBeLessThan(440);
expect([...corner]).toEqual([255, 255, 255]);
});
test("uses the enabled Alibaba provider key for a DashScope detail planner", () => {
expect(
resolveDetailAgentApiKey({
configuredApiKey: "stale-setting-key",
configuredBaseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1",
enabledAlibabaProviders: [
{
apiKey: "current-provider-key",
baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1",
},
],
}),
).toBe("current-provider-key");
});
test("uses the dedicated high-accuracy vision model for detail replacement", () => {
expect(DEFAULT_DETAIL_REPLACEMENT_VISION_MODEL_ID).toBe("qwen3.7-plus");
expect(normalizeDetailReplacementVisionModelConfig(null)).toEqual({
modelId: "qwen3.7-plus",
});
expect(
normalizeDetailReplacementVisionModelConfig({ modelId: "custom-vision" }),
).toEqual({ modelId: "custom-vision" });
});

View File

@ -8,6 +8,7 @@ import {
normalizeReplacementProductImageIndexes,
parseReplacementProductIdentityProfile,
} from "@/lib/ai/replacement-product-identity";
import { selectDetailReplacementGenerationInputs } from "@/lib/ai/detail-replacement-inputs";
test("parses a reusable multi-view product structure profile", () => {
const profile = parseReplacementProductIdentityProfile(
@ -68,12 +69,47 @@ test("makes product topology authoritative over the old detail product", () => {
expect(prompt).toContain("旧商品结构必须完整移除");
});
test("keeps non-Pro detail replacement bound to Qwen Image 3 in database config", () => {
expect(DEFAULT_DETAIL_REPLACEMENT_IMAGE_MODEL_ID).toBe("qwen-image-3.0");
test("binds non-Pro detail replacement to GPT Image 2.5 Flare in database config", () => {
expect(DEFAULT_DETAIL_REPLACEMENT_IMAGE_MODEL_ID).toBe("gpt-image-2.5-flare");
expect(normalizeDetailReplacementImageModelConfig(null)).toEqual({
modelId: "qwen-image-3.0",
modelId: "gpt-image-2.5-flare",
});
expect(
normalizeDetailReplacementImageModelConfig({ modelId: "custom-image-model" }),
).toEqual({ modelId: "custom-image-model" });
});
test("keeps the product, required model identity, and final layout within three inputs", () => {
const selected = selectDetailReplacementGenerationInputs({
productImageUrls: ["https://assets.test/product-1.png", "https://assets.test/product-2.png"],
modelImageUrl: "https://assets.test/model.png",
includeModelImage: true,
rerunReferenceImageUrls: ["https://assets.test/rerun.png"],
referenceImageUrl: "https://assets.test/layout.png",
});
expect(selected.imageUrls).toEqual([
"https://assets.test/product-1.png",
"https://assets.test/model.png",
"https://assets.test/layout.png",
]);
expect(selected.inputImageRoleByUrl).toEqual({
"https://assets.test/product-1.png": "product",
"https://assets.test/model.png": "model",
"https://assets.test/layout.png": "layout",
});
});
test("prioritizes an explicit rerun reference while always keeping the final layout", () => {
const selected = selectDetailReplacementGenerationInputs({
productImageUrls: ["https://assets.test/product-1.png", "https://assets.test/product-2.png"],
rerunReferenceImageUrls: ["https://assets.test/rerun.png"],
referenceImageUrl: "https://assets.test/layout.png",
});
expect(selected.imageUrls).toEqual([
"https://assets.test/product-1.png",
"https://assets.test/rerun.png",
"https://assets.test/layout.png",
]);
});

View File

@ -681,7 +681,18 @@ test.describe("VIP98K reference-image routing", () => {
isHidden: false,
supportsQuality: true,
quality: "auto",
supportedAspectRatios: ["auto"],
supportedAspectRatios: [
"auto",
"1:1",
"4:3",
"3:4",
"16:9",
"9:16",
"3:2",
"2:3",
"4:5",
"5:4",
],
},
{
modelId: "gpt-image-2.5-sunburst",
@ -693,6 +704,83 @@ test.describe("VIP98K reference-image routing", () => {
]);
});
test("repairs stale Flare catalog metadata with verified multi-image and size controls", () => {
const [preset] = buildVip98kModelPresets([
{
id: "gpt-image-2.5-flare",
type: "image",
capabilities: ["image_to_image", "openai.images.generations"],
invocation: {
kind: "tool",
method: "POST",
path: "/v1/images/generations",
content_type: "application/json",
headers: { "X-Relay-Route-Option": "flare-route" },
request_example: {
model: "gpt-image-2.5-flare",
n: 1,
prompt: "Edit",
quality: "auto",
stream: false,
},
},
},
]);
expect(preset).toMatchObject({
modelId: "gpt-image-2.5-flare",
supportsVision: true,
isHidden: false,
capabilities: {
supportsMultiImageReference: true,
maxReferenceImages: 8,
supportedImageSizes: ["1K", "2K", "4K"],
supportedAspectRatios: [
"auto",
"1:1",
"4:3",
"3:4",
"16:9",
"9:16",
"3:2",
"2:3",
"4:5",
"5:4",
],
},
defaultParams: { imageSize: "2K" },
imageGenConfig: {
supportsSize: true,
supports1K: true,
supports2K: true,
supports4K: true,
supportedImageSizes: ["1K", "2K", "4K"],
},
});
const metadata = preset?.capabilities?.vip98k as {
invocation?: Vip98kTaskInvocation;
};
const invocation = selectVip98kTaskInvocation({
modelId: "gpt-image-2.5-flare",
configuredInvocation: metadata.invocation || null,
imageCount: 2,
});
expect(invocation?.headers).toEqual({
"X-Relay-Route-Option": "flare-route",
});
expect(invocation?.requestExample.image).toEqual([]);
expect(invocation?.inputSchema).toMatchObject({
properties: {
aspect_ratio: { type: "string" },
image: { type: "array" },
size: {
enum: ["1024x1024", "2048x2048", "2880x2880"],
},
},
});
});
test("does not inherit multi-image support from broad authorized metadata", () => {
const presets = buildVip98kModelPresets([
{
@ -775,9 +863,36 @@ test.describe("VIP98K reference-image routing", () => {
preserveResolutionBucket: true,
}),
).toBe("4K");
expect(
resolveVip98kImageTaskSize({
imageSize: "2K",
aspectRatio: "1:1",
fallbackSize: "1024x1024",
preserveResolutionBucket: true,
useGptImage25AspectSizing: true,
}),
).toBe("2K");
expect(
resolveVip98kImageTaskSize({
imageSize: "2K",
aspectRatio: "2:3",
fallbackSize: "1024x1024",
preserveResolutionBucket: true,
useGptImage25AspectSizing: true,
}),
).toBe("1664x2512");
expect(
resolveVip98kImageTaskSize({
imageSize: "4K",
aspectRatio: "2:3",
fallbackSize: "1024x1024",
preserveResolutionBucket: true,
useGptImage25AspectSizing: true,
}),
).toBe("2352x3520");
});
test("advertises only catalog resolutions and explicit aspect-ratio controls", () => {
test("advertises verified Flare resolutions and aspect-ratio controls", () => {
const [preset] = buildVip98kModelPresets([
{
id: "gpt-image-2.5-flare",
@ -810,7 +925,18 @@ test.describe("VIP98K reference-image routing", () => {
supports2K: true,
supports4K: true,
supportedImageSizes: ["1K", "2K", "4K"],
supportedAspectRatios: ["auto"],
supportedAspectRatios: [
"auto",
"1:1",
"4:3",
"3:4",
"16:9",
"9:16",
"3:2",
"2:3",
"4:5",
"5:4",
],
});
});
});