diff --git a/app/(chat)/api/chat/_lib/detail-replacement.ts b/app/(chat)/api/chat/_lib/detail-replacement.ts index a8aab1970..defadd30d 100644 --- a/app/(chat)/api/chat/_lib/detail-replacement.ts +++ b/app/(chat)/api/chat/_lib/detail-replacement.ts @@ -20,6 +20,7 @@ import { type ReplacementProductIdentityProfile, } from "@/lib/ai/replacement-product-identity"; import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime"; +import { resolveDetailReplacementVisionRuntime } from "@/lib/ai/detail-replacement-vision-runtime"; import { inlineVisionImages } from "@/lib/ai/vision-inline"; import { getEnabledAiModelsWithProvider, @@ -594,6 +595,7 @@ async function resolveReplacementProductIdentityProfile(params: { timeoutMs: 12_000, retries: 2, loggerPrefix: "[Chat API] Replacement product identity", + trimTransparentPadding: true, }); const effectiveImages = inlineDataUrls.length === productImageUrls.length @@ -2449,6 +2451,16 @@ export async function analyzeReplacementReferenceImage(params: { export async function resolveReplacementScenePlanningRuntime(params: { models?: Awaited>; }): Promise { + try { + const dedicatedVisionRuntime = await resolveDetailReplacementVisionRuntime(); + if (dedicatedVisionRuntime) return dedicatedVisionRuntime; + } catch (error) { + chatApiLogger.warn( + "[Chat API] Failed to resolve dedicated detail replacement vision model:", + error, + ); + } + try { const siteSettings = await getSiteSettings().catch(() => null); const localDetailAgentResolved = await resolveDetailAgentLocalMoonshotModel({ diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 69cb10969..7677c4c98 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -364,6 +364,7 @@ import { resolveReplacementReferenceGeometry, resolveReplacementScenePlanningRuntime, } from "./_lib/detail-replacement"; +import { selectDetailReplacementGenerationInputs } from "@/lib/ai/detail-replacement-inputs"; import { normalizeAgentReportForDisplay, sanitizeAgentOutputSettingsSection, @@ -5938,16 +5939,14 @@ export async function POST(request: Request) { replacementExecutionPlan.scenePlan, ) || selectedProductContextImageUrls.length === 0); - const sourceImageUrls = Array.from( - new Set([ - ...selectedProductContextImageUrls, - ...(shouldIncludeModelImage && modelInputImageUrl - ? [modelInputImageUrl] - : []), - ...rerunReferenceImageUrls, - reference.imageUrl, - ]), - ); + const generationInputs = selectDetailReplacementGenerationInputs({ + productImageUrls: selectedProductContextImageUrls, + modelImageUrl: modelInputImageUrl, + includeModelImage: shouldIncludeModelImage, + rerunReferenceImageUrls, + referenceImageUrl: reference.imageUrl, + }); + const sourceImageUrls = generationInputs.imageUrls; chatApiLogger.info("[Chat API] Direct detail replacement image generation started:", { referenceIndex: referenceIndex + 1, totalReferences: replacementReferences.length, @@ -5955,10 +5954,10 @@ export async function POST(request: Request) { pageKind: replacementExecutionPlan.scenePlan.pageKind, generationRoute: replacementExecutionPlan.scenePlan.generationRoute, packPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy, - selectedProductImages: selectedProductContextImageUrls.length, + selectedProductImages: generationInputs.productImageUrls.length, sourceProductImages: productInputImageUrls.length, - includeModelImage: shouldIncludeModelImage, - rerunReferenceImages: rerunReferenceImageUrls.length, + includeModelImage: Boolean(generationInputs.modelImageUrl), + rerunReferenceImages: generationInputs.rerunImageUrls.length, sourceImageCount: sourceImageUrls.length, whiteLayout: replacementExecutionPlan.scenePlan.whiteBackgroundLikely, copyCoupled: replacementExecutionPlan.scenePlan.copyImageCoupled, @@ -5973,10 +5972,10 @@ export async function POST(request: Request) { referenceLabel: reference.label, referenceIndex, totalReferences: replacementReferences.length, - hasModelImage: shouldIncludeModelImage, - productImageCount: selectedProductContextImageUrls.length, + hasModelImage: Boolean(generationInputs.modelImageUrl), + productImageCount: generationInputs.productImageUrls.length, originalProductImageCount: productInputImageUrls.length, - rerunReferenceCount: rerunReferenceImageUrls.length, + rerunReferenceCount: generationInputs.rerunImageUrls.length, templateReferenceCount: 0, scenePlan: replacementExecutionPlan.scenePlan, referenceGeometry, @@ -6008,6 +6007,10 @@ export async function POST(request: Request) { imageSize: resolvedImageSize, imageCount: 1, referenceImages: sourceImageUrls, + promptExtend: false, + providerImageParams: { + inputImageRoleByUrl: generationInputs.inputImageRoleByUrl, + }, debugLabel: "agent-detail-replacement", debugMeta: { referenceLabel: reference.label, @@ -6830,19 +6833,31 @@ export async function POST(request: Request) { ) || (isDetailReplacementPreset && selectedProductContextImageUrls.length === 0)); - const sourceImageUrls = Array.from( - new Set( - [ - ...selectedProductContextImageUrls, - ...(shouldIncludeModelImage && modelInputImageUrl - ? [modelInputImageUrl] - : []), - ...replacementTemplateReferenceImageUrls, - ...rerunReferenceImageUrls, - reference.imageUrl, - ].filter(Boolean), - ), - ); + const detailGenerationInputs = isDetailReplacementPreset + ? selectDetailReplacementGenerationInputs({ + productImageUrls: selectedProductContextImageUrls, + modelImageUrl: modelInputImageUrl, + includeModelImage: shouldIncludeModelImage, + templateReferenceImageUrls: replacementTemplateReferenceImageUrls, + rerunReferenceImageUrls, + referenceImageUrl: reference.imageUrl, + }) + : null; + const sourceImageUrls = detailGenerationInputs + ? detailGenerationInputs.imageUrls + : Array.from( + new Set( + [ + ...selectedProductContextImageUrls, + ...(shouldIncludeModelImage && modelInputImageUrl + ? [modelInputImageUrl] + : []), + ...replacementTemplateReferenceImageUrls, + ...rerunReferenceImageUrls, + reference.imageUrl, + ].filter(Boolean), + ), + ); const replacementPrompt = buildAgentReplacementPrompt({ basePrompt: baseReplacementPrompt || @@ -6859,11 +6874,19 @@ export async function POST(request: Request) { referenceLabel: reference.label, referenceIndex, totalReferences: replacementReferences.length, - hasModelImage: shouldIncludeModelImage, - productImageCount: selectedProductContextImageUrls.length, + hasModelImage: detailGenerationInputs + ? Boolean(detailGenerationInputs.modelImageUrl) + : shouldIncludeModelImage, + productImageCount: detailGenerationInputs + ? detailGenerationInputs.productImageUrls.length + : selectedProductContextImageUrls.length, originalProductImageCount: productInputImageUrls.length, - rerunReferenceCount: rerunReferenceImageUrls.length, - templateReferenceCount: replacementTemplateReferenceImageUrls.length, + rerunReferenceCount: detailGenerationInputs + ? detailGenerationInputs.rerunImageUrls.length + : rerunReferenceImageUrls.length, + templateReferenceCount: detailGenerationInputs + ? detailGenerationInputs.templateImageUrls.length + : replacementTemplateReferenceImageUrls.length, pairedProductIndex, scenePlan: replacementExecutionPlan.scenePlan, referenceGeometry, @@ -6884,11 +6907,20 @@ export async function POST(request: Request) { sourceImageCount: sourceImageUrls.length, generationAspectRatio, imageOrder: { - productImages: selectedProductContextImageUrls.length, + productImages: + detailGenerationInputs !== null + ? detailGenerationInputs.productImageUrls.length + : selectedProductContextImageUrls.length, originalProductImages: productInputImageUrls.length, - hasModelImage: shouldIncludeModelImage, - templateReferenceImages: replacementTemplateReferenceImageUrls.length, - rerunReferenceImages: rerunReferenceImageUrls.length, + hasModelImage: detailGenerationInputs + ? Boolean(detailGenerationInputs.modelImageUrl) + : shouldIncludeModelImage, + templateReferenceImages: detailGenerationInputs + ? detailGenerationInputs.templateImageUrls.length + : replacementTemplateReferenceImageUrls.length, + rerunReferenceImages: detailGenerationInputs + ? detailGenerationInputs.rerunImageUrls.length + : rerunReferenceImageUrls.length, finalReferenceImageIndex: sourceImageUrls.length, }, }); @@ -6906,6 +6938,17 @@ export async function POST(request: Request) { imageSize: resolvedImageSize, imageCount: 1, referenceImages: sourceImageUrls, + promptExtend: isDetailReplacementPreset + ? false + : typeof mergedImageParams.promptExtend === "boolean" + ? mergedImageParams.promptExtend + : undefined, + providerImageParams: detailGenerationInputs + ? { + inputImageRoleByUrl: + detailGenerationInputs.inputImageRoleByUrl, + } + : undefined, debugLabel: `agent-${replacementSourceType}`, debugMeta: { referenceLabel: reference.label, diff --git a/lib/ai/detail-agent-model-config.ts b/lib/ai/detail-agent-model-config.ts index 711eb17c1..f6eec45f4 100644 --- a/lib/ai/detail-agent-model-config.ts +++ b/lib/ai/detail-agent-model-config.ts @@ -25,6 +25,36 @@ export type DetailAgentLocalMoonshotConfig = { displayName: string; }; +const ALIBABA_DASHSCOPE_HOST = "dashscope.aliyuncs.com"; + +export function isAlibabaDashScopeBaseUrl(value: unknown) { + try { + return new URL(String(value || "").trim()).hostname === ALIBABA_DASHSCOPE_HOST; + } catch { + return false; + } +} + +export function resolveDetailAgentApiKey(params: { + configuredApiKey: string; + configuredBaseUrl: string; + enabledAlibabaProviders: Array<{ + apiKey?: string | null; + baseUrl?: string | null; + }>; +}) { + if (isAlibabaDashScopeBaseUrl(params.configuredBaseUrl)) { + const provider = params.enabledAlibabaProviders.find( + (item) => + isAlibabaDashScopeBaseUrl(item.baseUrl) && + Boolean(String(item.apiKey || "").trim()), + ); + if (provider?.apiKey) return provider.apiKey.trim(); + } + + return String(params.configuredApiKey || "").trim(); +} + export function isDetailAgentLocalMoonshotOption(value: unknown) { return String(value || "").trim() === DETAIL_AGENT_LOCAL_MOONSHOT_OPTION_ID; } diff --git a/lib/ai/detail-agent-model-runtime.ts b/lib/ai/detail-agent-model-runtime.ts index 354ebfccd..d17444e96 100644 --- a/lib/ai/detail-agent-model-runtime.ts +++ b/lib/ai/detail-agent-model-runtime.ts @@ -2,13 +2,14 @@ import "server-only"; import { createOpenAI } from "@ai-sdk/openai"; import type { LanguageModel } from "ai"; -import { getSystemSetting } from "@/lib/db/queries"; +import { getAiProvidersByType, getSystemSetting } from "@/lib/db/queries"; import { DETAIL_AGENT_LOCAL_MOONSHOT_BASE_URL, DETAIL_AGENT_LOCAL_MOONSHOT_OPTION_ID, DETAIL_AGENT_LOCAL_MOONSHOT_SETTING_KEY, isDetailAgentLocalMoonshotOption, normalizeDetailAgentLocalMoonshotConfig, + resolveDetailAgentApiKey, } from "@/lib/ai/detail-agent-model-config"; import { createMoonshotKimiFetch } from "@/lib/ai/detail-agent-kimi-profile"; @@ -23,13 +24,22 @@ export async function resolveDetailAgentLocalMoonshotModel(params: { key: DETAIL_AGENT_LOCAL_MOONSHOT_SETTING_KEY, }).catch(() => null); const config = normalizeDetailAgentLocalMoonshotConfig(setting?.value); + const enabledAlibabaProviders = await getAiProvidersByType({ + type: "alibaba", + enabledOnly: true, + }).catch(() => []); + const apiKey = resolveDetailAgentApiKey({ + configuredApiKey: config.apiKey, + configuredBaseUrl: config.baseUrl, + enabledAlibabaProviders, + }); - if (!config.enabled || !config.apiKey.trim()) { + if (!config.enabled || !apiKey) { return null; } const provider = createOpenAI({ - apiKey: config.apiKey, + apiKey, baseURL: config.baseUrl || DETAIL_AGENT_LOCAL_MOONSHOT_BASE_URL, fetch: createMoonshotKimiFetch(), }); diff --git a/lib/ai/detail-replacement-image-model-config.ts b/lib/ai/detail-replacement-image-model-config.ts index d7c2d0d35..f2b53aa48 100644 --- a/lib/ai/detail-replacement-image-model-config.ts +++ b/lib/ai/detail-replacement-image-model-config.ts @@ -1,7 +1,7 @@ export const DETAIL_REPLACEMENT_IMAGE_MODEL_CONFIG_KEY = "detail_replacement_image_model_config"; -export const DEFAULT_DETAIL_REPLACEMENT_IMAGE_MODEL_ID = "qwen-image-3.0"; +export const DEFAULT_DETAIL_REPLACEMENT_IMAGE_MODEL_ID = "gpt-image-2.5-flare"; export type DetailReplacementImageModelConfig = { modelId: string; diff --git a/lib/ai/detail-replacement-inputs.ts b/lib/ai/detail-replacement-inputs.ts new file mode 100644 index 000000000..f9fb69dda --- /dev/null +++ b/lib/ai/detail-replacement-inputs.ts @@ -0,0 +1,146 @@ +export type DetailReplacementGenerationInputRole = + | "product" + | "model" + | "supplement" + | "layout"; + +export function selectDetailReplacementGenerationInputs(params: { + productImageUrls: string[]; + modelImageUrl?: string | null; + includeModelImage?: boolean; + templateReferenceImageUrls?: string[]; + rerunReferenceImageUrls?: string[]; + referenceImageUrl: string; + maxImages?: number; +}) { + const normalizeUrls = (values: Array) => + Array.from( + new Set( + values + .map((value) => String(value || "").trim()) + .filter((value) => /^https?:\/\//i.test(value)), + ), + ); + const referenceImageUrl = String(params.referenceImageUrl || "").trim(); + const productImageUrls = normalizeUrls(params.productImageUrls).filter( + (url) => url !== referenceImageUrl, + ); + const modelImageUrl = params.includeModelImage + ? normalizeUrls([params.modelImageUrl]).find((url) => url !== referenceImageUrl) || null + : null; + const rerunReferenceImageUrls = normalizeUrls( + params.rerunReferenceImageUrls || [], + ).filter((url) => url !== referenceImageUrl); + const templateReferenceImageUrls = normalizeUrls( + params.templateReferenceImageUrls || [], + ).filter((url) => url !== referenceImageUrl); + const maxImages = Math.max(2, Math.floor(params.maxImages || 3)); + const contentCapacity = maxImages - 1; + const selectedProductImageUrls: string[] = []; + const selectedTemplateImageUrls: string[] = []; + const selectedRerunImageUrls: string[] = []; + let selectedModelImageUrl: string | null = null; + + if (productImageUrls[0]) selectedProductImageUrls.push(productImageUrls[0]); + if (modelImageUrl && selectedProductImageUrls.length < contentCapacity) { + selectedModelImageUrl = modelImageUrl; + } + + const usedCount = () => + selectedProductImageUrls.length + + selectedTemplateImageUrls.length + + selectedRerunImageUrls.length + + (selectedModelImageUrl ? 1 : 0); + const addSupplement = (url: string | undefined, kind: "template" | "rerun") => { + if (!url || usedCount() >= contentCapacity) return; + const selectedSupplementImageUrls = [ + ...selectedTemplateImageUrls, + ...selectedRerunImageUrls, + ]; + if ( + selectedProductImageUrls.includes(url) || + selectedSupplementImageUrls.includes(url) || + selectedModelImageUrl === url + ) { + return; + } + if (kind === "template") selectedTemplateImageUrls.push(url); + else selectedRerunImageUrls.push(url); + }; + + if (!selectedModelImageUrl) { + addSupplement(rerunReferenceImageUrls[0], "rerun"); + if (usedCount() < contentCapacity && productImageUrls[1]) { + selectedProductImageUrls.push(productImageUrls[1]); + } + addSupplement(templateReferenceImageUrls[0], "template"); + } else if (selectedProductImageUrls.length === 0) { + addSupplement(rerunReferenceImageUrls[0], "rerun"); + addSupplement(templateReferenceImageUrls[0], "template"); + } + + while (usedCount() < contentCapacity) { + const nextProduct = productImageUrls.find( + (url) => !selectedProductImageUrls.includes(url), + ); + if (nextProduct) { + selectedProductImageUrls.push(nextProduct); + continue; + } + const nextRerun = rerunReferenceImageUrls.find( + (url) => + !selectedRerunImageUrls.includes(url) && + !selectedTemplateImageUrls.includes(url) && + !selectedProductImageUrls.includes(url) && + selectedModelImageUrl !== url, + ); + if (nextRerun) { + selectedRerunImageUrls.push(nextRerun); + continue; + } + const nextTemplate = templateReferenceImageUrls.find( + (url) => + !selectedTemplateImageUrls.includes(url) && + !selectedRerunImageUrls.includes(url) && + !selectedProductImageUrls.includes(url) && + selectedModelImageUrl !== url, + ); + if (!nextTemplate) break; + selectedTemplateImageUrls.push(nextTemplate); + } + + const entries: Array<{ + url: string; + role: DetailReplacementGenerationInputRole; + }> = [ + ...selectedProductImageUrls.map((url) => ({ url, role: "product" as const })), + ...(selectedModelImageUrl + ? [{ url: selectedModelImageUrl, role: "model" as const }] + : []), + ...selectedTemplateImageUrls.map((url) => ({ + url, + role: "supplement" as const, + })), + ...selectedRerunImageUrls.map((url) => ({ + url, + role: "supplement" as const, + })), + { url: referenceImageUrl, role: "layout" as const }, + ].filter((entry) => /^https?:\/\//i.test(entry.url)); + + return { + entries, + imageUrls: entries.map((entry) => entry.url), + inputImageRoleByUrl: Object.fromEntries( + entries.map((entry) => [entry.url, entry.role]), + ) as Record, + productImageUrls: selectedProductImageUrls, + modelImageUrl: selectedModelImageUrl, + supplementImageUrls: [ + ...selectedTemplateImageUrls, + ...selectedRerunImageUrls, + ], + templateImageUrls: selectedTemplateImageUrls, + rerunImageUrls: selectedRerunImageUrls, + }; +} diff --git a/lib/ai/detail-replacement-vision-model-config.ts b/lib/ai/detail-replacement-vision-model-config.ts new file mode 100644 index 000000000..7cda71974 --- /dev/null +++ b/lib/ai/detail-replacement-vision-model-config.ts @@ -0,0 +1,16 @@ +export const DETAIL_REPLACEMENT_VISION_MODEL_CONFIG_KEY = + "detail_replacement_vision_model_config"; + +export const DEFAULT_DETAIL_REPLACEMENT_VISION_MODEL_ID = "qwen3.7-plus"; + +export function normalizeDetailReplacementVisionModelConfig(value: unknown) { + const record = + value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; + return { + modelId: + String(record.modelId || "").trim() || + DEFAULT_DETAIL_REPLACEMENT_VISION_MODEL_ID, + }; +} diff --git a/lib/ai/detail-replacement-vision-runtime.ts b/lib/ai/detail-replacement-vision-runtime.ts new file mode 100644 index 000000000..d38608b55 --- /dev/null +++ b/lib/ai/detail-replacement-vision-runtime.ts @@ -0,0 +1,56 @@ +import "server-only"; + +import { createOpenAI } from "@ai-sdk/openai"; +import { isAlibabaDashScopeBaseUrl } from "@/lib/ai/detail-agent-model-config"; +import { + DETAIL_REPLACEMENT_VISION_MODEL_CONFIG_KEY, + normalizeDetailReplacementVisionModelConfig, +} from "@/lib/ai/detail-replacement-vision-model-config"; +import { getAiProvidersByType, getSystemSetting } from "@/lib/db/queries"; + +const ALIBABA_COMPATIBLE_BASE_URL = + "https://dashscope.aliyuncs.com/compatible-mode/v1"; + +function createNonThinkingVisionFetch(baseFetch: typeof fetch = fetch): typeof fetch { + return async (input, init) => { + if (typeof init?.body !== "string") return baseFetch(input, init); + + try { + const body = JSON.parse(init.body) as Record; + return baseFetch(input, { + ...init, + body: JSON.stringify({ ...body, enable_thinking: false }), + }); + } catch { + return baseFetch(input, init); + } + }; +} + +export async function resolveDetailReplacementVisionRuntime() { + const [setting, providers] = await Promise.all([ + getSystemSetting({ key: DETAIL_REPLACEMENT_VISION_MODEL_CONFIG_KEY }).catch( + () => null, + ), + getAiProvidersByType({ type: "alibaba", enabledOnly: true }).catch(() => []), + ]); + const provider = providers.find( + (item) => + Boolean(item.apiKey?.trim()) && + (!item.baseUrl?.trim() || isAlibabaDashScopeBaseUrl(item.baseUrl)), + ); + if (!provider?.apiKey) return null; + + const config = normalizeDetailReplacementVisionModelConfig(setting?.value); + const runtime = createOpenAI({ + apiKey: provider.apiKey.trim(), + baseURL: provider.baseUrl?.trim() || ALIBABA_COMPATIBLE_BASE_URL, + fetch: createNonThinkingVisionFetch(), + }); + + return { + kind: "configured_vision" as const, + model: runtime.chat(config.modelId), + modelName: config.modelId, + }; +} diff --git a/lib/ai/providers/alibaba-image-input.ts b/lib/ai/providers/alibaba-image-input.ts new file mode 100644 index 000000000..8e7dd86ab --- /dev/null +++ b/lib/ai/providers/alibaba-image-input.ts @@ -0,0 +1,164 @@ +import sharp from "sharp"; + +export type AlibabaImageInputRole = + | "product" + | "model" + | "supplement" + | "layout"; + +export type PreparedAlibabaImageInput = { + buffer: Buffer; + extension: "png" | "jpg"; + transformed: boolean; + transparentProductTrimmed: boolean; +}; + +function clamp(value: number, min: number, max: number) { + return Math.max(min, Math.min(max, value)); +} + +export function selectAlibabaInputImageUrls( + imageUrls: string[], + maxImages = 3, +) { + const normalized = Array.from( + new Set( + imageUrls + .map((url) => String(url || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)), + ), + ); + const limit = Math.max(1, Math.floor(maxImages)); + if (normalized.length <= limit) return normalized; + + return [...normalized.slice(0, limit - 1), normalized.at(-1)!]; +} + +async function trimTransparentProduct( + buffer: Buffer, + maxSide: number, + jpegQuality: number, +) { + const trimmed = await sharp(buffer, { + animated: false, + failOn: "none", + }) + .rotate() + .trim({ + background: { r: 0, g: 0, b: 0, alpha: 0 }, + threshold: 8, + }) + .toBuffer({ resolveWithObject: true }); + const padding = clamp( + Math.round(Math.max(trimmed.info.width, trimmed.info.height) * 0.04), + 24, + 160, + ); + + return sharp(trimmed.data, { failOn: "none" }) + .flatten({ background: "#ffffff" }) + .extend({ + top: padding, + bottom: padding, + left: padding, + right: padding, + background: "#ffffff", + }) + .resize(maxSide, maxSide, { + fit: "inside", + withoutEnlargement: true, + }) + .jpeg({ quality: jpegQuality, mozjpeg: true }) + .toBuffer(); +} + +export async function prepareAlibabaImageInputBuffer(params: { + buffer: Buffer; + contentType?: string | null; + imageUrl?: string | null; + inputRole?: AlibabaImageInputRole | null; + maxInputBytes: number; + maxSide: number; + modelId: string; +}): Promise { + const metadata = await sharp(params.buffer, { failOn: "none" }).metadata(); + const format = String(metadata.format || "").toLowerCase(); + const contentType = String(params.contentType || "").toLowerCase(); + const cleanUrl = String(params.imageUrl || "") + .split("?")[0] + ?.toLowerCase(); + const isGif = + format === "gif" || contentType.includes("gif") || cleanUrl.endsWith(".gif"); + const isTooLarge = params.buffer.length > params.maxInputBytes; + const isTooWide = Number(metadata.width || 0) > params.maxSide; + const isTooTall = Number(metadata.height || 0) > params.maxSide; + const isQwenImage = /qwen-image/i.test(params.modelId); + const shouldTrimTransparentProduct = Boolean( + isQwenImage && params.inputRole === "product" && metadata.hasAlpha, + ); + + if (shouldTrimTransparentProduct) { + const prepared = await trimTransparentProduct(params.buffer, params.maxSide, 90); + if (prepared.length > params.maxInputBytes) { + const compressed = await sharp(prepared, { failOn: "none" }) + .resize(3072, 3072, { fit: "inside", withoutEnlargement: true }) + .jpeg({ quality: 82, mozjpeg: true }) + .toBuffer(); + return { + buffer: compressed, + extension: "jpg", + transformed: true, + transparentProductTrimmed: true, + }; + } + return { + buffer: prepared, + extension: "jpg", + transformed: true, + transparentProductTrimmed: true, + }; + } + + if (!isGif && !isTooLarge && !isTooWide && !isTooTall) { + return { + buffer: params.buffer, + extension: format === "jpeg" || format === "jpg" ? "jpg" : "png", + transformed: false, + transparentProductTrimmed: false, + }; + } + + const resizeTo = async (side: number, quality?: number) => { + const pipeline = sharp(params.buffer, { + animated: false, + failOn: "none", + }) + .rotate() + .resize(side, side, { fit: "inside", withoutEnlargement: true }); + return quality + ? pipeline.jpeg({ quality, mozjpeg: true }).toBuffer() + : pipeline.png({ compressionLevel: 8 }).toBuffer(); + }; + + let prepared = await resizeTo(params.maxSide); + let extension: "png" | "jpg" = "png"; + if (prepared.length > params.maxInputBytes) { + prepared = await resizeTo(params.maxSide, 88); + extension = "jpg"; + } + if (prepared.length > params.maxInputBytes) { + prepared = await resizeTo(3072, 82); + extension = "jpg"; + } + if (prepared.length > params.maxInputBytes) { + prepared = await resizeTo(2048, 76); + extension = "jpg"; + } + + return { + buffer: prepared, + extension, + transformed: true, + transparentProductTrimmed: false, + }; +} diff --git a/lib/ai/providers/alibaba-image.ts b/lib/ai/providers/alibaba-image.ts index 4f50256ca..505e253d8 100644 --- a/lib/ai/providers/alibaba-image.ts +++ b/lib/ai/providers/alibaba-image.ts @@ -19,6 +19,11 @@ import { resolveAlibabaImageAsyncEndpoint, resolveAlibabaImageTaskEndpoint, } from "./alibaba-image-api"; +import { + prepareAlibabaImageInputBuffer, + selectAlibabaInputImageUrls, + type AlibabaImageInputRole, +} from "./alibaba-image-input"; interface AlibabaImageConfig { apiKey: string; @@ -33,6 +38,7 @@ export interface AlibabaImageParams { promptExtend?: boolean; // 是否开启提示词智能优化 watermark?: boolean; // 是否添加水印 seed?: number; // 随机种子 + inputImageRoleByUrl?: Record; } // 阿里生图 API 响应 @@ -268,7 +274,8 @@ async function uploadPreparedAlibabaInputImage( async function prepareAlibabaInputImageUrl( imageUrl: string, - modelId: string + modelId: string, + inputRole?: AlibabaImageInputRole, ): Promise { const normalizedUrl = String(imageUrl || "").trim(); if (!/^https?:\/\//i.test(normalizedUrl)) return normalizedUrl; @@ -286,25 +293,25 @@ async function prepareAlibabaInputImageUrl( ); } - const format = String(metadata.format || "").toLowerCase(); - const type = String(contentType || "").toLowerCase(); - const cleanUrl = normalizedUrl.split("?")[0]?.toLowerCase() || ""; - const isGif = format === "gif" || type.includes("gif") || cleanUrl.endsWith(".gif"); - const isTooLarge = buffer.length > maxInputBytes; - const isTooWide = Number(metadata.width || 0) > ALIBABA_MAX_INPUT_SIDE; - const isTooTall = Number(metadata.height || 0) > ALIBABA_MAX_INPUT_SIDE; - - if (!isGif && !isTooLarge && !isTooWide && !isTooTall) { + const prepared = await prepareAlibabaImageInputBuffer({ + buffer, + contentType, + imageUrl: normalizedUrl, + inputRole, + maxInputBytes, + maxSide: ALIBABA_MAX_INPUT_SIDE, + modelId, + }); + if (!prepared.transformed) { return normalizedUrl; } console.warn("[Alibaba Image] Normalizing input image before provider request:", { bytes: buffer.length, contentType, - format, + format: metadata.format, height: metadata.height, - isGif, - isTooLarge, + inputRole: inputRole || null, maxInputBytes, modelId, sourceHost: (() => { @@ -314,64 +321,16 @@ async function prepareAlibabaInputImageUrl( return ""; } })(), + transparentProductTrimmed: prepared.transparentProductTrimmed, width: metadata.width, }); - const basePipeline = sharp(buffer, { - animated: false, - failOn: "none", - }) - .rotate() - .resize(ALIBABA_MAX_INPUT_SIDE, ALIBABA_MAX_INPUT_SIDE, { - fit: "inside", - withoutEnlargement: true, - }); - - let prepared = await basePipeline.png({ compressionLevel: 8 }).toBuffer(); - let extension: "png" | "jpg" = "png"; - - if (prepared.length > maxInputBytes) { - prepared = await sharp(buffer, { animated: false, failOn: "none" }) - .rotate() - .resize(ALIBABA_MAX_INPUT_SIDE, ALIBABA_MAX_INPUT_SIDE, { - fit: "inside", - withoutEnlargement: true, - }) - .jpeg({ quality: 88, mozjpeg: true }) - .toBuffer(); - extension = "jpg"; - } - - if (prepared.length > maxInputBytes) { - prepared = await sharp(buffer, { animated: false, failOn: "none" }) - .rotate() - .resize(3072, 3072, { - fit: "inside", - withoutEnlargement: true, - }) - .jpeg({ quality: 82, mozjpeg: true }) - .toBuffer(); - extension = "jpg"; - } - - if (prepared.length > maxInputBytes) { - prepared = await sharp(buffer, { animated: false, failOn: "none" }) - .rotate() - .resize(2048, 2048, { - fit: "inside", - withoutEnlargement: true, - }) - .jpeg({ quality: 76, mozjpeg: true }) - .toBuffer(); - extension = "jpg"; - } - - if (prepared.length > maxInputBytes) { + if (prepared.buffer.length > maxInputBytes) { const limitMb = Math.floor(maxInputBytes / 1024 / 1024); throw new Error(`输入图片压缩后仍超过 ${limitMb}MB,请先压缩图片后重试`); } - return uploadPreparedAlibabaInputImage(prepared, extension); + return uploadPreparedAlibabaInputImage(prepared.buffer, prepared.extension); } /** @@ -809,9 +768,16 @@ export function createAlibabaImage({ apiKey, baseURL }: AlibabaImageConfig) { throw new Error("No prompt found in message"); } + const selectedImageUrls = selectAlibabaInputImageUrls(imageUrls, 3); const preparedImageUrls: string[] = []; - for (const imageUrl of Array.from(new Set(imageUrls)).slice(0, 3)) { - preparedImageUrls.push(await prepareAlibabaInputImageUrl(imageUrl, activeModelId)); + for (const imageUrl of selectedImageUrls) { + preparedImageUrls.push( + await prepareAlibabaInputImageUrl( + imageUrl, + activeModelId, + options.imageParams?.inputImageRoleByUrl?.[imageUrl], + ), + ); } // qwen-image-3.0 同时支持文生图与图生图;旧编辑模型仍必须上传参考图。 diff --git a/lib/ai/providers/vip98k.ts b/lib/ai/providers/vip98k.ts index 05af412db..e391fb561 100644 --- a/lib/ai/providers/vip98k.ts +++ b/lib/ai/providers/vip98k.ts @@ -11,6 +11,7 @@ import sharp from "sharp"; import { getDefaultStorageConfig, getSystemSetting } from "@/lib/db/queries"; import { bindVip98kImages, + hasVerifiedVip98kOpenAiImageArrayContract, normalizeVip98kInvocationHeaders, resolveVip98kImageTaskSize, selectVip98kTaskInvocation, @@ -1478,6 +1479,9 @@ function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvocation | aspectRatio, fallbackSize: body.size, preserveResolutionBucket, + useGptImage25AspectSizing: + preserveResolutionBucket && + hasVerifiedVip98kOpenAiImageArrayContract(params.modelId), }); if (resolvedTaskSize) { body.size = resolvedTaskSize; @@ -1485,7 +1489,7 @@ function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvocation | hasNativeAspectRatio = Boolean(aspectRatio) && resolvedTaskSize !== "auto" && - !preserveResolutionBucket; + (!preserveResolutionBucket || resolvedTaskSize.includes("x")); } } } diff --git a/lib/ai/replacement-product-identity.ts b/lib/ai/replacement-product-identity.ts index 178f3d207..c83f7dce2 100644 --- a/lib/ai/replacement-product-identity.ts +++ b/lib/ai/replacement-product-identity.ts @@ -134,7 +134,9 @@ export function buildReplacementProductIdentityAnalysisPrompt(productImageCount: "只分析商品本身,不分析透明背景、白边、阴影或后期抠图瑕疵。", "identitySummary:概括商品类别、整体轮廓、材质、颜色和稳定识别特征。", "structureSummary:列出不可改变的几何拓扑,包括构件数量、左右对应关系、腿/支架/横梁/侧框/层板/把手/转轴的形状、粗细、连接点、开口方向、排列和比例。不要把不同视角下的同一构件误判为不同设计。", + "先区分透视投影产生的斜线和真实斜向构件:只有实际存在连接两个节点的斜杆时才能判定为 K/X/A/V 型;矩形侧框向画面深处延伸形成的斜边仍是正交方管,不是斜撑。structureSummary 必须明确真实斜撑是否存在。", "structuralRisks:列出最容易被生成模型误画的结构,以及必须禁止的新增、删除、弯曲、分叉、熔合、悬空、错接、镜像和左右不一致。", + "structuralRisks 只写生成时的误画风险,不做承重、抗扭、强度或长期使用等工程安全推测。", "viewSummaries:逐图说明拍摄角度、可见面和最清楚展示的结构,imageIndex 必须使用 1 开始的原始图片编号。", "对于桌、椅、柜架、置物架等家具,必须明确腿型和框架拓扑,例如口字型、K/X/A/V 型或直腿;不得含糊写成‘保持结构一致’。", "只返回严格 JSON,不要解释。", diff --git a/lib/ai/vip98k-models.ts b/lib/ai/vip98k-models.ts index 12337489b..9a516fa8b 100644 --- a/lib/ai/vip98k-models.ts +++ b/lib/ai/vip98k-models.ts @@ -1,4 +1,8 @@ -import { normalizeVip98kInvocationHeaders } from "@/lib/ai/vip98k-task-invocation"; +import { + hasVerifiedVip98kGptImage25ResolutionTiers, + hasVerifiedVip98kOpenAiImageArrayContract, + normalizeVip98kInvocationHeaders, +} from "@/lib/ai/vip98k-task-invocation"; export const VIP98K_BASE_URL = "https://api.vip98k.com"; @@ -415,7 +419,10 @@ function hasMediaReferenceInput( // catalog metadata can advertise broad multimodal capability even when the // published request body only accepts a text prompt. if (type === "image" && getInvocationCandidates(model).length > 0) { - return hasMediaRequestField; + return ( + hasMediaRequestField || + hasVerifiedVip98kOpenAiImageArrayContract(modelId) + ); } return ( @@ -427,6 +434,11 @@ function hasMediaReferenceInput( } function hasMultiImageReferenceInput(model: Vip98kCatalogModel) { + if ( + hasVerifiedVip98kOpenAiImageArrayContract(getModelId(model).toLowerCase()) + ) { + return true; + } const multiImageKeys = new Set([ "file-urls", "images", @@ -535,6 +547,9 @@ function getSupportedResolutions(model: Vip98kCatalogModel) { } function getSupportedImageSizes(model: Vip98kCatalogModel) { + if (hasVerifiedVip98kGptImage25ResolutionTiers(getModelId(model))) { + return ["1K", "2K", "4K"]; + } const schemaSizes = getInvocationCandidates(model).flatMap((invocation) => { const properties = toRecord(toRecord(invocation.input_schema)?.properties) || {}; @@ -701,17 +716,19 @@ function createImagePreset( const invocationKeys = new Set(getInvocationRequestKeys(model)); const supportsReferenceImages = hasMediaReferenceInput(model, "image"); const supportsMultiImageReference = hasMultiImageReferenceInput(model); + const hasVerifiedResolutionTiers = + hasVerifiedVip98kGptImage25ResolutionTiers(modelId); const supportsNativeSize = [ "size", "image-size", "resolution", "width", "height", - ].some((key) => invocationKeys.has(key)); + ].some((key) => invocationKeys.has(key)) || hasVerifiedResolutionTiers; const supportsNativeAspectRatio = [ "ratio", "aspect-ratio", - ].some((key) => invocationKeys.has(key)); + ].some((key) => invocationKeys.has(key)) || hasVerifiedResolutionTiers; const supportsQuality = invocationKeys.has("quality"); const supportedModes = [ ...(capabilities.includes("text-to-image") || capabilities.length === 0 diff --git a/lib/ai/vip98k-task-invocation.ts b/lib/ai/vip98k-task-invocation.ts index 2a868cb3a..70836d753 100644 --- a/lib/ai/vip98k-task-invocation.ts +++ b/lib/ai/vip98k-task-invocation.ts @@ -18,6 +18,12 @@ const VIP98K_IMAGE_SIZE_LONGEST_EDGE: Record = { "4K": 4096, }; +const GPT_IMAGE_25_TARGET_PIXELS: Record = { + "1K": 1024 * 1024, + "2K": 2048 * 2048, + "4K": 2880 * 2880, +}; + function normalizeVip98kPixelSize(value: unknown) { const match = String(value || "") .trim() @@ -45,6 +51,7 @@ export function resolveVip98kImageTaskSize(params: { aspectRatio?: unknown; fallbackSize?: unknown; preserveResolutionBucket?: boolean; + useGptImage25AspectSizing?: boolean; }) { const explicitSize = normalizeVip98kPixelSize(params.imageSize); if (explicitSize) return explicitSize; @@ -54,6 +61,35 @@ export function resolveVip98kImageTaskSize(params: { .match(/^(\d+(?:\.\d+)?)\s*:\s*(\d+(?:\.\d+)?)$/); const sizeBucket = String(params.imageSize || "").trim().toUpperCase(); const longestEdge = VIP98K_IMAGE_SIZE_LONGEST_EDGE[sizeBucket]; + if ( + params.useGptImage25AspectSizing && + ratioMatch?.[1] && + ratioMatch[2] && + sizeBucket in GPT_IMAGE_25_TARGET_PIXELS + ) { + const ratioWidth = Number(ratioMatch[1]); + const ratioHeight = Number(ratioMatch[2]); + if (ratioWidth > 0 && ratioHeight > 0 && ratioWidth !== ratioHeight) { + const ratio = ratioWidth / ratioHeight; + const targetPixels = GPT_IMAGE_25_TARGET_PIXELS[sizeBucket] || 0; + let width = Math.sqrt(targetPixels * ratio); + let height = Math.sqrt(targetPixels / ratio); + const maxEdgeScale = Math.min(1, 3840 / Math.max(width, height)); + width *= maxEdgeScale; + height *= maxEdgeScale; + const clampDimension = (value: number) => + Math.max(16, Math.min(3840, value)); + const roundToSixteen = (value: number) => + clampDimension(Math.round(value / 16) * 16); + const floorToSixteen = (value: number) => + clampDimension(Math.floor(value / 16) * 16); + const shouldFloorWidth = sizeBucket !== "4K" && width < height; + const shouldFloorHeight = sizeBucket !== "4K" && height < width; + return `${shouldFloorWidth ? floorToSixteen(width) : roundToSixteen(width)}x${ + shouldFloorHeight ? floorToSixteen(height) : roundToSixteen(height) + }`; + } + } if (longestEdge && params.preserveResolutionBucket) return sizeBucket; if (ratioMatch?.[1] && ratioMatch[2] && longestEdge) { const ratioWidth = Number(ratioMatch[1]); @@ -99,6 +135,27 @@ const KNOWN_IMAGE_TOOL_INPUTS: Record = { "openai/gpt-image-2/edit": "images", }; +const VERIFIED_OPENAI_IMAGE_ARRAY_MODEL_IDS = new Set([ + "gpt-image-2.5", + "gpt-image-2.5-flare", +]); + +const VERIFIED_GPT_IMAGE_25_RESOLUTION_MODEL_IDS = new Set([ + "gpt-image-2.5-flare", +]); + +export function hasVerifiedVip98kOpenAiImageArrayContract(modelId: string) { + return VERIFIED_OPENAI_IMAGE_ARRAY_MODEL_IDS.has( + modelId.trim().toLowerCase(), + ); +} + +export function hasVerifiedVip98kGptImage25ResolutionTiers(modelId: string) { + return VERIFIED_GPT_IMAGE_25_RESOLUTION_MODEL_IDS.has( + modelId.trim().toLowerCase(), + ); +} + const KNOWN_IMAGE_PROCESSING_TOOL_REQUESTS: Record< string, Record @@ -296,6 +353,68 @@ function createKnownProcessingToolInvocation( }; } +function createVerifiedOpenAiImageInvocation(modelId: string) { + return { + kind: "tool", + method: "POST", + path: "/v1/images/generations", + contentType: "application/json", + requestExample: { + image: [], + model: modelId, + n: 1, + prompt: "", + quality: "auto", + size: "1024x1024", + stream: false, + }, + inputSchema: { + properties: { + aspect_ratio: { type: "string" }, + }, + type: "object", + }, + resultMethod: "GET", + resultPath: "", + } satisfies Vip98kTaskInvocation; +} + +function addVerifiedOpenAiImageFields( + invocation: Vip98kTaskInvocation, + modelId: string, +): Vip98kTaskInvocation { + const inputSchema = invocation.inputSchema || {}; + const properties = + inputSchema.properties && + typeof inputSchema.properties === "object" && + !Array.isArray(inputSchema.properties) + ? (inputSchema.properties as Record) + : {}; + return { + ...invocation, + requestExample: { + ...invocation.requestExample, + image: [], + }, + inputSchema: { + ...inputSchema, + properties: { + ...properties, + image: { items: { type: "string" }, type: "array" }, + aspect_ratio: { type: "string" }, + ...(hasVerifiedVip98kGptImage25ResolutionTiers(modelId) + ? { + size: { + enum: ["1024x1024", "2048x2048", "2880x2880"], + type: "string", + }, + } + : {}), + }, + }, + }; +} + /** * A frontend alias may resolve to a different VIP98K tool at runtime. In that * case the selected model's invocation cannot be reused because its endpoint @@ -306,6 +425,14 @@ export function resolveVip98kTaskInvocation( configuredInvocation: Vip98kTaskInvocation | null ) { const normalizedModelId = modelId.trim().toLowerCase(); + if (hasVerifiedVip98kOpenAiImageArrayContract(normalizedModelId)) { + const invocation = + configuredInvocation && + invocationTargetsModel(configuredInvocation, normalizedModelId) + ? configuredInvocation + : createVerifiedOpenAiImageInvocation(modelId); + return addVerifiedOpenAiImageFields(invocation, normalizedModelId); + } const processingRequest = KNOWN_IMAGE_PROCESSING_TOOL_REQUESTS[normalizedModelId]; if (processingRequest) { if (invocationTargetsModel(configuredInvocation, normalizedModelId)) { diff --git a/lib/ai/vision-inline.ts b/lib/ai/vision-inline.ts index e8407fce9..2014778b3 100644 --- a/lib/ai/vision-inline.ts +++ b/lib/ai/vision-inline.ts @@ -7,6 +7,7 @@ type InlineVisionOptions = { retries?: number; loggerPrefix?: string; jpegQuality?: number; + trimTransparentPadding?: boolean; }; type InlineVisionResult = { @@ -51,15 +52,51 @@ async function fetchImageWithRetry(url: string, timeoutMs: number, retries: numb throw lastError instanceof Error ? lastError : new Error("图片拉取失败"); } -async function downscaleForVision( +export async function downscaleForVision( inputBuffer: Buffer, maxSide: number, jpegQuality: number, + trimTransparentPadding = false, ) { const image = sharp(inputBuffer, { failOn: "none" }); const metadata = await image.metadata(); const hasAlpha = Boolean(metadata.hasAlpha); + if (hasAlpha && trimTransparentPadding) { + const trimmed = await sharp(inputBuffer, { failOn: "none" }) + .rotate() + .trim({ + background: { r: 0, g: 0, b: 0, alpha: 0 }, + threshold: 8, + }) + .toBuffer({ resolveWithObject: true }); + const padding = Math.max( + 12, + Math.min( + 48, + Math.round(Math.max(trimmed.info.width, trimmed.info.height) * 0.04), + ), + ); + const output = await sharp(trimmed.data, { failOn: "none" }) + .flatten({ background: "#ffffff" }) + .extend({ + top: padding, + bottom: padding, + left: padding, + right: padding, + background: "#ffffff", + }) + .resize({ + width: maxSide, + height: maxSide, + fit: "inside", + withoutEnlargement: true, + }) + .jpeg({ quality: jpegQuality, mozjpeg: true }) + .toBuffer(); + return { mime: "image/jpeg", buffer: output }; + } + const resized = image .rotate() .resize({ @@ -100,7 +137,12 @@ export async function inlineVisionImages( for (const url of candidates) { try { const buffer = await fetchImageWithRetry(url, timeoutMs, retries); - const processed = await downscaleForVision(buffer, maxSide, jpegQuality); + const processed = await downscaleForVision( + buffer, + maxSide, + jpegQuality, + options.trimTransparentPadding === true, + ); const dataUrl = `data:${processed.mime};base64,${processed.buffer.toString("base64")}`; inlineDataUrls.push(dataUrl); } catch (error) { diff --git a/lib/db/migrations/0100_add_gpt_image25_detail_replacement.sql b/lib/db/migrations/0100_add_gpt_image25_detail_replacement.sql new file mode 100644 index 000000000..df29444d0 --- /dev/null +++ b/lib/db/migrations/0100_add_gpt_image25_detail_replacement.sql @@ -0,0 +1,100 @@ +WITH preferred_provider AS ( + SELECT p."id" + FROM "AiProvider" p + LEFT JOIN "AiModel" m + ON m."providerId" = p."id" + AND m."modelId" = 'gpt-image-2.5-flare' + WHERE p."type" = 'vip98k' + ORDER BY + CASE WHEN m."id" IS NOT NULL THEN 0 ELSE 1 END, + CASE WHEN p."isEnabled" THEN 0 ELSE 1 END, + p."createdAt" + LIMIT 1 +) +INSERT INTO "AiModel" ( + "providerId", + "modelId", + "displayName", + "description", + "supportsVision", + "supportsImageGen", + "isEnabled", + "isHidden", + "isDefault", + "sortOrder", + "capabilities", + "defaultParams", + "imageGenConfig", + "updatedAt" +) +SELECT + preferred_provider."id", + 'gpt-image-2.5-flare', + 'gpt-image-2.5-flare', + 'VIP98K 多图编辑模型,支持 1K、2K 和 4K', + true, + true, + true, + false, + false, + 103, + '{"provider":"vip98k","kind":"image","mediaType":"image","modelType":"image","outputType":"image","supportedModes":["text-to-image","image-to-image","reference-images"],"supportedImageSizes":["1K","2K","4K"],"supportedAspectRatios":["auto","1:1","4:3","3:4","16:9","9:16","3:2","2:3","4:5","5:4"],"supportsMultiImageReference":true,"maxReferenceImages":8}'::jsonb, + '{"provider":"vip98k","kind":"image","endpoint":"gpt-image-2.5-flare","apiFormat":"vip98k-external-task","taskApi":"external-task","supportedModes":["text-to-image","image-to-image","reference-images"],"imageSize":"2K","quality":"auto","aspectRatio":"auto","imageCount":1}'::jsonb, + '{"maxImages":1,"maxReferenceImages":8,"supportsSize":true,"supportsQuality":true,"supports1K":true,"supports2K":true,"supports4K":true,"supportedImageSizes":["1K","2K","4K"],"supportedAspectRatios":["auto","1:1","4:3","3:4","16:9","9:16","3:2","2:3","4:5","5:4"],"supportsPromptExtend":false,"supportsNegativePrompt":false,"supportsWatermark":false,"supportsSeed":false,"creditCostPerImage":0,"creditCostMultiplierBySize":{"1K":1,"2K":1,"4K":1},"enableLoadingPlaceholder":true,"loadingColor":"yellow"}'::jsonb, + now() +FROM preferred_provider +WHERE NOT EXISTS ( + SELECT 1 + FROM "AiModel" existing + WHERE existing."providerId" = preferred_provider."id" + AND existing."modelId" = 'gpt-image-2.5-flare' +); +--> statement-breakpoint +WITH preferred_provider AS ( + SELECT p."id" + FROM "AiProvider" p + LEFT JOIN "AiModel" m + ON m."providerId" = p."id" + AND m."modelId" = 'gpt-image-2.5-flare' + WHERE p."type" = 'vip98k' + ORDER BY + CASE WHEN m."id" IS NOT NULL THEN 0 ELSE 1 END, + CASE WHEN p."isEnabled" THEN 0 ELSE 1 END, + p."createdAt" + LIMIT 1 +) +UPDATE "AiModel" target +SET + "supportsVision" = true, + "supportsImageGen" = true, + "isEnabled" = true, + "isHidden" = false, + "capabilities" = COALESCE(target."capabilities", '{}'::jsonb) || + '{"provider":"vip98k","kind":"image","mediaType":"image","modelType":"image","outputType":"image","supportedModes":["text-to-image","image-to-image","reference-images"],"supportedImageSizes":["1K","2K","4K"],"supportedAspectRatios":["auto","1:1","4:3","3:4","16:9","9:16","3:2","2:3","4:5","5:4"],"supportsMultiImageReference":true,"maxReferenceImages":8}'::jsonb, + "defaultParams" = COALESCE(target."defaultParams", '{}'::jsonb) || + '{"provider":"vip98k","kind":"image","endpoint":"gpt-image-2.5-flare","apiFormat":"vip98k-external-task","taskApi":"external-task","supportedModes":["text-to-image","image-to-image","reference-images"],"imageSize":"2K","quality":"auto","aspectRatio":"auto","imageCount":1}'::jsonb, + "imageGenConfig" = COALESCE(target."imageGenConfig", '{}'::jsonb) || + '{"maxImages":1,"maxReferenceImages":8,"supportsSize":true,"supportsQuality":true,"supports1K":true,"supports2K":true,"supports4K":true,"supportedImageSizes":["1K","2K","4K"],"supportedAspectRatios":["auto","1:1","4:3","3:4","16:9","9:16","3:2","2:3","4:5","5:4"],"supportsPromptExtend":false,"supportsNegativePrompt":false,"supportsWatermark":false,"supportsSeed":false,"creditCostPerImage":0,"creditCostMultiplierBySize":{"1K":1,"2K":1,"4K":1},"enableLoadingPlaceholder":true,"loadingColor":"yellow"}'::jsonb, + "updatedAt" = now() +FROM preferred_provider +WHERE target."providerId" = preferred_provider."id" + AND target."modelId" = 'gpt-image-2.5-flare'; +--> statement-breakpoint +INSERT INTO "SystemSettings" ("key", "value", "updatedAt") +VALUES ( + 'detail_replacement_image_model_config', + '{"modelId":"gpt-image-2.5-flare"}'::jsonb, + now() +) +ON CONFLICT ("key") DO UPDATE +SET + "value" = EXCLUDED."value", + "updatedAt" = EXCLUDED."updatedAt"; +--> statement-breakpoint +INSERT INTO "SystemSettings" ("key", "value", "updatedAt") +VALUES ( + 'detail_replacement_vision_model_config', + '{"modelId":"qwen3.7-plus"}'::jsonb, + now() +) +ON CONFLICT ("key") DO NOTHING; diff --git a/lib/db/migrations/meta/_journal.json b/lib/db/migrations/meta/_journal.json index 06c56f056..547687599 100644 --- a/lib/db/migrations/meta/_journal.json +++ b/lib/db/migrations/meta/_journal.json @@ -603,6 +603,13 @@ "when": 1788220800000, "tag": "0099_set_detail_replacement_qwen_image3", "breakpoints": true + }, + { + "idx": 84, + "version": "7", + "when": 1788998400000, + "tag": "0100_add_gpt_image25_detail_replacement", + "breakpoints": true } ] } diff --git a/playwright.helpers.config.ts b/playwright.helpers.config.ts index e03170b42..2ae2f5f8e 100644 --- a/playwright.helpers.config.ts +++ b/playwright.helpers.config.ts @@ -18,6 +18,7 @@ export default defineConfig({ /routes\/model-classification\.test\.ts/, /routes\/replacement-layout-lock\.test\.ts/, /routes\/replacement-product-identity\.test\.ts/, + /routes\/alibaba-image-input\.test\.ts/, /routes\/model3d-style-pro-regressions\.test\.ts/, /routes\/clawd-request-lifecycle\.test\.ts/, /routes\/canvas-skill-tag-order\.test\.ts/, diff --git a/tests/routes/alibaba-image-input.test.ts b/tests/routes/alibaba-image-input.test.ts new file mode 100644 index 000000000..25aa93f28 --- /dev/null +++ b/tests/routes/alibaba-image-input.test.ts @@ -0,0 +1,142 @@ +import { expect, test } from "@playwright/test"; +import sharp from "sharp"; +import { + prepareAlibabaImageInputBuffer, + selectAlibabaInputImageUrls, +} from "@/lib/ai/providers/alibaba-image-input"; +import { + resolveDetailAgentApiKey, +} from "@/lib/ai/detail-agent-model-config"; +import { + DEFAULT_DETAIL_REPLACEMENT_VISION_MODEL_ID, + normalizeDetailReplacementVisionModelConfig, +} from "@/lib/ai/detail-replacement-vision-model-config"; +import { selectDetailReplacementGenerationInputs } from "@/lib/ai/detail-replacement-inputs"; + +test("preserves the final layout image when Alibaba accepts only three inputs", () => { + expect( + selectAlibabaInputImageUrls([ + "https://assets.test/product.png", + "https://assets.test/model.png", + "https://assets.test/supplement.png", + "https://assets.test/layout.png", + ]), + ).toEqual([ + "https://assets.test/product.png", + "https://assets.test/model.png", + "https://assets.test/layout.png", + ]); +}); + +test("packs product, required model identity, and final layout into the three-image contract", () => { + const selected = selectDetailReplacementGenerationInputs({ + productImageUrls: [ + "https://assets.test/product-main.png", + "https://assets.test/product-side.png", + ], + modelImageUrl: "https://assets.test/person.png", + includeModelImage: true, + rerunReferenceImageUrls: ["https://assets.test/old-result.png"], + referenceImageUrl: "https://assets.test/layout.png", + }); + + expect(selected.imageUrls).toEqual([ + "https://assets.test/product-main.png", + "https://assets.test/person.png", + "https://assets.test/layout.png", + ]); + expect(selected.inputImageRoleByUrl).toEqual({ + "https://assets.test/product-main.png": "product", + "https://assets.test/person.png": "model", + "https://assets.test/layout.png": "layout", + }); +}); + +test("uses a rerun result as the second input without dropping the final layout", () => { + const selected = selectDetailReplacementGenerationInputs({ + productImageUrls: ["https://assets.test/product.png"], + includeModelImage: false, + rerunReferenceImageUrls: ["https://assets.test/old-result.png"], + referenceImageUrl: "https://assets.test/layout.png", + }); + + expect(selected.imageUrls).toEqual([ + "https://assets.test/product.png", + "https://assets.test/old-result.png", + "https://assets.test/layout.png", + ]); +}); + +test("crops transparent product padding and flattens it onto white for Qwen", async () => { + const product = await sharp({ + create: { + width: 1000, + height: 1000, + channels: 4, + background: { r: 0, g: 0, b: 0, alpha: 0 }, + }, + }) + .composite([ + { + input: await sharp({ + create: { + width: 600, + height: 240, + channels: 4, + background: { r: 25, g: 25, b: 25, alpha: 1 }, + }, + }) + .png() + .toBuffer(), + left: 200, + top: 380, + }, + ]) + .png() + .toBuffer(); + const prepared = await prepareAlibabaImageInputBuffer({ + buffer: product, + contentType: "image/png", + inputRole: "product", + maxInputBytes: 9.5 * 1024 * 1024, + maxSide: 4096, + modelId: "qwen-image-3.0-pro", + }); + const metadata = await sharp(prepared.buffer).metadata(); + const corner = await sharp(prepared.buffer) + .extract({ left: 0, top: 0, width: 1, height: 1 }) + .raw() + .toBuffer(); + + expect(prepared.transformed).toBe(true); + expect(prepared.transparentProductTrimmed).toBe(true); + expect(prepared.extension).toBe("jpg"); + expect(metadata.width).toBeLessThan(800); + expect(metadata.height).toBeLessThan(440); + expect([...corner]).toEqual([255, 255, 255]); +}); + +test("uses the enabled Alibaba provider key for a DashScope detail planner", () => { + expect( + resolveDetailAgentApiKey({ + configuredApiKey: "stale-setting-key", + configuredBaseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1", + enabledAlibabaProviders: [ + { + apiKey: "current-provider-key", + baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1", + }, + ], + }), + ).toBe("current-provider-key"); +}); + +test("uses the dedicated high-accuracy vision model for detail replacement", () => { + expect(DEFAULT_DETAIL_REPLACEMENT_VISION_MODEL_ID).toBe("qwen3.7-plus"); + expect(normalizeDetailReplacementVisionModelConfig(null)).toEqual({ + modelId: "qwen3.7-plus", + }); + expect( + normalizeDetailReplacementVisionModelConfig({ modelId: "custom-vision" }), + ).toEqual({ modelId: "custom-vision" }); +}); diff --git a/tests/routes/replacement-product-identity.test.ts b/tests/routes/replacement-product-identity.test.ts index 85aa2bbd5..a09629001 100644 --- a/tests/routes/replacement-product-identity.test.ts +++ b/tests/routes/replacement-product-identity.test.ts @@ -8,6 +8,7 @@ import { normalizeReplacementProductImageIndexes, parseReplacementProductIdentityProfile, } from "@/lib/ai/replacement-product-identity"; +import { selectDetailReplacementGenerationInputs } from "@/lib/ai/detail-replacement-inputs"; test("parses a reusable multi-view product structure profile", () => { const profile = parseReplacementProductIdentityProfile( @@ -68,12 +69,47 @@ test("makes product topology authoritative over the old detail product", () => { expect(prompt).toContain("旧商品结构必须完整移除"); }); -test("keeps non-Pro detail replacement bound to Qwen Image 3 in database config", () => { - expect(DEFAULT_DETAIL_REPLACEMENT_IMAGE_MODEL_ID).toBe("qwen-image-3.0"); +test("binds non-Pro detail replacement to GPT Image 2.5 Flare in database config", () => { + expect(DEFAULT_DETAIL_REPLACEMENT_IMAGE_MODEL_ID).toBe("gpt-image-2.5-flare"); expect(normalizeDetailReplacementImageModelConfig(null)).toEqual({ - modelId: "qwen-image-3.0", + modelId: "gpt-image-2.5-flare", }); expect( normalizeDetailReplacementImageModelConfig({ modelId: "custom-image-model" }), ).toEqual({ modelId: "custom-image-model" }); }); + +test("keeps the product, required model identity, and final layout within three inputs", () => { + const selected = selectDetailReplacementGenerationInputs({ + productImageUrls: ["https://assets.test/product-1.png", "https://assets.test/product-2.png"], + modelImageUrl: "https://assets.test/model.png", + includeModelImage: true, + rerunReferenceImageUrls: ["https://assets.test/rerun.png"], + referenceImageUrl: "https://assets.test/layout.png", + }); + + expect(selected.imageUrls).toEqual([ + "https://assets.test/product-1.png", + "https://assets.test/model.png", + "https://assets.test/layout.png", + ]); + expect(selected.inputImageRoleByUrl).toEqual({ + "https://assets.test/product-1.png": "product", + "https://assets.test/model.png": "model", + "https://assets.test/layout.png": "layout", + }); +}); + +test("prioritizes an explicit rerun reference while always keeping the final layout", () => { + const selected = selectDetailReplacementGenerationInputs({ + productImageUrls: ["https://assets.test/product-1.png", "https://assets.test/product-2.png"], + rerunReferenceImageUrls: ["https://assets.test/rerun.png"], + referenceImageUrl: "https://assets.test/layout.png", + }); + + expect(selected.imageUrls).toEqual([ + "https://assets.test/product-1.png", + "https://assets.test/rerun.png", + "https://assets.test/layout.png", + ]); +}); diff --git a/tests/routes/vip98k-reference-routing.test.ts b/tests/routes/vip98k-reference-routing.test.ts index 5c06fb4fc..cbe5f97d3 100644 --- a/tests/routes/vip98k-reference-routing.test.ts +++ b/tests/routes/vip98k-reference-routing.test.ts @@ -681,7 +681,18 @@ test.describe("VIP98K reference-image routing", () => { isHidden: false, supportsQuality: true, quality: "auto", - supportedAspectRatios: ["auto"], + supportedAspectRatios: [ + "auto", + "1:1", + "4:3", + "3:4", + "16:9", + "9:16", + "3:2", + "2:3", + "4:5", + "5:4", + ], }, { modelId: "gpt-image-2.5-sunburst", @@ -693,6 +704,83 @@ test.describe("VIP98K reference-image routing", () => { ]); }); + test("repairs stale Flare catalog metadata with verified multi-image and size controls", () => { + const [preset] = buildVip98kModelPresets([ + { + id: "gpt-image-2.5-flare", + type: "image", + capabilities: ["image_to_image", "openai.images.generations"], + invocation: { + kind: "tool", + method: "POST", + path: "/v1/images/generations", + content_type: "application/json", + headers: { "X-Relay-Route-Option": "flare-route" }, + request_example: { + model: "gpt-image-2.5-flare", + n: 1, + prompt: "Edit", + quality: "auto", + stream: false, + }, + }, + }, + ]); + + expect(preset).toMatchObject({ + modelId: "gpt-image-2.5-flare", + supportsVision: true, + isHidden: false, + capabilities: { + supportsMultiImageReference: true, + maxReferenceImages: 8, + supportedImageSizes: ["1K", "2K", "4K"], + supportedAspectRatios: [ + "auto", + "1:1", + "4:3", + "3:4", + "16:9", + "9:16", + "3:2", + "2:3", + "4:5", + "5:4", + ], + }, + defaultParams: { imageSize: "2K" }, + imageGenConfig: { + supportsSize: true, + supports1K: true, + supports2K: true, + supports4K: true, + supportedImageSizes: ["1K", "2K", "4K"], + }, + }); + + const metadata = preset?.capabilities?.vip98k as { + invocation?: Vip98kTaskInvocation; + }; + const invocation = selectVip98kTaskInvocation({ + modelId: "gpt-image-2.5-flare", + configuredInvocation: metadata.invocation || null, + imageCount: 2, + }); + expect(invocation?.headers).toEqual({ + "X-Relay-Route-Option": "flare-route", + }); + expect(invocation?.requestExample.image).toEqual([]); + expect(invocation?.inputSchema).toMatchObject({ + properties: { + aspect_ratio: { type: "string" }, + image: { type: "array" }, + size: { + enum: ["1024x1024", "2048x2048", "2880x2880"], + }, + }, + }); + }); + test("does not inherit multi-image support from broad authorized metadata", () => { const presets = buildVip98kModelPresets([ { @@ -775,9 +863,36 @@ test.describe("VIP98K reference-image routing", () => { preserveResolutionBucket: true, }), ).toBe("4K"); + expect( + resolveVip98kImageTaskSize({ + imageSize: "2K", + aspectRatio: "1:1", + fallbackSize: "1024x1024", + preserveResolutionBucket: true, + useGptImage25AspectSizing: true, + }), + ).toBe("2K"); + expect( + resolveVip98kImageTaskSize({ + imageSize: "2K", + aspectRatio: "2:3", + fallbackSize: "1024x1024", + preserveResolutionBucket: true, + useGptImage25AspectSizing: true, + }), + ).toBe("1664x2512"); + expect( + resolveVip98kImageTaskSize({ + imageSize: "4K", + aspectRatio: "2:3", + fallbackSize: "1024x1024", + preserveResolutionBucket: true, + useGptImage25AspectSizing: true, + }), + ).toBe("2352x3520"); }); - test("advertises only catalog resolutions and explicit aspect-ratio controls", () => { + test("advertises verified Flare resolutions and aspect-ratio controls", () => { const [preset] = buildVip98kModelPresets([ { id: "gpt-image-2.5-flare", @@ -810,7 +925,18 @@ test.describe("VIP98K reference-image routing", () => { supports2K: true, supports4K: true, supportedImageSizes: ["1K", "2K", "4K"], - supportedAspectRatios: ["auto"], + supportedAspectRatios: [ + "auto", + "1:1", + "4:3", + "3:4", + "16:9", + "9:16", + "3:2", + "2:3", + "4:5", + "5:4", + ], }); }); });