fix: optimize replacement prompt and preserve reference size
This commit is contained in:
Your Name
parent
63a01e6a5a
commit
1eb05c980d
@ -1319,6 +1319,75 @@ function buildAgentToolImagesMarkdown(
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||||
return buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength,
|
||||
) as ArrayBuffer;
|
||||
}
|
||||
|
||||
async function createAgentImageThumbnailBuffer(buffer: Buffer): Promise<Buffer> {
|
||||
const sharp = (await import("sharp")).default;
|
||||
return sharp(buffer, { failOn: "none" })
|
||||
.rotate()
|
||||
.resize(500, null, {
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.jpeg({ quality: 82 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
async function normalizeReplacementResultImage(params: {
|
||||
generatedImageUrl: string;
|
||||
referenceImageUrl: string;
|
||||
uploadPrefix: string;
|
||||
}): Promise<{
|
||||
originalUrl: string;
|
||||
thumbnailUrl: string;
|
||||
imageSize: string;
|
||||
}> {
|
||||
const sharp = (await import("sharp")).default;
|
||||
const [generatedBuffer, referenceBuffer] = await Promise.all([
|
||||
fetchImageBufferFromUrl(params.generatedImageUrl),
|
||||
fetchImageBufferFromUrl(params.referenceImageUrl),
|
||||
]);
|
||||
|
||||
const referenceMetadata = await sharp(referenceBuffer, { failOn: "none" })
|
||||
.rotate()
|
||||
.metadata();
|
||||
const targetWidth = referenceMetadata.width;
|
||||
const targetHeight = referenceMetadata.height;
|
||||
|
||||
if (!targetWidth || !targetHeight) {
|
||||
throw new Error("无法读取参考图尺寸");
|
||||
}
|
||||
|
||||
const normalizedBuffer = await sharp(generatedBuffer, { failOn: "none" })
|
||||
.rotate()
|
||||
.resize(targetWidth, targetHeight, {
|
||||
fit: "fill",
|
||||
withoutEnlargement: false,
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
const thumbnailBuffer = await createAgentImageThumbnailBuffer(normalizedBuffer);
|
||||
|
||||
const uniqueSuffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const originalFilename = `${params.uploadPrefix}-orig-${uniqueSuffix}.png`;
|
||||
const thumbnailFilename = `${params.uploadPrefix}-thumb-${uniqueSuffix}.jpg`;
|
||||
const [originalUpload, thumbnailUpload] = await Promise.all([
|
||||
uploadFile(originalFilename, bufferToArrayBuffer(normalizedBuffer)),
|
||||
uploadFile(thumbnailFilename, bufferToArrayBuffer(thumbnailBuffer)),
|
||||
]);
|
||||
|
||||
return {
|
||||
originalUrl: originalUpload.url,
|
||||
thumbnailUrl: thumbnailUpload.url,
|
||||
imageSize: `${targetWidth}x${targetHeight}`,
|
||||
};
|
||||
}
|
||||
|
||||
function buildAgentToolVideoMarkdown(videoUrl: string) {
|
||||
const normalizedUrl = String(videoUrl || "").trim();
|
||||
if (!normalizedUrl) return "";
|
||||
@ -3080,33 +3149,82 @@ function buildAgentReplacementPrompt(params: {
|
||||
totalReferences: number;
|
||||
hasModelImage: boolean;
|
||||
productImageCount: number;
|
||||
templateReferenceCount: number;
|
||||
pairedProductIndex?: number;
|
||||
}) {
|
||||
const imageOrderLines: string[] = [];
|
||||
let nextImageIndex = 1;
|
||||
|
||||
if (params.productImageCount > 0) {
|
||||
const productRangeLabel =
|
||||
params.productImageCount === 1
|
||||
? `图片${nextImageIndex}`
|
||||
: `图片${nextImageIndex}-${nextImageIndex + params.productImageCount - 1}`;
|
||||
imageOrderLines.push(
|
||||
`${productRangeLabel}：同一产品的白底图/细节图，必须综合理解所有产品图，不可只看第一张。`
|
||||
);
|
||||
nextImageIndex += params.productImageCount;
|
||||
}
|
||||
|
||||
if (params.hasModelImage) {
|
||||
imageOrderLines.push(
|
||||
`图片${nextImageIndex}：模特图，仅在当前参考图本身包含人物展示时才允许参考。`
|
||||
);
|
||||
nextImageIndex += 1;
|
||||
}
|
||||
|
||||
if (params.templateReferenceCount > 0) {
|
||||
const templateRangeLabel =
|
||||
params.templateReferenceCount === 1
|
||||
? `图片${nextImageIndex}`
|
||||
: `图片${nextImageIndex}-${nextImageIndex + params.templateReferenceCount - 1}`;
|
||||
imageOrderLines.push(
|
||||
`${templateRangeLabel}：模板补充参考图，只能补充风格/材质氛围，不能覆盖最终参考图版式。`
|
||||
);
|
||||
nextImageIndex += params.templateReferenceCount;
|
||||
}
|
||||
|
||||
imageOrderLines.push(
|
||||
`图片${nextImageIndex}：当前参考图底板（唯一版式基准图），必须保留它的文字排版、信息区、分割线、留白、机位和整体结构。`
|
||||
);
|
||||
|
||||
const instructionLines =
|
||||
params.modeType === "detail_replacement"
|
||||
? [
|
||||
"【执行模式】详情替换",
|
||||
`当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张详情参考图：${params.referenceLabel}`,
|
||||
"图像输入顺序规则必须严格遵守：前面的图片是待替换进去的商品输入图，如有模特图会出现在更前面的输入图里；最后一张图片才是当前详情参考图。",
|
||||
"【图片顺序规则】",
|
||||
...imageOrderLines,
|
||||
"你现在做的不是重新生成一张新图，而是编辑最后一张参考图本身。",
|
||||
"如果前面商品图、模特图与最后一张参考图底板存在冲突，始终以最后一张参考图底板的版式、机位、构图和信息层级为最高优先级。",
|
||||
"最后一张参考图是唯一底板，必须保留这张图的文字排版、品牌字样、标题区、卖点区、分割线、留白、机位、构图和信息层级。",
|
||||
"前面的商品图和模特图只用于替换参考图中的产品主体和与产品直接相关的场景内容，不能改变参考图的版式设计。",
|
||||
params.productImageCount > 1
|
||||
? `前面的 ${params.productImageCount} 张商品输入图都属于同一产品的多角度/多细节白底图，请综合这些图片理解产品结构、材质、颜色、比例、边角、开孔、五金与细节，不可只参考第一张。`
|
||||
: "前面的商品输入图就是当前要替换进去的产品白底图，请准确保持该产品的结构、材质、颜色、比例和细节。",
|
||||
"请先识别最后一张参考图中需要被替换的产品主体所在区域，再在原位置完成替换，不要改动无关区域。",
|
||||
"请严格保留最后一张参考图的版式结构、镜头关系、构图、光影氛围、留白位置和信息层级，只替换其中的商品主体。",
|
||||
"前面的输入商品图用于提供商品主体、材质、颜色和细节特征；如有模特图，请只在参考图本身包含人物展示时再参考模特图。",
|
||||
"绝对不要把最后一张参考图原样返回，也不要忽略前面的商品输入图重新生成无关场景。",
|
||||
"绝对不要忽略最后一张参考图去生成一张无关的新场景图。",
|
||||
"绝对不要移动、删除、改写、重排参考图里的任何文字与排版元素。",
|
||||
"输出结果必须保持最后一张参考图底板的原始纵横比与画布尺寸，不允许擅自改成长图、横图或裁掉任何版面区域。",
|
||||
"如果参考图中没有可替换的产品主体，应直接沿用参考图，不要硬生成。",
|
||||
"不要改动参考图的整体布局，不要新增无关主体，不要删除关键版式元素。",
|
||||
]
|
||||
: [
|
||||
"【执行模式】SKU替换",
|
||||
`当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张 SKU 参考图：${params.referenceLabel}`,
|
||||
"【图片顺序规则】",
|
||||
...imageOrderLines,
|
||||
params.pairedProductIndex !== undefined
|
||||
? `请优先使用第 ${params.pairedProductIndex + 1} 张输入商品图作为本张参考图的一对一替换主体。`
|
||||
: "请使用输入商品图作为本张参考图的一对一替换主体。",
|
||||
params.productImageCount > 1
|
||||
? "其余商品输入图同样是这个产品的补充角度/细节参考，请一并综合理解产品信息，不可忽略。"
|
||||
: "输入商品图就是当前产品主体，请保持产品信息准确一致。",
|
||||
"图像输入顺序规则必须严格遵守：前面的图片是待替换商品图，最后一张图片是当前 SKU 参考图。",
|
||||
"如果前面商品图与最后一张参考图存在冲突，始终以最后一张参考图的机位、场景、光影和构图为准。",
|
||||
"请严格保留最后一张参考图的场景、机位、透视、光影和构图关系，只替换商品主体本身。",
|
||||
"输出结果必须保持最后一张参考图的原始纵横比与画布尺寸。",
|
||||
"输出要自然融合，不要改变参考图原有风格和景别。",
|
||||
];
|
||||
|
||||
@ -6001,16 +6119,28 @@ export async function POST(request: Request) {
|
||||
String(selectedChatModel || "").trim() !== "auto"
|
||||
? String(selectedChatModel || "").trim()
|
||||
: presetModelId;
|
||||
const resolvedAspectRatio =
|
||||
const configuredReplacementAspectRatio =
|
||||
typeof mergedImageParams.aspectRatio === "string"
|
||||
? mergedImageParams.aspectRatio
|
||||
: effectiveImageParams?.aspectRatio || defaultImageQualityParams.aspectRatio;
|
||||
const resolvedAspectRatio = "auto";
|
||||
const resolvedImageSize =
|
||||
typeof mergedImageParams.imageSize === "string"
|
||||
? mergedImageParams.imageSize
|
||||
: effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize;
|
||||
let resolvedModelDisplayName =
|
||||
selectedAgentPreset.defaultModel?.displayName || selectedAgentPreset.name;
|
||||
if (
|
||||
configuredReplacementAspectRatio &&
|
||||
configuredReplacementAspectRatio !== "auto"
|
||||
) {
|
||||
console.info(
|
||||
"[Chat API] Replacement mode forced aspect ratio to auto to preserve reference canvas:",
|
||||
{
|
||||
configuredAspectRatio: configuredReplacementAspectRatio,
|
||||
},
|
||||
);
|
||||
}
|
||||
const replacementVisionModels = await getEnabledAiModelsWithProvider().catch(
|
||||
(error) => {
|
||||
console.warn(
|
||||
@ -6032,11 +6162,14 @@ export async function POST(request: Request) {
|
||||
originalUrl: string;
|
||||
thumbnailUrl?: string | null;
|
||||
};
|
||||
imageSizeLabel?: string | null;
|
||||
modelDisplayName?: string | null;
|
||||
sourceImageUrls: string[];
|
||||
sourceType: "detail_replacement" | "sku_replacement";
|
||||
streamLabel: string;
|
||||
}) => {
|
||||
const persistedImageSize =
|
||||
String(params.imageSizeLabel || "").trim() || resolvedImageSize;
|
||||
const assistantText = prependImageModelBadge(
|
||||
[
|
||||
`### ${params.reference.label}`,
|
||||
@ -6045,7 +6178,7 @@ export async function POST(request: Request) {
|
||||
{
|
||||
originalUrl: params.image.originalUrl,
|
||||
thumbnailUrl: params.image.thumbnailUrl,
|
||||
imageSize: resolvedImageSize,
|
||||
imageSize: persistedImageSize,
|
||||
},
|
||||
]),
|
||||
]
|
||||
@ -6090,7 +6223,7 @@ export async function POST(request: Request) {
|
||||
optimizedPrompt: baseReplacementPrompt || undefined,
|
||||
requestedImageCount: 1,
|
||||
aspectRatio: resolvedAspectRatio,
|
||||
imageSize: resolvedImageSize,
|
||||
imageSize: persistedImageSize,
|
||||
},
|
||||
);
|
||||
|
||||
@ -6219,6 +6352,7 @@ export async function POST(request: Request) {
|
||||
totalReferences: replacementReferences.length,
|
||||
hasModelImage: Boolean(modelInputImageUrl),
|
||||
productImageCount: productInputImageUrls.length,
|
||||
templateReferenceCount: replacementTemplateReferenceImageUrls.length,
|
||||
pairedProductIndex,
|
||||
});
|
||||
console.info("[Chat API] Replacement image generation started:", {
|
||||
@ -6229,6 +6363,12 @@ export async function POST(request: Request) {
|
||||
pairedProductIndex:
|
||||
typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null,
|
||||
sourceImageCount: sourceImageUrls.length,
|
||||
imageOrder: {
|
||||
productImages: productInputImageUrls.length,
|
||||
hasModelImage: Boolean(modelInputImageUrl),
|
||||
templateReferenceImages: replacementTemplateReferenceImageUrls.length,
|
||||
finalReferenceImageIndex: sourceImageUrls.length,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
@ -6249,17 +6389,48 @@ export async function POST(request: Request) {
|
||||
if (!firstImage) {
|
||||
throw new Error("未返回结果图片");
|
||||
}
|
||||
let finalImage = firstImage;
|
||||
let finalImageSizeLabel = resolvedImageSize;
|
||||
try {
|
||||
const normalizedImage = await normalizeReplacementResultImage({
|
||||
generatedImageUrl: firstImage.originalUrl,
|
||||
referenceImageUrl: reference.imageUrl,
|
||||
uploadPrefix: isDetailReplacementPreset
|
||||
? "agent-detail-replacement"
|
||||
: "agent-sku-replacement",
|
||||
});
|
||||
finalImage = {
|
||||
originalUrl: normalizedImage.originalUrl,
|
||||
thumbnailUrl: normalizedImage.thumbnailUrl,
|
||||
};
|
||||
finalImageSizeLabel = normalizedImage.imageSize;
|
||||
} catch (normalizeError) {
|
||||
console.warn(
|
||||
"[Chat API] Failed to normalize replacement output to reference size, using original model output:",
|
||||
{
|
||||
referenceLabel: reference.label,
|
||||
referenceImageUrl: reference.imageUrl,
|
||||
resultImageUrl: firstImage.originalUrl,
|
||||
error:
|
||||
normalizeError instanceof Error
|
||||
? normalizeError.message
|
||||
: "unknown",
|
||||
},
|
||||
);
|
||||
}
|
||||
console.info("[Chat API] Replacement image generation completed:", {
|
||||
mode: replacementSourceType,
|
||||
referenceIndex: referenceIndex + 1,
|
||||
totalReferences: replacementReferences.length,
|
||||
referenceLabel: reference.label,
|
||||
modelDisplayName: resolvedModelDisplayName,
|
||||
resultImageUrl: firstImage.originalUrl,
|
||||
resultImageUrl: finalImage.originalUrl,
|
||||
resultImageSize: finalImageSizeLabel,
|
||||
});
|
||||
await emitReplacementResultMessage({
|
||||
reference,
|
||||
image: firstImage,
|
||||
image: finalImage,
|
||||
imageSizeLabel: finalImageSizeLabel,
|
||||
modelDisplayName: resolvedModelDisplayName,
|
||||
sourceImageUrls: Array.from(
|
||||
new Set([
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user