fix world labs task polling fallback for dynamic response payloads
This commit is contained in:
134
parent
7602365bb8
commit
c27a707776
@ -368,33 +368,96 @@ async function updatePersistedWorldAsset(params: {
|
||||
function extractWorldId(operation: WorldLabsOperation) {
|
||||
const response = getObjectValue(operation.response);
|
||||
const metadata = getObjectValue(operation.metadata);
|
||||
return getStringValue(
|
||||
response?.world_id,
|
||||
response?.worldId,
|
||||
metadata?.world_id,
|
||||
metadata?.worldId
|
||||
);
|
||||
const nestedWorldId =
|
||||
getStringValue(response?.world_id, response?.worldId) ||
|
||||
findNestedString(response, ["world_id", "worldId"]);
|
||||
const metadataWorldId =
|
||||
getStringValue(metadata?.world_id, metadata?.worldId) ||
|
||||
findNestedString(metadata, ["world_id", "worldId"]);
|
||||
|
||||
return getStringValue(nestedWorldId, metadataWorldId);
|
||||
}
|
||||
|
||||
function isWorldLikePayload(value: unknown): value is WorldLabsWorld {
|
||||
const record = getObjectValue(value);
|
||||
if (!record) return false;
|
||||
|
||||
const hasCoreWorldField =
|
||||
typeof record.world_id === "string" ||
|
||||
typeof record.world_marble_url === "string" ||
|
||||
typeof record.display_name === "string" ||
|
||||
typeof record.worldId === "string";
|
||||
const hasPossibleWorldContainerFields = record.assets || record.permission || record.tags || record.world_prompt;
|
||||
const hasModelAliasOnly =
|
||||
typeof record.model === "string" && /world/i.test(record.model);
|
||||
|
||||
return Boolean(hasCoreWorldField || hasPossibleWorldContainerFields || hasModelAliasOnly);
|
||||
}
|
||||
|
||||
function extractWorldCandidate(value: unknown, depth = 0): WorldLabsWorld | null {
|
||||
if (depth > 8) return null;
|
||||
const record = getObjectValue(value);
|
||||
if (!record) return null;
|
||||
|
||||
if (isWorldLikePayload(record)) return record;
|
||||
|
||||
const childCandidates: Array<Record<string, unknown> | null> = [
|
||||
getObjectValue(record.world),
|
||||
getObjectValue(record.result),
|
||||
getObjectValue(record.output),
|
||||
getObjectValue(record.data),
|
||||
getObjectValue(record.payload),
|
||||
];
|
||||
|
||||
for (const child of childCandidates) {
|
||||
if (!child) continue;
|
||||
if (isWorldLikePayload(child)) return child;
|
||||
const nested = extractWorldCandidate(child, depth + 1);
|
||||
if (nested) return nested;
|
||||
}
|
||||
|
||||
for (const child of Object.values(record)) {
|
||||
const nested = extractWorldCandidate(child, depth + 1);
|
||||
if (nested) return nested;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractWorldFromOperation(operation: WorldLabsOperation) {
|
||||
const response = operation.response;
|
||||
if (response && typeof response === "object" && "world_marble_url" in (response as Record<string, unknown>)) {
|
||||
return response as WorldLabsWorld;
|
||||
}
|
||||
return null;
|
||||
return extractWorldCandidate(operation.response);
|
||||
}
|
||||
|
||||
function hasUsableWorldAssets(world?: WorldLabsWorld | null) {
|
||||
const assets = world?.assets || null;
|
||||
if (!assets) return false;
|
||||
const spzUrls = assets.splats?.spz_urls;
|
||||
const meshRecord = getObjectValue(assets.mesh);
|
||||
const imageryRecord = getObjectValue(assets.imagery);
|
||||
const splatsRecord = getObjectValue(assets.splats);
|
||||
const spzUrls =
|
||||
splatsRecord?.spz_urls || splatsRecord?.spzUrls || splatsRecord?.splat_urls;
|
||||
return Boolean(
|
||||
getStringValue(assets.thumbnail_url) ||
|
||||
getStringValue(assets.imagery?.pano_url) ||
|
||||
getStringValue(assets.mesh?.collider_mesh_url) ||
|
||||
getStringValue(
|
||||
imageryRecord?.pano_url,
|
||||
imageryRecord?.panoUrl,
|
||||
imageryRecord?.panorama_url,
|
||||
imageryRecord?.panoramaUrl
|
||||
) ||
|
||||
getStringValue(
|
||||
meshRecord?.collider_mesh_url,
|
||||
meshRecord?.colliderMeshUrl,
|
||||
meshRecord?.collider_mesh,
|
||||
meshRecord?.mesh_url,
|
||||
meshRecord?.meshUrl,
|
||||
meshRecord?.model_url,
|
||||
meshRecord?.modelUrl
|
||||
) ||
|
||||
(spzUrls &&
|
||||
typeof spzUrls === "object" &&
|
||||
Object.values(spzUrls).some((value) => typeof value === "string" && value.trim()))
|
||||
Object.values(spzUrls).some(
|
||||
(value) => typeof value === "string" && value.trim()
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
@ -908,22 +971,68 @@ export async function GET(request: Request, { params }: RouteParams) {
|
||||
let world = extractWorldFromOperation(operation);
|
||||
const worldId = extractWorldId(operation);
|
||||
if ((!world || !hasUsableWorldAssets(world)) && worldId) {
|
||||
const detailedWorld = await worldLabsRequest<WorldLabsWorld>(config, {
|
||||
method: "GET",
|
||||
path: `/marble/v1/worlds/${encodeURIComponent(worldId)}`,
|
||||
timeoutMs: resolveWorldLabsStatusTimeoutMs(),
|
||||
});
|
||||
world = world
|
||||
? {
|
||||
...world,
|
||||
...detailedWorld,
|
||||
assets: detailedWorld.assets || world.assets || null,
|
||||
}
|
||||
: detailedWorld;
|
||||
try {
|
||||
const detailedWorld = await worldLabsRequest<WorldLabsWorld>(config, {
|
||||
method: "GET",
|
||||
path: `/marble/v1/worlds/${encodeURIComponent(worldId)}`,
|
||||
timeoutMs: resolveWorldLabsStatusTimeoutMs(),
|
||||
});
|
||||
world = world
|
||||
? {
|
||||
...world,
|
||||
...detailedWorld,
|
||||
assets: detailedWorld.assets || world.assets || null,
|
||||
}
|
||||
: detailedWorld;
|
||||
} catch (error) {
|
||||
logger.warn("world detail fetch failed, retry with running status", {
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
taskIdLength: normalizedTaskId.length,
|
||||
providerId: config.providerId || null,
|
||||
worldId,
|
||||
error: error instanceof Error ? error.message : "unknown",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!world) {
|
||||
throw new Error("World Labs 任务已完成，但未返回 world 数据");
|
||||
const pendingPayload = {
|
||||
...(getObjectValue(persistedWorldAsset?.assetPayload) || {}),
|
||||
providerId: config.providerId || null,
|
||||
providerName: config.providerName || null,
|
||||
worldId: worldId || null,
|
||||
worldUrl: null,
|
||||
displayName: null,
|
||||
model: null,
|
||||
assetsPending: true,
|
||||
operationMetadata: operation.metadata ?? null,
|
||||
};
|
||||
await updatePersistedWorldAsset({
|
||||
id: persistedWorldAsset?.id,
|
||||
status: "running",
|
||||
progress: 95,
|
||||
assetPayload: pendingPayload,
|
||||
});
|
||||
logger.warn("task completed but world payload is missing", {
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
taskIdLength: normalizedTaskId.length,
|
||||
providerId: config.providerId || null,
|
||||
worldId: worldId || null,
|
||||
});
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
taskId: normalizedTaskId,
|
||||
type: "world",
|
||||
status: "running",
|
||||
progress: 95,
|
||||
output: operation.response ?? null,
|
||||
worldId: worldId || null,
|
||||
worldUrl: null,
|
||||
previewImageUrl: null,
|
||||
assetPayload: pendingPayload,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (!hasUsableWorldAssets(world)) {
|
||||
|
||||
@ -46,7 +46,10 @@ import {
|
||||
type AiWashControlLevel,
|
||||
type AiWashPreset,
|
||||
} from "@/lib/ai-wash/config";
|
||||
import { downloadCanvasImageAsPng } from "@/lib/client/canvas-image-download";
|
||||
import {
|
||||
downloadCanvasImageAsPng,
|
||||
getCanvasImageDownloadErrorMessage,
|
||||
} from "@/lib/client/canvas-image-download";
|
||||
import { buildImageProxyUrl } from "@/lib/image-delivery";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { imageGenerationService } from "@/services/canvas";
|
||||
@ -1019,7 +1022,7 @@ export function AiWashWorkspace({
|
||||
});
|
||||
toast.success("图片已开始下载");
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : "下载失败");
|
||||
toast.error(getCanvasImageDownloadErrorMessage(error));
|
||||
} finally {
|
||||
setDownloadingResultId((current) =>
|
||||
current === downloadId ? null : current
|
||||
|
||||
@ -64,6 +64,7 @@ import {
|
||||
} from "@/lib/client/upload-with-retry";
|
||||
import {
|
||||
downloadCanvasImageAsPng,
|
||||
getCanvasImageDownloadErrorMessage,
|
||||
type CanvasImageDownloadProgress,
|
||||
} from "@/lib/client/canvas-image-download";
|
||||
import { canvasService } from "@/services/canvas";
|
||||
@ -4312,11 +4313,7 @@ export function CanvasChatPanel({
|
||||
toast.message("已取消下载");
|
||||
} else {
|
||||
console.error("下载原图失败:", error);
|
||||
toast.error(
|
||||
error instanceof Error
|
||||
? `${error.message}，请稍后重试或重新生成图片`
|
||||
: "图片源文件暂时不可访问，请稍后重试或重新生成图片"
|
||||
);
|
||||
toast.error(getCanvasImageDownloadErrorMessage(error));
|
||||
}
|
||||
} finally {
|
||||
if (downloadAbortControllerRef.current === controller) {
|
||||
|
||||
@ -170,6 +170,7 @@ import {
|
||||
import { fetchUploadWithRetry } from "@/lib/client/upload-with-retry";
|
||||
import {
|
||||
downloadCanvasImageAsPng,
|
||||
getCanvasImageDownloadErrorMessage,
|
||||
type CanvasImageDownloadProgress,
|
||||
} from "@/lib/client/canvas-image-download";
|
||||
import type { DetailUiDslV2Document } from "@/lib/detail-ui-dsl-v2";
|
||||
@ -8331,11 +8332,7 @@ export function ImageNode({
|
||||
toast.message("已取消下载");
|
||||
} else {
|
||||
console.error("下载失败:", error);
|
||||
toast.error(
|
||||
error instanceof Error
|
||||
? `${error.message}，源图可能已失效，请稍后重试或重新生成图片`
|
||||
: "图片源文件暂时不可访问，请稍后重试或重新生成图片"
|
||||
);
|
||||
toast.error(getCanvasImageDownloadErrorMessage(error));
|
||||
}
|
||||
} finally {
|
||||
if (downloadAbortControllerRef.current === controller) {
|
||||
|
||||
@ -29,7 +29,10 @@ import {
|
||||
SelectTrigger,
|
||||
} from "@/components/ui/select";
|
||||
import { imageGenerationService } from "@/services/canvas";
|
||||
import { downloadCanvasImageAsPng } from "@/lib/client/canvas-image-download";
|
||||
import {
|
||||
downloadCanvasImageAsPng,
|
||||
getCanvasImageDownloadErrorMessage,
|
||||
} from "@/lib/client/canvas-image-download";
|
||||
import { cn, fetcher } from "@/lib/utils";
|
||||
import type {
|
||||
PublicStyleProCategory,
|
||||
@ -1513,7 +1516,7 @@ export function StyleProWorkspace({
|
||||
});
|
||||
toast.success("图片已开始下载");
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : "下载失败");
|
||||
toast.error(getCanvasImageDownloadErrorMessage(error));
|
||||
} finally {
|
||||
setDownloadingImageId((current) =>
|
||||
current === downloadId ? null : current
|
||||
|
||||
190
lib/canvas/layout-replace-model-selection.ts
Normal file
190
lib/canvas/layout-replace-model-selection.ts
Normal file
@ -0,0 +1,190 @@
|
||||
type LayoutReplaceModelDescriptor = {
|
||||
capabilities?: unknown;
|
||||
displayName?: string | null;
|
||||
healthStatus?: string | null;
|
||||
id: string;
|
||||
imageGenConfig?: unknown;
|
||||
isDefault?: boolean | null;
|
||||
isSubModel?: boolean | null;
|
||||
modelId?: string | null;
|
||||
supportsImageGen?: boolean | null;
|
||||
supportsVision?: boolean | null;
|
||||
};
|
||||
|
||||
type LayoutReplaceProviderDescriptor = {
|
||||
type?: string | null;
|
||||
};
|
||||
|
||||
export type CanvasLayoutReplaceModelCandidate = {
|
||||
model: LayoutReplaceModelDescriptor;
|
||||
provider: LayoutReplaceProviderDescriptor;
|
||||
};
|
||||
|
||||
const MODEL_PREFERENCE_PATTERNS: RegExp[] = [
|
||||
/^snapgen\/gpt-image-2$/i,
|
||||
/^openai\/gpt-image-2-developer\/edit$/i,
|
||||
/^openai\/gpt-image-2\/edit$/i,
|
||||
/^bytedance\/seedream-v5(?:\.0)?-pro\/edit$/i,
|
||||
/^qwen\/qwen-image-2(?:\.0)?-pro\/edit$/i,
|
||||
/gpt-image.*\/edit/i,
|
||||
/seedream.*\/edit/i,
|
||||
/qwen-image.*\/edit/i,
|
||||
/(?:^|[\s/_-])edit(?:$|[\s/_-])/i,
|
||||
];
|
||||
|
||||
const PROVIDER_PREFERENCE = new Map([
|
||||
["vip98k", 30],
|
||||
["atlascloud", 25],
|
||||
["openai-image", 20],
|
||||
["volcano-vision", 15],
|
||||
["snapgen-image", 10],
|
||||
]);
|
||||
|
||||
function readPositiveNumber(source: unknown, keys: string[]) {
|
||||
if (!source || typeof source !== "object") return null;
|
||||
const record = source as Record<string, unknown>;
|
||||
for (const key of keys) {
|
||||
const parsed = Number(record[key]);
|
||||
if (Number.isFinite(parsed) && parsed > 0) return parsed;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function getConfiguredReferenceImageLimit(model: LayoutReplaceModelDescriptor) {
|
||||
return (
|
||||
readPositiveNumber(model.imageGenConfig, [
|
||||
"maxReferenceImages",
|
||||
"maxInputImages",
|
||||
]) ||
|
||||
readPositiveNumber(model.capabilities, [
|
||||
"maxReferenceImages",
|
||||
"maxInputImages",
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
function getModelDescriptor(candidate: CanvasLayoutReplaceModelCandidate) {
|
||||
return JSON.stringify([
|
||||
candidate.model.modelId,
|
||||
candidate.model.displayName,
|
||||
candidate.model.capabilities,
|
||||
candidate.model.imageGenConfig,
|
||||
]).toLowerCase();
|
||||
}
|
||||
|
||||
export function isCanvasLayoutReplaceModelEligible(
|
||||
candidate: CanvasLayoutReplaceModelCandidate,
|
||||
) {
|
||||
if (String(candidate.provider.type || "").trim() === "fal-image") return false;
|
||||
if (!candidate.model.supportsImageGen || candidate.model.isSubModel) return false;
|
||||
if (candidate.model.supportsVision === false) return false;
|
||||
if (candidate.model.healthStatus === "auto_disabled") return false;
|
||||
|
||||
const referenceImageLimit = getConfiguredReferenceImageLimit(candidate.model);
|
||||
return referenceImageLimit === null || referenceImageLimit >= 2;
|
||||
}
|
||||
|
||||
function getCanvasLayoutReplaceModelScore(
|
||||
candidate: CanvasLayoutReplaceModelCandidate,
|
||||
) {
|
||||
const modelId = String(candidate.model.modelId || "").trim();
|
||||
const descriptor = getModelDescriptor(candidate);
|
||||
const preferenceIndex = MODEL_PREFERENCE_PATTERNS.findIndex(
|
||||
(pattern) => pattern.test(modelId) || pattern.test(descriptor),
|
||||
);
|
||||
const modelScore =
|
||||
preferenceIndex >= 0 ? (MODEL_PREFERENCE_PATTERNS.length - preferenceIndex) * 100 : 0;
|
||||
const healthScore = candidate.model.healthStatus === "healthy" ? 50 : 0;
|
||||
const providerScore =
|
||||
PROVIDER_PREFERENCE.get(candidate.provider.type || "") || 0;
|
||||
const defaultScore = candidate.model.isDefault ? 5 : 0;
|
||||
return modelScore + healthScore + providerScore + defaultScore;
|
||||
}
|
||||
|
||||
export function rankCanvasLayoutReplaceModels(
|
||||
models: CanvasLayoutReplaceModelCandidate[],
|
||||
requestedModel?: string | null,
|
||||
) {
|
||||
const eligible = models.filter(isCanvasLayoutReplaceModelEligible);
|
||||
const requested = String(requestedModel || "").trim();
|
||||
|
||||
return eligible.sort((left, right) => {
|
||||
if (requested && requested !== "auto") {
|
||||
const leftRequested =
|
||||
left.model.id === requested || left.model.modelId === requested;
|
||||
const rightRequested =
|
||||
right.model.id === requested || right.model.modelId === requested;
|
||||
if (leftRequested !== rightRequested) return leftRequested ? -1 : 1;
|
||||
}
|
||||
|
||||
const scoreDifference =
|
||||
getCanvasLayoutReplaceModelScore(right) -
|
||||
getCanvasLayoutReplaceModelScore(left);
|
||||
if (scoreDifference !== 0) return scoreDifference;
|
||||
return left.model.id.localeCompare(right.model.id);
|
||||
});
|
||||
}
|
||||
|
||||
export function buildCanvasLayoutReplacePrompt(params: {
|
||||
referenceHeight?: number;
|
||||
referenceWidth?: number;
|
||||
sourceProductAspectRatio?: number;
|
||||
supplementalPrompt?: string;
|
||||
}) {
|
||||
const geometry =
|
||||
params.referenceWidth && params.referenceHeight
|
||||
? `${params.referenceWidth}x${params.referenceHeight}`
|
||||
: "与图2完全相同";
|
||||
const supplement = String(params.supplementalPrompt || "").trim();
|
||||
const sourceProductAspectRatio =
|
||||
Number.isFinite(params.sourceProductAspectRatio) &&
|
||||
Number(params.sourceProductAspectRatio) > 0
|
||||
? Number(params.sourceProductAspectRatio)
|
||||
: null;
|
||||
const sourceProductRatioConstraint = sourceProductAspectRatio
|
||||
? `【商品外框比例硬约束】图1透明轮廓测得商品有效外框宽高比为 ${sourceProductAspectRatio.toFixed(3)}:1。结果中完整商品外框的宽高比必须保持在 ${(sourceProductAspectRatio * 0.96).toFixed(3)} 至 ${(sourceProductAspectRatio * 1.04).toFixed(3)} 之间。若与图2原商品占位冲突，应缩短商品整体高度或扩大整体宽度，绝对不能压窄商品宽度或纵向拉长。生成前必须测量外框复核。`
|
||||
: "";
|
||||
const targetCanvasAspectRatio =
|
||||
params.referenceWidth && params.referenceHeight
|
||||
? params.referenceWidth / params.referenceHeight
|
||||
: null;
|
||||
const targetProductHeightCoverage = 0.76;
|
||||
const targetProductWidthCoverage =
|
||||
sourceProductAspectRatio && targetCanvasAspectRatio
|
||||
? Math.max(
|
||||
0.52,
|
||||
Math.min(
|
||||
0.86,
|
||||
(sourceProductAspectRatio / targetCanvasAspectRatio) *
|
||||
targetProductHeightCoverage,
|
||||
),
|
||||
)
|
||||
: null;
|
||||
const targetProductLeftEdge = targetProductWidthCoverage
|
||||
? (1 - targetProductWidthCoverage) / 2
|
||||
: null;
|
||||
const targetProductRightEdge = targetProductLeftEdge
|
||||
? 1 - targetProductLeftEdge
|
||||
: null;
|
||||
const targetProductCoverageConstraint = targetProductWidthCoverage
|
||||
? `【商品画布占比硬约束】新商品完整外框应占最终画布宽度约 ${Math.round(targetProductWidthCoverage * 100)}%，允许范围 ${Math.round((targetProductWidthCoverage - 0.03) * 100)}% 至 ${Math.round((targetProductWidthCoverage + 0.03) * 100)}%；左外沿应到达画布 x≈${Math.round(Number(targetProductLeftEdge) * 100)}% 处，右外沿应到达 x≈${Math.round(Number(targetProductRightEdge) * 100)}% 处，两侧都不得向中间收窄；外框高度约占画布 ${Math.round(targetProductHeightCoverage * 100)}%。图2旧商品原本偏窄的占位不得继承，这是唯一允许突破图2原商品边界的地方。扩大新商品时，边柜、台灯、花瓶、摆件和椅子保持原位，并继自然遮挡在新商品前方。`
|
||||
: "";
|
||||
|
||||
return [
|
||||
"执行单一商品换模版任务。图1是新商品身份图，图2是目标场景与构图模版，最终只输出一张完整成图。",
|
||||
"【两图职责】以图2为最终画面底图和唯一场景真值；识别并完整移除图2中的原商品，只在原商品占据的合理空间内放入图1的新商品。图1只定义新商品本身，不得把图1的纯色/透明背景带入结果。",
|
||||
"【商品刚性身份锁定，最高优先级】把图1商品视为已经抠好的真实商品像素层，优先完整复用，不要重新绘制或重新设计。只允许对整个商品做等比缩放和整体平移；禁止透视拉伸、视角重构、横向或纵向拉伸、压扁、加宽、缩短、局部缩放、局部重绘、拼接、融合旧商品、改款、简化或增删部件。宁可保留图1的原始轻微侧视角，也不得把它强行改成正视图。",
|
||||
"【结构逐项核对】结果必须保持图1商品的完整外轮廓和长宽高比例，以及每块板件、柜门、玻璃、滑轨、把手、层板、洞洞板、抽屉、圆形拉手、桌面、横梁和四条桌腿的数量、形状、位置、尺寸关系、颜色及材质。尤其必须保留桌面下左侧带圆形拉手的窄抽屉及其右侧抽屉分区，不得合并成两个大抽屉。商品完整出现一次，不得裁切、复制或被旧商品残影穿插。",
|
||||
sourceProductRatioConstraint,
|
||||
targetProductCoverageConstraint,
|
||||
"【场景与前景物件锁定】逐像素保留图2的相机机位、画幅、墙面、地面、边柜、台灯、花瓶、椅子、光影和空间关系，也必须保留兔子摆件、书、玩偶、相框、杯子及桌面小物的原样、数量和位置。先把这些独立前景物件与椅子视为不可编辑覆盖层，只移除其后方的旧商品，再让它们按原遮挡关系覆盖到新商品前方；不得把摆件当作旧商品的一部分清除。",
|
||||
`【尺寸与放置】输出画布为 ${geometry}。根据图2原商品的地面接触点和整体占位，对图1商品只做统一等比缩放与整体平移；不做局部透视变形。宁可让边缘占位略有差异，也要优先保持图1商品的真实比例和结构。`,
|
||||
"【融合要求】清除图2旧商品后补全被遮挡背景，再放入图1商品；添加与图2一致的自然接触阴影、环境反射、景深和色温，但这些融合不得改变商品结构、材质纹理或深浅配色。",
|
||||
supplement
|
||||
? `【用户补充】${supplement}。补充要求只能约束替换细节，不得放宽上述商品刚性身份锁定与场景锁定。`
|
||||
: "",
|
||||
"【生成前自检】逐项比较图1与结果中的商品轮廓、比例、柜体分区、孔洞、门轨、抽屉和腿；任一结构发生拉伸、缺失、增生、错位或与旧商品融合，都必须先修正再输出。不要输出对比图、拼图、说明文字、边框、水印或多个方案。",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
}
|
||||
70
lib/canvas/layout-replace-source-profile.ts
Normal file
70
lib/canvas/layout-replace-source-profile.ts
Normal file
@ -0,0 +1,70 @@
|
||||
import "server-only";
|
||||
|
||||
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||||
|
||||
export type CanvasLayoutReplaceSourceProfile = {
|
||||
boundingHeight: number;
|
||||
boundingWidth: number;
|
||||
silhouetteAspectRatio: number;
|
||||
sourceHeight: number;
|
||||
sourceWidth: number;
|
||||
transparentPixelRatio: number;
|
||||
};
|
||||
|
||||
export async function resolveCanvasLayoutReplaceSourceProfile(
|
||||
sourceImageUrl: string,
|
||||
): Promise<CanvasLayoutReplaceSourceProfile | null> {
|
||||
try {
|
||||
const sharp = (await import("sharp")).default;
|
||||
const sourceBuffer = await fetchImageBufferFromUrl(sourceImageUrl);
|
||||
const { data, info } = await sharp(sourceBuffer, { failOn: "none" })
|
||||
.rotate()
|
||||
.ensureAlpha()
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true });
|
||||
const pixelCount = info.width * info.height;
|
||||
if (pixelCount <= 0 || info.channels < 4) return null;
|
||||
|
||||
let transparentPixelCount = 0;
|
||||
let minX = info.width;
|
||||
let minY = info.height;
|
||||
let maxX = -1;
|
||||
let maxY = -1;
|
||||
|
||||
for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) {
|
||||
const alpha = data[pixelIndex * info.channels + 3] || 0;
|
||||
if (alpha < 250) transparentPixelCount += 1;
|
||||
if (alpha < 24) continue;
|
||||
|
||||
const x = pixelIndex % info.width;
|
||||
const y = Math.floor(pixelIndex / info.width);
|
||||
if (x < minX) minX = x;
|
||||
if (x > maxX) maxX = x;
|
||||
if (y < minY) minY = y;
|
||||
if (y > maxY) maxY = y;
|
||||
}
|
||||
|
||||
const transparentPixelRatio = transparentPixelCount / pixelCount;
|
||||
if (transparentPixelRatio < 0.01 || maxX < minX || maxY < minY) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const boundingWidth = maxX - minX + 1;
|
||||
const boundingHeight = maxY - minY + 1;
|
||||
if (boundingWidth < 2 || boundingHeight < 2) return null;
|
||||
|
||||
return {
|
||||
boundingHeight,
|
||||
boundingWidth,
|
||||
silhouetteAspectRatio: boundingWidth / boundingHeight,
|
||||
sourceHeight: info.height,
|
||||
sourceWidth: info.width,
|
||||
transparentPixelRatio,
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn("[Canvas Layout Replace] Failed to resolve source profile:", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -2,25 +2,22 @@ import "server-only";
|
||||
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import {
|
||||
analyzeReplacementExecutionPlan,
|
||||
buildAgentReplacementPrompt,
|
||||
createAgentImageThumbnailBuffer,
|
||||
normalizeReplacementResultImage,
|
||||
resolveDirectReplacementModelRef,
|
||||
resolveReplacementReferenceGeometry,
|
||||
resolveReplacementScenePlanningRuntime,
|
||||
type AgentDetailReplacementStrategy,
|
||||
} from "@/app/(chat)/api/chat/_lib/detail-replacement";
|
||||
import { saveGeneratedImages } from "@/lib/ai/runtime/generated-image-persistence";
|
||||
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||||
import { db } from "@/lib/db";
|
||||
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||||
import { generatedImage } from "@/lib/db/schema";
|
||||
import {
|
||||
generateCanvasImage,
|
||||
type CanvasGenerateImageResult,
|
||||
} from "@/lib/canvas/generate-image";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import {
|
||||
buildCanvasLayoutReplacePrompt,
|
||||
rankCanvasLayoutReplaceModels,
|
||||
} from "@/lib/canvas/layout-replace-model-selection";
|
||||
import { resolveCanvasLayoutReplaceSourceProfile } from "@/lib/canvas/layout-replace-source-profile";
|
||||
import { db } from "@/lib/db";
|
||||
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||||
import { generatedImage } from "@/lib/db/schema";
|
||||
|
||||
export interface CanvasLayoutReplaceTaskRequest {
|
||||
taskKind: "canvas-layout-replace";
|
||||
@ -63,13 +60,6 @@ function normalizeLayoutReplacePromptSupplement(value: unknown) {
|
||||
.slice(0, 600);
|
||||
}
|
||||
|
||||
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||||
return buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength,
|
||||
) as ArrayBuffer;
|
||||
}
|
||||
|
||||
async function saveLayoutReplaceResultToGallery(params: {
|
||||
request: CanvasLayoutReplaceTaskRequest;
|
||||
result: CanvasGenerateImageResult;
|
||||
@ -86,15 +76,16 @@ async function saveLayoutReplaceResultToGallery(params: {
|
||||
.where(
|
||||
and(
|
||||
eq(generatedImage.userId, params.userId),
|
||||
eq(generatedImage.originalUrl, originalUrl)
|
||||
)
|
||||
eq(generatedImage.originalUrl, originalUrl),
|
||||
),
|
||||
)
|
||||
.limit(1);
|
||||
if (existing) return;
|
||||
|
||||
const imageSize = params.request.imageSize || "2K";
|
||||
const prompt = params.result.prompt || "AI 模版替换结果";
|
||||
const modelId = params.result.modelRef || params.result.model || "canvas-layout-replace";
|
||||
const modelId =
|
||||
params.result.modelRef || params.result.model || "canvas-layout-replace";
|
||||
await saveGeneratedImages(
|
||||
``,
|
||||
params.userId,
|
||||
@ -112,413 +103,10 @@ async function saveLayoutReplaceResultToGallery(params: {
|
||||
requestedImageCount: 1,
|
||||
imageSize,
|
||||
skipBilling: true,
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function selectLayoutPatchMask(params: {
|
||||
generated: Buffer;
|
||||
height: number;
|
||||
reference: Buffer;
|
||||
width: number;
|
||||
}) {
|
||||
const { generated, height, reference, width } = params;
|
||||
const pixelCount = width * height;
|
||||
const rawMask = new Uint8Array(pixelCount);
|
||||
const diffThreshold = 34;
|
||||
|
||||
for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) {
|
||||
const offset = pixelIndex * 3;
|
||||
const diff = Math.max(
|
||||
Math.abs((generated[offset] || 0) - (reference[offset] || 0)),
|
||||
Math.abs((generated[offset + 1] || 0) - (reference[offset + 1] || 0)),
|
||||
Math.abs((generated[offset + 2] || 0) - (reference[offset + 2] || 0)),
|
||||
);
|
||||
if (diff >= diffThreshold) {
|
||||
rawMask[pixelIndex] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
const visited = new Uint8Array(pixelCount);
|
||||
const queue = new Int32Array(pixelCount);
|
||||
const components: Array<{
|
||||
area: number;
|
||||
maxX: number;
|
||||
maxY: number;
|
||||
minX: number;
|
||||
minY: number;
|
||||
}> = [];
|
||||
|
||||
for (let start = 0; start < pixelCount; start += 1) {
|
||||
if (!rawMask[start] || visited[start]) continue;
|
||||
let head = 0;
|
||||
let tail = 0;
|
||||
let area = 0;
|
||||
let minX = width;
|
||||
let minY = height;
|
||||
let maxX = 0;
|
||||
let maxY = 0;
|
||||
queue[tail] = start;
|
||||
tail += 1;
|
||||
visited[start] = 1;
|
||||
|
||||
while (head < tail) {
|
||||
const current = queue[head];
|
||||
head += 1;
|
||||
area += 1;
|
||||
const x = current % width;
|
||||
const y = Math.floor(current / width);
|
||||
if (x < minX) minX = x;
|
||||
if (x > maxX) maxX = x;
|
||||
if (y < minY) minY = y;
|
||||
if (y > maxY) maxY = y;
|
||||
|
||||
const left = current - 1;
|
||||
const right = current + 1;
|
||||
const up = current - width;
|
||||
const down = current + width;
|
||||
if (x > 0 && rawMask[left] && !visited[left]) {
|
||||
visited[left] = 1;
|
||||
queue[tail] = left;
|
||||
tail += 1;
|
||||
}
|
||||
if (x < width - 1 && rawMask[right] && !visited[right]) {
|
||||
visited[right] = 1;
|
||||
queue[tail] = right;
|
||||
tail += 1;
|
||||
}
|
||||
if (y > 0 && rawMask[up] && !visited[up]) {
|
||||
visited[up] = 1;
|
||||
queue[tail] = up;
|
||||
tail += 1;
|
||||
}
|
||||
if (y < height - 1 && rawMask[down] && !visited[down]) {
|
||||
visited[down] = 1;
|
||||
queue[tail] = down;
|
||||
tail += 1;
|
||||
}
|
||||
}
|
||||
|
||||
components.push({ area, maxX, maxY, minX, minY });
|
||||
}
|
||||
|
||||
const minArea = Math.max(900, Math.round(pixelCount * 0.0015));
|
||||
const minBoxArea = Math.max(3200, Math.round(pixelCount * 0.008));
|
||||
const candidates = components
|
||||
.filter((component) => {
|
||||
const boxArea =
|
||||
(component.maxX - component.minX + 1) *
|
||||
(component.maxY - component.minY + 1);
|
||||
return component.area >= minArea && boxArea >= minBoxArea;
|
||||
})
|
||||
.sort((left, right) => right.area - left.area);
|
||||
const primary = candidates[0];
|
||||
if (!primary) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const selected = candidates.filter((component, index) => {
|
||||
if (index === 0) return true;
|
||||
if (component.area >= primary.area * 0.18) return true;
|
||||
const expandedPrimary = {
|
||||
maxX: Math.min(width - 1, primary.maxX + Math.round(width * 0.05)),
|
||||
maxY: Math.min(height - 1, primary.maxY + Math.round(height * 0.05)),
|
||||
minX: Math.max(0, primary.minX - Math.round(width * 0.05)),
|
||||
minY: Math.max(0, primary.minY - Math.round(height * 0.05)),
|
||||
};
|
||||
return !(
|
||||
component.maxX < expandedPrimary.minX ||
|
||||
component.minX > expandedPrimary.maxX ||
|
||||
component.maxY < expandedPrimary.minY ||
|
||||
component.minY > expandedPrimary.maxY
|
||||
);
|
||||
});
|
||||
|
||||
const selectedMask = new Uint8Array(pixelCount);
|
||||
for (const component of selected) {
|
||||
for (let y = component.minY; y <= component.maxY; y += 1) {
|
||||
const rowOffset = y * width;
|
||||
for (let x = component.minX; x <= component.maxX; x += 1) {
|
||||
const index = rowOffset + x;
|
||||
if (rawMask[index]) selectedMask[index] = 255;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
mask: selectedMask,
|
||||
maskSource: "generated-diff",
|
||||
selectedComponentCount: selected.length,
|
||||
selectedPixelCount: selected.reduce((sum, component) => sum + component.area, 0),
|
||||
};
|
||||
}
|
||||
|
||||
function selectReferenceProductMask(params: {
|
||||
height: number;
|
||||
reference: Buffer;
|
||||
width: number;
|
||||
}) {
|
||||
const { height, reference, width } = params;
|
||||
const pixelCount = width * height;
|
||||
const rawMask = new Uint8Array(pixelCount);
|
||||
|
||||
for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) {
|
||||
const offset = pixelIndex * 3;
|
||||
const red = reference[offset] || 0;
|
||||
const green = reference[offset + 1] || 0;
|
||||
const blue = reference[offset + 2] || 0;
|
||||
const luminance = 0.2126 * red + 0.7152 * green + 0.0722 * blue;
|
||||
const channelSpan = Math.max(red, green, blue) - Math.min(red, green, blue);
|
||||
if (luminance <= 222 || channelSpan >= 38) {
|
||||
rawMask[pixelIndex] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
const visited = new Uint8Array(pixelCount);
|
||||
const queue = new Int32Array(pixelCount);
|
||||
const components: Array<{
|
||||
area: number;
|
||||
boxArea: number;
|
||||
maxX: number;
|
||||
maxY: number;
|
||||
minX: number;
|
||||
minY: number;
|
||||
}> = [];
|
||||
|
||||
for (let start = 0; start < pixelCount; start += 1) {
|
||||
if (!rawMask[start] || visited[start]) continue;
|
||||
let head = 0;
|
||||
let tail = 0;
|
||||
let area = 0;
|
||||
let minX = width;
|
||||
let minY = height;
|
||||
let maxX = 0;
|
||||
let maxY = 0;
|
||||
queue[tail] = start;
|
||||
tail += 1;
|
||||
visited[start] = 1;
|
||||
|
||||
while (head < tail) {
|
||||
const current = queue[head];
|
||||
head += 1;
|
||||
area += 1;
|
||||
const x = current % width;
|
||||
const y = Math.floor(current / width);
|
||||
if (x < minX) minX = x;
|
||||
if (x > maxX) maxX = x;
|
||||
if (y < minY) minY = y;
|
||||
if (y > maxY) maxY = y;
|
||||
|
||||
const left = current - 1;
|
||||
const right = current + 1;
|
||||
const up = current - width;
|
||||
const down = current + width;
|
||||
if (x > 0 && rawMask[left] && !visited[left]) {
|
||||
visited[left] = 1;
|
||||
queue[tail] = left;
|
||||
tail += 1;
|
||||
}
|
||||
if (x < width - 1 && rawMask[right] && !visited[right]) {
|
||||
visited[right] = 1;
|
||||
queue[tail] = right;
|
||||
tail += 1;
|
||||
}
|
||||
if (y > 0 && rawMask[up] && !visited[up]) {
|
||||
visited[up] = 1;
|
||||
queue[tail] = up;
|
||||
tail += 1;
|
||||
}
|
||||
if (y < height - 1 && rawMask[down] && !visited[down]) {
|
||||
visited[down] = 1;
|
||||
queue[tail] = down;
|
||||
tail += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const boxArea = (maxX - minX + 1) * (maxY - minY + 1);
|
||||
components.push({ area, boxArea, maxX, maxY, minX, minY });
|
||||
}
|
||||
|
||||
const minArea = Math.max(1400, Math.round(pixelCount * 0.002));
|
||||
const minBoxArea = Math.max(6000, Math.round(pixelCount * 0.012));
|
||||
const candidates = components
|
||||
.filter((component) => {
|
||||
const boxWidth = component.maxX - component.minX + 1;
|
||||
const boxHeight = component.maxY - component.minY + 1;
|
||||
const aspect = boxWidth / Math.max(1, boxHeight);
|
||||
const centerY = (component.minY + component.maxY) / 2;
|
||||
const looksLikeCtaButton = aspect >= 6 && boxHeight <= height * 0.08;
|
||||
const looksLikeBottomUtility = centerY >= height * 0.78 && aspect >= 3.5;
|
||||
return (
|
||||
component.area >= minArea &&
|
||||
component.boxArea >= minBoxArea &&
|
||||
!looksLikeCtaButton &&
|
||||
!looksLikeBottomUtility
|
||||
);
|
||||
})
|
||||
.sort((left, right) => right.area - left.area);
|
||||
const primary = candidates[0];
|
||||
if (!primary) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const selected = candidates.filter((component, index) => {
|
||||
if (index === 0) return true;
|
||||
if (component.area >= primary.area * 0.22) return true;
|
||||
const expandedPrimary = {
|
||||
maxX: Math.min(width - 1, primary.maxX + Math.round(width * 0.08)),
|
||||
maxY: Math.min(height - 1, primary.maxY + Math.round(height * 0.08)),
|
||||
minX: Math.max(0, primary.minX - Math.round(width * 0.08)),
|
||||
minY: Math.max(0, primary.minY - Math.round(height * 0.08)),
|
||||
};
|
||||
return !(
|
||||
component.maxX < expandedPrimary.minX ||
|
||||
component.minX > expandedPrimary.maxX ||
|
||||
component.maxY < expandedPrimary.minY ||
|
||||
component.minY > expandedPrimary.maxY
|
||||
);
|
||||
});
|
||||
|
||||
const selectedMask = new Uint8Array(pixelCount);
|
||||
for (const component of selected) {
|
||||
for (let y = component.minY; y <= component.maxY; y += 1) {
|
||||
const rowOffset = y * width;
|
||||
for (let x = component.minX; x <= component.maxX; x += 1) {
|
||||
const index = rowOffset + x;
|
||||
if (rawMask[index]) selectedMask[index] = 255;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
mask: selectedMask,
|
||||
maskSource: "reference-product",
|
||||
selectedComponentCount: selected.length,
|
||||
selectedPixelCount: selected.reduce((sum, component) => sum + component.area, 0),
|
||||
};
|
||||
}
|
||||
|
||||
async function buildLayoutLockedCompositeImage(params: {
|
||||
generatedImageUrl: string;
|
||||
referenceGeometry: { height: number; width: number };
|
||||
referenceImageUrl: string;
|
||||
}) {
|
||||
const sharp = (await import("sharp")).default;
|
||||
const [referenceSourceBuffer, generatedSourceBuffer] = await Promise.all([
|
||||
fetchImageBufferFromUrl(params.referenceImageUrl),
|
||||
fetchImageBufferFromUrl(params.generatedImageUrl),
|
||||
]);
|
||||
const targetWidth = params.referenceGeometry.width;
|
||||
const targetHeight = params.referenceGeometry.height;
|
||||
const referenceBuffer = await sharp(referenceSourceBuffer, { failOn: "none" })
|
||||
.rotate()
|
||||
.resize(targetWidth, targetHeight, {
|
||||
fit: "fill",
|
||||
withoutEnlargement: false,
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
const generatedBuffer = await sharp(generatedSourceBuffer, { failOn: "none" })
|
||||
.rotate()
|
||||
.resize(targetWidth, targetHeight, {
|
||||
background: { r: 255, g: 255, b: 255, alpha: 1 },
|
||||
fit: "contain",
|
||||
position: "centre",
|
||||
withoutEnlargement: false,
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
const [referenceRaw, generatedRaw] = await Promise.all([
|
||||
sharp(referenceBuffer, { failOn: "none" })
|
||||
.removeAlpha()
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true }),
|
||||
sharp(generatedBuffer, { failOn: "none" })
|
||||
.removeAlpha()
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true }),
|
||||
]);
|
||||
if (
|
||||
referenceRaw.info.width !== generatedRaw.info.width ||
|
||||
referenceRaw.info.height !== generatedRaw.info.height
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const selectedMask =
|
||||
selectReferenceProductMask({
|
||||
height: referenceRaw.info.height,
|
||||
reference: referenceRaw.data,
|
||||
width: referenceRaw.info.width,
|
||||
}) ||
|
||||
selectLayoutPatchMask({
|
||||
generated: generatedRaw.data,
|
||||
height: referenceRaw.info.height,
|
||||
reference: referenceRaw.data,
|
||||
width: referenceRaw.info.width,
|
||||
});
|
||||
if (!selectedMask) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const usesReferenceProductMask = selectedMask.maskSource === "reference-product";
|
||||
const firstBlurRadius = usesReferenceProductMask ? 14 : 10;
|
||||
const maskThreshold = usesReferenceProductMask ? 12 : 2;
|
||||
const pixelCount = targetWidth * targetHeight;
|
||||
const alphaMask = await sharp(selectedMask.mask, {
|
||||
raw: {
|
||||
channels: 1,
|
||||
height: targetHeight,
|
||||
width: targetWidth,
|
||||
},
|
||||
})
|
||||
.blur(firstBlurRadius)
|
||||
.threshold(maskThreshold)
|
||||
.blur(4)
|
||||
.extractChannel(0)
|
||||
.raw()
|
||||
.toBuffer();
|
||||
if (alphaMask.length !== pixelCount) {
|
||||
throw new Error(
|
||||
`排版替换蒙版通道异常: expected ${pixelCount}, got ${alphaMask.length}`,
|
||||
);
|
||||
}
|
||||
const generatedPatchRaw = Buffer.alloc(pixelCount * 4);
|
||||
const sourceChannels = generatedRaw.info.channels;
|
||||
for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) {
|
||||
const sourceOffset = pixelIndex * sourceChannels;
|
||||
const targetOffset = pixelIndex * 4;
|
||||
generatedPatchRaw[targetOffset] = generatedRaw.data[sourceOffset] || 0;
|
||||
generatedPatchRaw[targetOffset + 1] =
|
||||
generatedRaw.data[sourceOffset + 1] || 0;
|
||||
generatedPatchRaw[targetOffset + 2] =
|
||||
generatedRaw.data[sourceOffset + 2] || 0;
|
||||
generatedPatchRaw[targetOffset + 3] = alphaMask[pixelIndex] || 0;
|
||||
}
|
||||
const generatedPatch = await sharp(generatedPatchRaw, {
|
||||
raw: {
|
||||
channels: 4,
|
||||
height: targetHeight,
|
||||
width: targetWidth,
|
||||
},
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
const compositeBuffer = await sharp(referenceBuffer, { failOn: "none" })
|
||||
.composite([{ input: generatedPatch }])
|
||||
.png()
|
||||
.toBuffer();
|
||||
|
||||
return {
|
||||
buffer: compositeBuffer,
|
||||
height: targetHeight,
|
||||
maskSource: selectedMask.maskSource,
|
||||
selectedComponentCount: selectedMask.selectedComponentCount,
|
||||
selectedPixelCount: selectedMask.selectedPixelCount,
|
||||
width: targetWidth,
|
||||
};
|
||||
}
|
||||
|
||||
export async function executeCanvasLayoutReplaceTask(params: {
|
||||
request: CanvasLayoutReplaceTaskRequest;
|
||||
userId: string;
|
||||
@ -536,85 +124,59 @@ export async function executeCanvasLayoutReplaceTask(params: {
|
||||
console.warn("[Canvas Layout Replace] Failed to load models:", error);
|
||||
return undefined;
|
||||
});
|
||||
const resolvedModelRef = await resolveDirectReplacementModelRef({
|
||||
selectedChatModel: params.request.model,
|
||||
finalModelId: params.request.model,
|
||||
models,
|
||||
});
|
||||
if (!resolvedModelRef || resolvedModelRef === "auto") {
|
||||
throw new Error("当前风格化未配置可用的生图模型");
|
||||
const rankedModels = rankCanvasLayoutReplaceModels(
|
||||
models || [],
|
||||
params.request.model,
|
||||
);
|
||||
const selectedModel = rankedModels[0];
|
||||
if (!selectedModel) {
|
||||
throw new Error("AI 模版替换未配置可用的非 FAL 多图编辑模型");
|
||||
}
|
||||
|
||||
const replacementStrategy: AgentDetailReplacementStrategy = "product_only";
|
||||
const referenceLabel =
|
||||
String(params.request.referenceLabel || "").trim() || "画布排版参考图";
|
||||
const referenceGeometry =
|
||||
await resolveReplacementReferenceGeometry(referenceImageUrl);
|
||||
const planningRuntime = await resolveReplacementScenePlanningRuntime({ models });
|
||||
const replacementExecutionPlan = await analyzeReplacementExecutionPlan({
|
||||
modeType: "detail_replacement",
|
||||
replacementStrategy,
|
||||
referenceImageUrl,
|
||||
referenceLabel,
|
||||
productImageUrls: [sourceImageUrl],
|
||||
modelImageUrl: null,
|
||||
workflowPrompt: null,
|
||||
planningRuntime,
|
||||
models,
|
||||
const [referenceGeometry, sourceProfile] = await Promise.all([
|
||||
resolveReplacementReferenceGeometry(referenceImageUrl),
|
||||
resolveCanvasLayoutReplaceSourceProfile(sourceImageUrl),
|
||||
]);
|
||||
const finalPrompt = buildCanvasLayoutReplacePrompt({
|
||||
referenceHeight: referenceGeometry?.height,
|
||||
referenceWidth: referenceGeometry?.width,
|
||||
sourceProductAspectRatio: sourceProfile?.silhouetteAspectRatio,
|
||||
supplementalPrompt: normalizeLayoutReplacePromptSupplement(
|
||||
params.request.supplementalPrompt,
|
||||
),
|
||||
});
|
||||
|
||||
const basePrompt = "仅替换参考详情图中的产品主体";
|
||||
const replacementPrompt = buildAgentReplacementPrompt({
|
||||
basePrompt,
|
||||
modeType: "detail_replacement",
|
||||
replacementStrategy,
|
||||
marketplaceProductDetailReference: false,
|
||||
referenceLabel,
|
||||
referenceIndex: 0,
|
||||
totalReferences: 1,
|
||||
hasModelImage: false,
|
||||
productImageCount: 1,
|
||||
originalProductImageCount: 1,
|
||||
rerunReferenceCount: 0,
|
||||
templateReferenceCount: 0,
|
||||
scenePlan: replacementExecutionPlan.scenePlan,
|
||||
referenceGeometry,
|
||||
});
|
||||
const supplement = normalizeLayoutReplacePromptSupplement(
|
||||
params.request.supplementalPrompt,
|
||||
);
|
||||
const finalPrompt = [
|
||||
replacementPrompt,
|
||||
supplement
|
||||
? `【画布排版替换补充】${supplement}。这段只用于理解新产品材质、颜色、结构或替换细节；不得改写、移动、删除、重排参考图中的任何文字、图标、按钮、标签、分割线、装饰、留白或页面结构。`
|
||||
: "",
|
||||
"【最终验收标准】输出是一张与最后一张参考图相同页面比例的单页结果；文字、图标、按钮、标签、分割线、品牌区、留白和信息层级必须视觉上与参考图一致，只允许旧产品主体/产品展示区被当前产品替换。",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
const retryVariants = rankedModels.slice(1, 3).map((candidate, index) => ({
|
||||
allowFailover: false,
|
||||
label: `non-fal-model-${index + 2}`,
|
||||
model: candidate.model.id,
|
||||
providerImageParams: { quality: "high" },
|
||||
}));
|
||||
|
||||
const generated = await generateCanvasImage({
|
||||
prompt: finalPrompt,
|
||||
displayPrompt: "排版替换结果",
|
||||
model: resolvedModelRef,
|
||||
model: selectedModel.model.id,
|
||||
ownerKey: params.userId,
|
||||
aspectRatio: referenceGeometry?.aspectRatio || "auto",
|
||||
imageSize: params.request.imageSize || "2K",
|
||||
imageCount: 1,
|
||||
promptExtend: false,
|
||||
providerImageParams: { quality: "high" },
|
||||
referenceImages: [sourceImageUrl, referenceImageUrl],
|
||||
allowFailover: false,
|
||||
retryVariants,
|
||||
debugLabel: "canvas-layout-replace",
|
||||
debugMeta: {
|
||||
...(params.request.debugMeta || {}),
|
||||
operation: "layout-replace",
|
||||
layoutReplaceServerAlgorithm: "agent-detail-replacement/product_only",
|
||||
layoutReplaceServerAlgorithm: "rigid-product-scene-replace/v2",
|
||||
sourceNodeId: params.request.sourceNodeId || null,
|
||||
referenceWidth: referenceGeometry?.width,
|
||||
referenceHeight: referenceGeometry?.height,
|
||||
referenceAspectRatio: referenceGeometry?.aspectRatio,
|
||||
pageKind: replacementExecutionPlan.scenePlan.pageKind,
|
||||
generationRoute: replacementExecutionPlan.scenePlan.generationRoute,
|
||||
promptPackPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy,
|
||||
sourceProductAspectRatio: sourceProfile?.silhouetteAspectRatio,
|
||||
requestedModelRef: params.request.model || null,
|
||||
selectedProviderType: selectedModel.provider.type || null,
|
||||
},
|
||||
});
|
||||
const firstImage = generated.images[0];
|
||||
@ -622,64 +184,20 @@ export async function executeCanvasLayoutReplaceTask(params: {
|
||||
throw new Error("排版替换未返回结果图片");
|
||||
}
|
||||
|
||||
let normalizedImage = await normalizeReplacementResultImage({
|
||||
const normalizedImage = await normalizeReplacementResultImage({
|
||||
generatedImageUrl: firstImage.originalUrl,
|
||||
generatedThumbnailUrl: firstImage.thumbnailUrl,
|
||||
referenceImageUrl,
|
||||
referenceGeometry,
|
||||
scenePlan: replacementExecutionPlan.scenePlan,
|
||||
uploadPrefix: "canvas-layout-replace",
|
||||
});
|
||||
|
||||
if (referenceGeometry) {
|
||||
const lockedComposite = await buildLayoutLockedCompositeImage({
|
||||
generatedImageUrl: firstImage.originalUrl,
|
||||
referenceGeometry,
|
||||
referenceImageUrl,
|
||||
}).catch((error) => {
|
||||
console.warn("[Canvas Layout Replace] Layout locked composite failed:", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
referenceImageUrl,
|
||||
resultImageUrl: firstImage.originalUrl,
|
||||
});
|
||||
return null;
|
||||
});
|
||||
|
||||
if (lockedComposite?.buffer) {
|
||||
const thumbnailBuffer = await createAgentImageThumbnailBuffer(
|
||||
lockedComposite.buffer,
|
||||
);
|
||||
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const [originalUpload, thumbnailUpload] = await Promise.all([
|
||||
uploadFile(
|
||||
`canvas-layout-replace-locked-orig-${suffix}.png`,
|
||||
bufferToArrayBuffer(lockedComposite.buffer),
|
||||
),
|
||||
uploadFile(
|
||||
`canvas-layout-replace-locked-thumb-${suffix}.jpg`,
|
||||
bufferToArrayBuffer(thumbnailBuffer),
|
||||
),
|
||||
]);
|
||||
normalizedImage = {
|
||||
originalUrl: originalUpload.url,
|
||||
thumbnailUrl: thumbnailUpload.url || originalUpload.url,
|
||||
imageSize: `${lockedComposite.width}x${lockedComposite.height}`,
|
||||
};
|
||||
console.info("[Canvas Layout Replace] Applied layout locked composite:", {
|
||||
maskSource: lockedComposite.maskSource,
|
||||
referenceWidth: lockedComposite.width,
|
||||
referenceHeight: lockedComposite.height,
|
||||
selectedComponentCount: lockedComposite.selectedComponentCount,
|
||||
selectedPixelCount: lockedComposite.selectedPixelCount,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const result: CanvasGenerateImageResult = {
|
||||
images: [
|
||||
{
|
||||
originalUrl: normalizedImage.originalUrl,
|
||||
thumbnailUrl: normalizedImage.thumbnailUrl || normalizedImage.originalUrl,
|
||||
thumbnailUrl:
|
||||
normalizedImage.thumbnailUrl || normalizedImage.originalUrl,
|
||||
},
|
||||
],
|
||||
model: generated.model,
|
||||
|
||||
@ -16,6 +16,46 @@ export type CanvasImageDownloadResult = {
|
||||
bytes: number;
|
||||
};
|
||||
|
||||
type CanvasImageDownloadErrorCode =
|
||||
| "rate_limited"
|
||||
| "source_unavailable"
|
||||
| "request_failed";
|
||||
|
||||
export class CanvasImageDownloadError extends Error {
|
||||
code: CanvasImageDownloadErrorCode;
|
||||
retryAfterSeconds?: number;
|
||||
status?: number;
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
options: {
|
||||
code: CanvasImageDownloadErrorCode;
|
||||
retryAfterSeconds?: number;
|
||||
status?: number;
|
||||
}
|
||||
) {
|
||||
super(message);
|
||||
this.name = "CanvasImageDownloadError";
|
||||
this.code = options.code;
|
||||
this.retryAfterSeconds = options.retryAfterSeconds;
|
||||
this.status = options.status;
|
||||
}
|
||||
}
|
||||
|
||||
export function getCanvasImageDownloadErrorMessage(error: unknown) {
|
||||
if (error instanceof CanvasImageDownloadError) {
|
||||
if (error.code === "rate_limited") {
|
||||
return error.retryAfterSeconds
|
||||
? `图片下载请求过于频繁，请 ${error.retryAfterSeconds} 秒后重试`
|
||||
: "图片下载请求过于频繁，请稍后重试";
|
||||
}
|
||||
if (error.code === "source_unavailable") {
|
||||
return `${error.message}，源图可能已失效，请稍后重试或重新生成图片`;
|
||||
}
|
||||
}
|
||||
return error instanceof Error ? error.message : "图片下载失败，请稍后重试";
|
||||
}
|
||||
|
||||
type DownloadCanvasImageParams = {
|
||||
url: string;
|
||||
name?: string;
|
||||
@ -24,6 +64,56 @@ type DownloadCanvasImageParams = {
|
||||
onProgress?: (progress: CanvasImageDownloadProgress) => void;
|
||||
};
|
||||
|
||||
type DownloadErrorPayload = {
|
||||
error?: string;
|
||||
retryAfter?: number;
|
||||
};
|
||||
|
||||
const MAX_RATE_LIMIT_RETRIES = 1;
|
||||
|
||||
function parseRetryAfterSeconds(
|
||||
response: Response,
|
||||
payload: DownloadErrorPayload | null
|
||||
) {
|
||||
const headerValue = response.headers.get("Retry-After")?.trim() || "";
|
||||
const numericHeader = Number(headerValue);
|
||||
if (Number.isFinite(numericHeader) && numericHeader >= 0) {
|
||||
return Math.min(60, Math.ceil(numericHeader));
|
||||
}
|
||||
|
||||
if (headerValue) {
|
||||
const retryAt = Date.parse(headerValue);
|
||||
if (Number.isFinite(retryAt)) {
|
||||
return Math.min(60, Math.max(0, Math.ceil((retryAt - Date.now()) / 1000)));
|
||||
}
|
||||
}
|
||||
|
||||
const payloadValue = Number(payload?.retryAfter);
|
||||
return Number.isFinite(payloadValue) && payloadValue >= 0
|
||||
? Math.min(60, Math.ceil(payloadValue))
|
||||
: 1;
|
||||
}
|
||||
|
||||
function waitForRetry(delayMs: number, signal?: AbortSignal) {
|
||||
if (delayMs <= 0) return Promise.resolve();
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const handleAbort = () => {
|
||||
clearTimeout(timer);
|
||||
reject(signal?.reason || new DOMException("已取消下载", "AbortError"));
|
||||
};
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener("abort", handleAbort);
|
||||
resolve();
|
||||
}, delayMs);
|
||||
|
||||
if (signal?.aborted) {
|
||||
handleAbort();
|
||||
return;
|
||||
}
|
||||
signal?.addEventListener("abort", handleAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
function parseAttachmentFilename(contentDisposition: string | null) {
|
||||
if (!contentDisposition) return "";
|
||||
const encodedMatch = contentDisposition.match(/filename\*=UTF-8''([^;]+)/i);
|
||||
@ -155,20 +245,59 @@ export async function downloadCanvasImageAsPng({
|
||||
percent: 8,
|
||||
});
|
||||
|
||||
const response = await fetch("/api/canvas/download-image", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
url: urls[0],
|
||||
urls,
|
||||
name: filenameBase,
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
let response: Response | null = null;
|
||||
let errorPayload: DownloadErrorPayload | null = null;
|
||||
|
||||
if (!response.ok) {
|
||||
const payload = (await response.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(payload?.error || `下载失败（${response.status}）`);
|
||||
for (let attempt = 0; attempt <= MAX_RATE_LIMIT_RETRIES; attempt++) {
|
||||
response = await fetch("/api/canvas/download-image", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
url: urls[0],
|
||||
urls,
|
||||
name: filenameBase,
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
|
||||
if (response.ok) break;
|
||||
errorPayload = (await response.json().catch(() => null)) as
|
||||
| DownloadErrorPayload
|
||||
| null;
|
||||
if (response.status !== 429) break;
|
||||
|
||||
const retryAfterSeconds = parseRetryAfterSeconds(response, errorPayload);
|
||||
if (attempt >= MAX_RATE_LIMIT_RETRIES) {
|
||||
throw new CanvasImageDownloadError(
|
||||
errorPayload?.error || "图片下载请求过于频繁",
|
||||
{
|
||||
code: "rate_limited",
|
||||
retryAfterSeconds,
|
||||
status: response.status,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
onProgress?.({
|
||||
phase: "preparing",
|
||||
message:
|
||||
retryAfterSeconds > 0
|
||||
? `请求较多，${retryAfterSeconds} 秒后自动重试...`
|
||||
: "请求较多，正在自动重试...",
|
||||
percent: 8,
|
||||
});
|
||||
await waitForRetry(retryAfterSeconds * 1000, signal);
|
||||
}
|
||||
|
||||
if (!response?.ok) {
|
||||
const status = response?.status;
|
||||
throw new CanvasImageDownloadError(
|
||||
errorPayload?.error || `下载失败（${status || "网络异常"}）`,
|
||||
{
|
||||
code: status === 502 ? "source_unavailable" : "request_failed",
|
||||
status,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const filename =
|
||||
|
||||
@ -83,6 +83,9 @@ export const rateLimiters = {
|
||||
// 文件上传接口：每分钟60次请求（独立于通用API限流）
|
||||
upload: new RateLimiter(60000, 60),
|
||||
|
||||
// 原图下载需要服务端转码：按登录用户独立限流，避免共享出口 IP 互相影响
|
||||
download: new RateLimiter(60000, 60),
|
||||
|
||||
// 图片代理接口：每分钟3000次请求（图片列表/画布高频加载）
|
||||
imageProxy: new RateLimiter(60000, 3000),
|
||||
|
||||
|
||||
70
proxy.ts
70
proxy.ts
@ -45,14 +45,27 @@ const manualIpBlockCache = new Map<
|
||||
function isHighFrequencyReadApi(pathname: string, method: string) {
|
||||
if (method !== "GET") return false;
|
||||
if (/^\/api\/canvas\/[^/]+\/detail-archives$/.test(pathname)) return true;
|
||||
if (/^\/api\/canvas\/[^/]+\/revision$/.test(pathname)) return true;
|
||||
if (/^\/api\/canvas\/layout-replace\/batches\/[^/]+$/.test(pathname)) {
|
||||
return true;
|
||||
}
|
||||
if (/^\/api\/canvas\/[^/]+$/.test(pathname)) return true;
|
||||
|
||||
return [
|
||||
"/api/ai-wash/config",
|
||||
"/api/models",
|
||||
"/api/canvas",
|
||||
"/api/canvas/model-aliases",
|
||||
"/api/canvas-skill-tags",
|
||||
"/api/canvas/clawd-chat/status",
|
||||
"/api/enhance/pro/config",
|
||||
"/api/featured",
|
||||
"/api/featured/ranking",
|
||||
"/api/home-animation",
|
||||
"/api/nav-links",
|
||||
"/api/scene-fusion/config",
|
||||
"/api/site-settings",
|
||||
"/api/style-pro/config",
|
||||
"/api/messages",
|
||||
"/api/history",
|
||||
"/api/vote",
|
||||
@ -61,6 +74,7 @@ function isHighFrequencyReadApi(pathname: string, method: string) {
|
||||
"/api/tutorial-videos",
|
||||
"/api/user/membership-summary",
|
||||
"/api/user/account-center",
|
||||
"/api/video-canvas/config",
|
||||
].includes(pathname);
|
||||
}
|
||||
|
||||
@ -502,6 +516,51 @@ export async function proxy(request: NextRequest, event: NextFetchEvent) {
|
||||
}
|
||||
);
|
||||
}
|
||||
} else if (
|
||||
pathname === "/api/canvas/download-image" &&
|
||||
request.method === "POST"
|
||||
) {
|
||||
const token = await resolveToken();
|
||||
const userId =
|
||||
typeof token?.sub === "string" && token.sub.length > 0
|
||||
? token.sub
|
||||
: null;
|
||||
const downloadIdentifier = userId
|
||||
? `user:${userId}:canvas-download`
|
||||
: `ip:${clientIP}:canvas-download`;
|
||||
const downloadRateLimit = checkRateLimitByIdentifier(
|
||||
downloadIdentifier,
|
||||
rateLimiters.download
|
||||
);
|
||||
if (!downloadRateLimit.allowed) {
|
||||
const retryAfter = Math.max(
|
||||
1,
|
||||
Math.ceil((downloadRateLimit.resetTime - Date.now()) / 1000)
|
||||
);
|
||||
console.warn(`[Security] Download rate limit exceeded for IP: ${clientIP}`, {
|
||||
pathname,
|
||||
userId,
|
||||
resetTime: new Date(downloadRateLimit.resetTime).toISOString(),
|
||||
});
|
||||
return withAccessLog(
|
||||
NextResponse.json(
|
||||
{
|
||||
error: `图片下载请求过于频繁，请 ${retryAfter} 秒后重试`,
|
||||
retryAfter,
|
||||
},
|
||||
{
|
||||
status: 429,
|
||||
headers: { "Retry-After": String(retryAfter) },
|
||||
}
|
||||
),
|
||||
{
|
||||
outcome: "rate_limited",
|
||||
severity: "high",
|
||||
reasons: ["图片下载限流"],
|
||||
userId,
|
||||
}
|
||||
);
|
||||
}
|
||||
} else if (pathname.startsWith("/api/canvas/") && request.method === "PUT") {
|
||||
const canvasRateLimit = checkRateLimit(request, rateLimiters.canvas);
|
||||
if (!canvasRateLimit.allowed) {
|
||||
@ -623,14 +682,21 @@ export async function proxy(request: NextRequest, event: NextFetchEvent) {
|
||||
} else if (pathname.startsWith("/api/")) {
|
||||
const rateLimit = checkRateLimit(request, rateLimiters.api);
|
||||
if (!rateLimit.allowed) {
|
||||
const retryAfter = Math.max(
|
||||
1,
|
||||
Math.ceil((rateLimit.resetTime - Date.now()) / 1000)
|
||||
);
|
||||
console.warn(`[Security] Rate limit exceeded for IP: ${clientIP}`, {
|
||||
pathname,
|
||||
resetTime: new Date(rateLimit.resetTime).toISOString(),
|
||||
});
|
||||
return withAccessLog(
|
||||
NextResponse.json(
|
||||
{ error: "Too many requests", retryAfter: Math.ceil((rateLimit.resetTime - Date.now()) / 1000) },
|
||||
{ status: 429 }
|
||||
{ error: "Too many requests", retryAfter },
|
||||
{
|
||||
status: 429,
|
||||
headers: { "Retry-After": String(retryAfter) },
|
||||
}
|
||||
),
|
||||
{
|
||||
outcome: "rate_limited",
|
||||
|
||||
57
scripts/prepare-official-model-avatars.mjs
Normal file
57
scripts/prepare-official-model-avatars.mjs
Normal file
@ -0,0 +1,57 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import sharp from 'sharp';
|
||||
|
||||
const sourceDir = process.argv[2] || '/tmp/official-model-logos';
|
||||
const outputDir = process.argv[3] || '/tmp/official-model-avatars';
|
||||
|
||||
const sources = {
|
||||
google: path.join(sourceDir, 'google.png'),
|
||||
openai: path.join(sourceDir, 'openai.svg'),
|
||||
qwen: path.join(sourceDir, 'qwen.png'),
|
||||
runway: path.join(sourceDir, 'runway.png'),
|
||||
bfl: path.join(sourceDir, 'bfl.png'),
|
||||
stepfun: path.join(sourceDir, 'stepfun.png'),
|
||||
volcengine: path.join(sourceDir, 'volcengine.png'),
|
||||
doubao: path.join(sourceDir, 'doubao.png'),
|
||||
};
|
||||
|
||||
const background = Buffer.from(`
|
||||
<svg width="128" height="128" viewBox="0 0 128 128" xmlns="http://www.w3.org/2000/svg">
|
||||
<rect width="128" height="128" rx="28" fill="#F1F2F4"/>
|
||||
<rect x="0.75" y="0.75" width="126.5" height="126.5" rx="27.25" fill="none" stroke="#D8DADE" stroke-width="1.5"/>
|
||||
</svg>
|
||||
`);
|
||||
|
||||
async function openAiMark() {
|
||||
const officialSvg = await fs.readFile(sources.openai, 'utf8');
|
||||
const blossom = officialSvg.match(/<path d="([^"]+)" fill="black"\/>/);
|
||||
if (!blossom) throw new Error('OpenAI Blossom path was not found in the official asset');
|
||||
|
||||
return Buffer.from(`
|
||||
<svg width="270" height="267" viewBox="145 226 270 267" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="${blossom[1]}" fill="#202226"/>
|
||||
</svg>
|
||||
`);
|
||||
}
|
||||
|
||||
async function makeAvatar(name, input) {
|
||||
const mark = await sharp(input)
|
||||
.resize(88, 88, { fit: 'contain', background: { r: 0, g: 0, b: 0, alpha: 0 } })
|
||||
.grayscale()
|
||||
.modulate({ brightness: 0.62 })
|
||||
.png()
|
||||
.toBuffer();
|
||||
|
||||
await sharp(background)
|
||||
.composite([{ input: mark, gravity: 'centre' }])
|
||||
.png({ compressionLevel: 9 })
|
||||
.toFile(path.join(outputDir, `${name}.png`));
|
||||
}
|
||||
|
||||
await fs.mkdir(outputDir, { recursive: true });
|
||||
for (const [name, input] of Object.entries(sources)) {
|
||||
await makeAvatar(name, name === 'openai' ? await openAiMark() : input);
|
||||
}
|
||||
|
||||
console.log(`Prepared ${Object.keys(sources).length} monochrome official model avatars in ${outputDir}`);
|
||||
36
scripts/upload-official-model-avatars.ts
Normal file
36
scripts/upload-official-model-avatars.ts
Normal file
@ -0,0 +1,36 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { uploadFile } from '../lib/storage/service';
|
||||
|
||||
const inputDir = process.argv[2];
|
||||
if (!inputDir) {
|
||||
throw new Error('Usage: tsx scripts/upload-official-model-avatars.ts <avatar-directory>');
|
||||
}
|
||||
|
||||
const brands = [
|
||||
'google',
|
||||
'openai',
|
||||
'qwen',
|
||||
'runway',
|
||||
'bfl',
|
||||
'stepfun',
|
||||
'volcengine',
|
||||
'doubao',
|
||||
] as const;
|
||||
|
||||
async function main() {
|
||||
const uploaded: Record<string, string> = {};
|
||||
for (const brand of brands) {
|
||||
const file = await readFile(path.join(inputDir, `${brand}.png`));
|
||||
const arrayBuffer = file.buffer.slice(file.byteOffset, file.byteOffset + file.byteLength) as ArrayBuffer;
|
||||
const result = await uploadFile(`official-model-avatar-${brand}.png`, arrayBuffer);
|
||||
uploaded[brand] = result.url;
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(uploaded, null, 2));
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
164
scripts/verify-canvas-layout-replace-quality.ts
Normal file
164
scripts/verify-canvas-layout-replace-quality.ts
Normal file
@ -0,0 +1,164 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import dotenv from "dotenv";
|
||||
import postgres from "postgres";
|
||||
import { createAtlasCloud } from "@/lib/ai/providers/atlascloud";
|
||||
import { createSnapgenImage } from "@/lib/ai/providers/snapgen-image";
|
||||
import { createVip98k } from "@/lib/ai/providers/vip98k";
|
||||
import { resolveModelRuntimeId } from "@/lib/ai/model-runtime";
|
||||
import { buildCanvasLayoutReplacePrompt } from "@/lib/canvas/layout-replace-model-selection";
|
||||
import { resolveCanvasLayoutReplaceSourceProfile } from "@/lib/canvas/layout-replace-source-profile";
|
||||
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config();
|
||||
|
||||
function getArgument(name: string) {
|
||||
const prefix = `--${name}=`;
|
||||
return process.argv.find((argument) => argument.startsWith(prefix))?.slice(prefix.length) || "";
|
||||
}
|
||||
|
||||
function requireHttpArgument(name: string) {
|
||||
const value = getArgument(name).trim();
|
||||
if (!/^https?:\/\//i.test(value)) {
|
||||
throw new Error(`缺少有效参数 --${name}=https://...`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function extractOriginalImageUrl(markdown: string) {
|
||||
const match = markdown.match(/!\[thumb\|\|\|([^|\]]+)(?:\|\|\|[^\]]*)?\]\([^)]+\)/);
|
||||
if (match?.[1] && /^https?:\/\//i.test(match[1])) return match[1];
|
||||
const direct = markdown.match(/!\[[^\]]*\]\((https?:\/\/[^)]+)\)/);
|
||||
return direct?.[1] || "";
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const databaseUrl = process.env.POSTGRES_URL;
|
||||
if (!databaseUrl) throw new Error("POSTGRES_URL 未配置");
|
||||
|
||||
const sourceImageUrl = requireHttpArgument("source");
|
||||
const referenceImageUrl = requireHttpArgument("reference");
|
||||
const modelRef = getArgument("model-ref").trim();
|
||||
if (!modelRef) throw new Error("缺少 --model-ref");
|
||||
const outputDirectory = resolve(
|
||||
getArgument("output-dir") || "/tmp/canvas-layout-replace-quality",
|
||||
);
|
||||
|
||||
const sql = postgres(databaseUrl, { max: 1 });
|
||||
try {
|
||||
const [candidate] = await sql<
|
||||
Array<{
|
||||
apiKey: string;
|
||||
baseUrl: string;
|
||||
capabilities: Record<string, unknown> | null;
|
||||
imageGenConfig: Record<string, unknown> | null;
|
||||
modelId: string;
|
||||
providerType: string;
|
||||
}>
|
||||
>`
|
||||
select
|
||||
p."apiKey" as "apiKey",
|
||||
p."baseUrl" as "baseUrl",
|
||||
p.type as "providerType",
|
||||
m."modelId" as "modelId",
|
||||
m."imageGenConfig" as "imageGenConfig",
|
||||
m.capabilities as "capabilities"
|
||||
from "AiModel" m
|
||||
join "AiProvider" p on p.id = m."providerId"
|
||||
where m.id = ${modelRef}
|
||||
and m."isEnabled" = true
|
||||
and p."isEnabled" = true
|
||||
limit 1
|
||||
`;
|
||||
if (!candidate) throw new Error("找不到启用的测试模型");
|
||||
if (!["atlascloud", "snapgen-image", "vip98k"].includes(candidate.providerType)) {
|
||||
throw new Error(`本地质量脚本不支持供应商 ${candidate.providerType}`);
|
||||
}
|
||||
|
||||
const sourceProfile =
|
||||
await resolveCanvasLayoutReplaceSourceProfile(sourceImageUrl);
|
||||
const prompt = buildCanvasLayoutReplacePrompt({
|
||||
referenceHeight: 726,
|
||||
referenceWidth: 543,
|
||||
sourceProductAspectRatio: sourceProfile?.silhouetteAspectRatio,
|
||||
});
|
||||
const runtimeModelId = resolveModelRuntimeId({ modelId: candidate.modelId });
|
||||
const generationOptions = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: prompt },
|
||||
{ type: "image_url", image_url: { url: sourceImageUrl } },
|
||||
{ type: "image_url", image_url: { url: referenceImageUrl } },
|
||||
],
|
||||
},
|
||||
],
|
||||
imageParams: {
|
||||
aspectRatio: "3:4",
|
||||
imageCount: 1,
|
||||
imageSize: "2K",
|
||||
promptExtend: false,
|
||||
quality: "high",
|
||||
},
|
||||
};
|
||||
let result: { text?: string };
|
||||
if (candidate.providerType === "atlascloud") {
|
||||
const model = createAtlasCloud({
|
||||
apiKey: candidate.apiKey,
|
||||
baseURL: candidate.baseUrl,
|
||||
}).chat(runtimeModelId);
|
||||
result = await model.doGenerate({
|
||||
...generationOptions,
|
||||
imageGenConfig:
|
||||
candidate.imageGenConfig as Parameters<
|
||||
typeof model.doGenerate
|
||||
>[0]["imageGenConfig"],
|
||||
});
|
||||
} else if (candidate.providerType === "snapgen-image") {
|
||||
const model = createSnapgenImage({
|
||||
apiKey: candidate.apiKey,
|
||||
baseURL: candidate.baseUrl,
|
||||
}).chat(runtimeModelId);
|
||||
result = await model.doGenerate({
|
||||
...generationOptions,
|
||||
imageGenConfig:
|
||||
candidate.imageGenConfig as Parameters<
|
||||
typeof model.doGenerate
|
||||
>[0]["imageGenConfig"],
|
||||
});
|
||||
} else {
|
||||
const model = createVip98k({
|
||||
apiKey: candidate.apiKey,
|
||||
baseURL: candidate.baseUrl,
|
||||
}).chat(runtimeModelId, {
|
||||
capabilities: candidate.capabilities,
|
||||
imageGenConfig: candidate.imageGenConfig,
|
||||
});
|
||||
result = await model.doGenerate({
|
||||
...generationOptions,
|
||||
imageGenConfig: candidate.imageGenConfig,
|
||||
});
|
||||
}
|
||||
const resultUrl = extractOriginalImageUrl(result.text || "");
|
||||
if (!resultUrl) throw new Error("模型返回中没有结果图片");
|
||||
|
||||
const response = await fetch(resultUrl, {
|
||||
signal: AbortSignal.timeout(120_000),
|
||||
});
|
||||
if (!response.ok) throw new Error(`下载测试结果失败（HTTP ${response.status}）`);
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
const outputPath = resolve(outputDirectory, `result-${Date.now()}.png`);
|
||||
await writeFile(outputPath, Buffer.from(await response.arrayBuffer()));
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({ modelId: runtimeModelId, outputPath, resultUrl })}\n`,
|
||||
);
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
process.exit(1);
|
||||
});
|
||||
107
tests/routes/canvas-download-rate-limit.test.ts
Normal file
107
tests/routes/canvas-download-rate-limit.test.ts
Normal file
@ -0,0 +1,107 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import {
|
||||
CanvasImageDownloadError,
|
||||
downloadCanvasImageAsPng,
|
||||
getCanvasImageDownloadErrorMessage,
|
||||
} from "@/lib/client/canvas-image-download";
|
||||
|
||||
function readSource(path: string) {
|
||||
return readFileSync(resolve(process.cwd(), path), "utf8");
|
||||
}
|
||||
|
||||
test.describe("canvas image download rate limiting", () => {
|
||||
test("isolates downloads by authenticated user and moves polling reads out of the generic bucket", () => {
|
||||
const proxySource = readSource("proxy.ts");
|
||||
const limiterSource = readSource("lib/security/rate-limiter.ts");
|
||||
const downloadBranchIndex = proxySource.indexOf(
|
||||
'pathname === "/api/canvas/download-image"'
|
||||
);
|
||||
const genericBranchIndex = proxySource.indexOf(
|
||||
'} else if (pathname.startsWith("/api/")) {'
|
||||
);
|
||||
|
||||
expect(limiterSource).toContain("download: new RateLimiter(60000, 60)");
|
||||
expect(proxySource).toContain(
|
||||
"`user:" + ["$", "{userId}"].join("") + ":canvas-download`"
|
||||
);
|
||||
expect(proxySource).toContain("rateLimiters.download");
|
||||
expect(downloadBranchIndex).toBeGreaterThan(0);
|
||||
expect(genericBranchIndex).toBeGreaterThan(downloadBranchIndex);
|
||||
expect(proxySource).toContain(
|
||||
'if (/^\\/api\\/canvas\\/[^/]+\\/revision$/.test(pathname)) return true;'
|
||||
);
|
||||
expect(proxySource).toContain('"/api/ai-wash/config"');
|
||||
expect(proxySource).toContain('"/api/canvas/model-aliases"');
|
||||
expect(proxySource).toContain('"/api/featured"');
|
||||
expect(proxySource).toContain('"/api/scene-fusion/config"');
|
||||
expect(proxySource).toContain('headers: { "Retry-After": String(retryAfter) }');
|
||||
});
|
||||
|
||||
test("retries one 429 response and keeps rate limiting separate from source expiry", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let requestCount = 0;
|
||||
globalThis.fetch = async () => {
|
||||
requestCount++;
|
||||
return new Response(
|
||||
JSON.stringify({ error: "Too many requests", retryAfter: 0 }),
|
||||
{
|
||||
status: 429,
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"Retry-After": "0",
|
||||
},
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
try {
|
||||
await expect(
|
||||
downloadCanvasImageAsPng({
|
||||
url: "https://example.com/image.png",
|
||||
})
|
||||
).rejects.toMatchObject({
|
||||
code: "rate_limited",
|
||||
status: 429,
|
||||
});
|
||||
expect(requestCount).toBe(2);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
|
||||
const rateLimitMessage = getCanvasImageDownloadErrorMessage(
|
||||
new CanvasImageDownloadError("Too many requests", {
|
||||
code: "rate_limited",
|
||||
status: 429,
|
||||
})
|
||||
);
|
||||
const sourceMessage = getCanvasImageDownloadErrorMessage(
|
||||
new CanvasImageDownloadError("图片源文件暂时不可访问", {
|
||||
code: "source_unavailable",
|
||||
status: 502,
|
||||
})
|
||||
);
|
||||
|
||||
expect(rateLimitMessage).toBe("图片下载请求过于频繁，请稍后重试");
|
||||
expect(rateLimitMessage).not.toContain("源图");
|
||||
expect(sourceMessage).toContain("源图可能已失效");
|
||||
});
|
||||
|
||||
test("uses the shared error formatter in every canvas image download entry", () => {
|
||||
for (const path of [
|
||||
"components/canvas/nodes/ImageNode.tsx",
|
||||
"components/canvas/CanvasChatPanel.tsx",
|
||||
"components/canvas/AiWashWorkspace.tsx",
|
||||
"components/style-pro/StyleProWorkspace.tsx",
|
||||
]) {
|
||||
const source = readSource(path);
|
||||
expect(source, path).toContain("getCanvasImageDownloadErrorMessage(error)");
|
||||
}
|
||||
|
||||
const imageNodeSource = readSource("components/canvas/nodes/ImageNode.tsx");
|
||||
expect(imageNodeSource).not.toContain(
|
||||
"`" + ["$", "{error.message}"].join("") + "，源图可能已失效"
|
||||
);
|
||||
});
|
||||
});
|
||||
@ -8,6 +8,10 @@ import {
|
||||
type LayoutReplaceSource,
|
||||
mergeUniqueLayoutReplaceSources,
|
||||
} from "@/lib/canvas/layout-replace-source-selection";
|
||||
import {
|
||||
buildCanvasLayoutReplacePrompt,
|
||||
rankCanvasLayoutReplaceModels,
|
||||
} from "@/lib/canvas/layout-replace-model-selection";
|
||||
import { useCanvasStore } from "@/store/canvas";
|
||||
import type { CanvasNode } from "@/types/canvas";
|
||||
|
||||
@ -81,11 +85,97 @@ test.describe("canvas AI layout replacement entry", () => {
|
||||
expect(taskQueue).toContain("ensureCanvasImageTaskActive");
|
||||
expect(taskExecutor).toContain("saveLayoutReplaceResultToGallery");
|
||||
expect(taskExecutor).toContain('sourceType: "canvas_layout_replace"');
|
||||
expect(taskExecutor).toContain(
|
||||
'layoutReplaceServerAlgorithm: "rigid-product-scene-replace/v2"'
|
||||
);
|
||||
expect(taskExecutor).toContain("allowFailover: false");
|
||||
expect(taskExecutor).not.toContain("buildLayoutLockedCompositeImage({");
|
||||
expect(taskExecutor).not.toContain("analyzeReplacementExecutionPlan({");
|
||||
|
||||
expect(migration).toContain('CREATE TABLE IF NOT EXISTS "CanvasLayoutReplaceBatch"');
|
||||
expect(migration).toContain('CREATE TABLE IF NOT EXISTS "CanvasLayoutReplaceBatchItem"');
|
||||
});
|
||||
|
||||
test("always excludes FAL and prefers a healthy multi-image edit model", () => {
|
||||
const candidates = [
|
||||
{
|
||||
model: {
|
||||
id: "snapgen-gpt-image",
|
||||
modelId: "snapgen/gpt-image-2",
|
||||
supportsImageGen: true,
|
||||
supportsVision: true,
|
||||
healthStatus: "healthy",
|
||||
capabilities: { maxReferenceImages: 8 },
|
||||
},
|
||||
provider: { type: "snapgen-image" },
|
||||
},
|
||||
{
|
||||
model: {
|
||||
id: "fal-default",
|
||||
modelId: "fal-ai/nano-banana-2",
|
||||
supportsImageGen: true,
|
||||
supportsVision: true,
|
||||
healthStatus: "healthy",
|
||||
isDefault: true,
|
||||
},
|
||||
provider: { type: "fal-image" },
|
||||
},
|
||||
{
|
||||
model: {
|
||||
id: "seedream-edit",
|
||||
modelId: "bytedance/seedream-v5.0-pro/edit",
|
||||
supportsImageGen: true,
|
||||
supportsVision: true,
|
||||
healthStatus: "healthy",
|
||||
capabilities: { maxReferenceImages: 8 },
|
||||
},
|
||||
provider: { type: "vip98k" },
|
||||
},
|
||||
{
|
||||
model: {
|
||||
id: "gpt-image-edit",
|
||||
modelId: "openai/gpt-image-2/edit",
|
||||
supportsImageGen: true,
|
||||
supportsVision: true,
|
||||
healthStatus: "healthy",
|
||||
capabilities: { maxReferenceImages: 10 },
|
||||
},
|
||||
provider: { type: "atlascloud" },
|
||||
},
|
||||
];
|
||||
|
||||
const ranked = rankCanvasLayoutReplaceModels(candidates, "fal-default");
|
||||
expect(ranked.map((item) => item.model.id)).toEqual([
|
||||
"snapgen-gpt-image",
|
||||
"gpt-image-edit",
|
||||
"seedream-edit",
|
||||
]);
|
||||
expect(ranked.every((item) => item.provider.type !== "fal-image")).toBe(true);
|
||||
});
|
||||
|
||||
test("uses a rigid product identity prompt instead of detail-page copy rules", () => {
|
||||
const prompt = buildCanvasLayoutReplacePrompt({
|
||||
referenceHeight: 726,
|
||||
referenceWidth: 543,
|
||||
sourceProductAspectRatio: 0.76,
|
||||
supplementalPrompt: "保留深胡桃木颜色",
|
||||
});
|
||||
|
||||
expect(prompt).toContain("图1是新商品身份图");
|
||||
expect(prompt).toContain("图2是目标场景与构图模版");
|
||||
expect(prompt).toContain("真实商品像素层");
|
||||
expect(prompt).toContain("横向或纵向拉伸");
|
||||
expect(prompt).toContain("543x726");
|
||||
expect(prompt).toContain("0.760:1");
|
||||
expect(prompt).toContain("绝对不能压窄商品宽度");
|
||||
expect(prompt).toContain("画布宽度约 77%");
|
||||
expect(prompt).toContain("画布 x≈11% 处");
|
||||
expect(prompt).toContain("x≈89% 处");
|
||||
expect(prompt).toContain("图2旧商品原本偏窄的占位不得继承");
|
||||
expect(prompt).toContain("保留深胡桃木颜色");
|
||||
expect(prompt).not.toContain("按钮、标签、分割线、品牌区");
|
||||
});
|
||||
|
||||
test("selects eligible canvas images without exceeding the remaining slots", () => {
|
||||
const store = useCanvasStore.getState();
|
||||
store.endCanvasImageSelection("layout-owner");
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user