4336 lines
142 KiB
TypeScript
4336 lines
142 KiB
TypeScript
import "server-only";
|
||
|
||
import sharp from "sharp";
|
||
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
|
||
import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
|
||
import { createBflImage } from "@/lib/ai/providers/bfl-image";
|
||
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
|
||
import { createFalImage } from "@/lib/ai/providers/fal-image";
|
||
import { createSnapgenImage } from "@/lib/ai/providers/snapgen-image";
|
||
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
|
||
import { createStepfunImage } from "@/lib/ai/providers/stepfun-image";
|
||
import { createMidjourney } from "@/lib/ai/providers/midjourney";
|
||
import { createRunway } from "@/lib/ai/providers/runway";
|
||
import { createKrea } from "@/lib/ai/providers/krea";
|
||
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
|
||
import { resolveModelRuntimeId } from "@/lib/ai/model-runtime";
|
||
import { modelSupportsTextToImageDescriptor } from "@/lib/ai/model-classification";
|
||
import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-capability";
|
||
import { filterReachableRemoteUrls } from "@/lib/net/url-access";
|
||
import {
|
||
acquireCanvasImageModelSlot,
|
||
getCanvasImageModelQueueStatus,
|
||
type CanvasImageModelQueueStatus,
|
||
normalizeCanvasImageModelKey,
|
||
recordCanvasImageModelExecutionDuration,
|
||
} from "./image-concurrency";
|
||
import {
|
||
classifyAiModelHealthFailure,
|
||
recordAiModelHealthFailure,
|
||
recordAiModelHealthSuccess,
|
||
shouldFailoverAiModelError,
|
||
} from "@/lib/ai/runtime/model-health";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
|
||
export interface CanvasLocalEditCompositeRegion {
|
||
x: number;
|
||
y: number;
|
||
width: number;
|
||
height: number;
|
||
}
|
||
|
||
export interface CanvasLocalEditCompositeConfig {
|
||
sourceImageUrl: string;
|
||
regions: CanvasLocalEditCompositeRegion[];
|
||
expandRegions?: boolean;
|
||
maskMode?: "region" | "diff";
|
||
featherMinPx?: number;
|
||
featherMaxPx?: number;
|
||
featherRatio?: number;
|
||
diffLowThreshold?: number;
|
||
diffHighThreshold?: number;
|
||
diffCompareBlurPx?: number;
|
||
}
|
||
|
||
export interface CanvasScaleAdjustPatchCompositeConfig {
|
||
clipMaskImageUrl?: string;
|
||
sourceImageUrl: string;
|
||
workspaceRegion: CanvasLocalEditCompositeRegion;
|
||
regions?: CanvasLocalEditCompositeRegion[];
|
||
maskMode?: "workspace" | "diff";
|
||
diffAlphaCutoff?: number;
|
||
diffAlphaMultiplier?: number;
|
||
diffLowThreshold?: number;
|
||
diffHighThreshold?: number;
|
||
diffCompareBlurPx?: number;
|
||
featherMinPx?: number;
|
||
featherMaxPx?: number;
|
||
featherRatio?: number;
|
||
}
|
||
|
||
export interface CanvasScaleAdjustVerificationConfig {
|
||
allowDeterministicFallback?: boolean;
|
||
allowOutsideChangedWhenChangedInside?: boolean;
|
||
protectedRegions?: CanvasLocalEditCompositeRegion[];
|
||
rejectFailedCandidates?: boolean;
|
||
regionPrompts?: string[];
|
||
sourceImageUrl: string;
|
||
regions: CanvasLocalEditCompositeRegion[];
|
||
targetRegions?: CanvasLocalEditCompositeRegion[];
|
||
scalePercent?: number;
|
||
}
|
||
|
||
export interface CanvasGenerateImageRetryVariant {
|
||
allowFailover?: boolean;
|
||
debugMeta?: Record<string, unknown>;
|
||
falSafetyTolerance?: string | number;
|
||
falSystemPrompt?: string;
|
||
falThinkingLevel?: "minimal" | "high";
|
||
imageCount?: number;
|
||
label?: string;
|
||
model?: string;
|
||
negativePrompt?: string;
|
||
prompt?: string;
|
||
promptExtend?: boolean;
|
||
providerImageParams?: Record<string, unknown>;
|
||
referenceImage?: string;
|
||
referenceImages?: string[];
|
||
scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig;
|
||
scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig;
|
||
}
|
||
|
||
export interface CanvasGenerateImageRequest {
|
||
prompt: string;
|
||
displayPrompt?: string;
|
||
negativePrompt?: string;
|
||
model?: string;
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
falSafetyTolerance?: string | number;
|
||
falSystemPrompt?: string;
|
||
falThinkingLevel?: "minimal" | "high";
|
||
promptExtend?: boolean;
|
||
providerImageParams?: Record<string, unknown>;
|
||
referenceImage?: string;
|
||
referenceImages?: string[];
|
||
billingFeature?: "multi-view" | "repaint";
|
||
ownerKey?: string;
|
||
debugLabel?: string;
|
||
debugMeta?: Record<string, unknown>;
|
||
localEditComposite?: CanvasLocalEditCompositeConfig;
|
||
scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig;
|
||
scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig;
|
||
retryVariants?: CanvasGenerateImageRetryVariant[];
|
||
internalRetryVariantIndex?: number;
|
||
allowFailover?: boolean;
|
||
internalFailoverTriedModelRefs?: string[];
|
||
internalFailoverTriedProviderTypes?: string[];
|
||
onQueueStatusChange?: (
|
||
status: CanvasImageModelQueueStatus & {
|
||
modelName?: string | null;
|
||
modelRef?: string | null;
|
||
}
|
||
) => void | Promise<void>;
|
||
}
|
||
|
||
export interface ParsedImage {
|
||
thumbnailUrl: string;
|
||
originalUrl: string;
|
||
}
|
||
|
||
export interface CanvasGenerateImageResult {
|
||
images: ParsedImage[];
|
||
model: string;
|
||
modelRef: string;
|
||
prompt: string;
|
||
}
|
||
|
||
class ScaleAdjustOutputRejectedError extends Error {
|
||
reason: string;
|
||
diffScores?: number[];
|
||
images?: ParsedImage[];
|
||
outsideDiffScores?: number[];
|
||
|
||
constructor(params: {
|
||
reason: string;
|
||
diffScores?: number[];
|
||
images?: ParsedImage[];
|
||
outsideDiffScores?: number[];
|
||
}) {
|
||
const message =
|
||
params.reason === "outside-frame-changed"
|
||
? "比例调整结果改变了框选区域外的画面，已拒绝该模型结果"
|
||
: params.reason === "rectangular-seam-detected"
|
||
? "比例调整结果出现明显矩形拼接硬边，已拒绝该模型结果"
|
||
: "比例调整结果没有产生有效变化，已拒绝该模型结果";
|
||
super(message);
|
||
this.name = "ScaleAdjustOutputRejectedError";
|
||
this.reason = params.reason;
|
||
this.diffScores = params.diffScores;
|
||
this.images = params.images;
|
||
this.outsideDiffScores = params.outsideDiffScores;
|
||
}
|
||
}
|
||
|
||
const LOCAL_EDIT_COMPOSITE_PADDING_RATIO = 0.14;
|
||
const LOCAL_EDIT_COMPOSITE_MIN_PADDING_RATIO = 0.012;
|
||
const LOCAL_EDIT_COMPOSITE_MAX_PADDING_RATIO = 0.06;
|
||
const LOCAL_EDIT_COMPOSITE_FEATHER_MIN_PX = 18;
|
||
const LOCAL_EDIT_COMPOSITE_FEATHER_MAX_PX = 72;
|
||
const LOCAL_EDIT_COMPOSITE_FEATHER_RATIO = 0.018;
|
||
const LOCAL_EDIT_COMPOSITE_DIFF_LOW_THRESHOLD = 10;
|
||
const LOCAL_EDIT_COMPOSITE_DIFF_HIGH_THRESHOLD = 34;
|
||
const SCALE_ADJUST_VISUAL_DIFF_THRESHOLD = 10;
|
||
const SCALE_ADJUST_OUTSIDE_DIFF_THRESHOLD = 18;
|
||
const SCALE_ADJUST_OUTSIDE_DIFF_MIN_CHANGED_RATIO = 0.28;
|
||
const SCALE_ADJUST_OUTSIDE_PIXEL_DIFF_THRESHOLD = 16;
|
||
const SCALE_ADJUST_OUTSIDE_PIXEL_CHANGED_RATIO_THRESHOLD = 0.16;
|
||
const SCALE_ADJUST_OUTSIDE_PIXEL_MEAN_DIFF_THRESHOLD = 14;
|
||
const SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN = 0.015;
|
||
const SCALE_ADJUST_MARKER_CLEANUP_MARGIN = 0.035;
|
||
const SCALE_ADJUST_MARKER_DILATE_RADIUS = 3;
|
||
const SCALE_ADJUST_MARKER_MIN_PIXEL_COUNT = 18;
|
||
const SCALE_ADJUST_DIFF_SAMPLE_SIZE = 112;
|
||
const SCALE_ADJUST_FOOTPRINT_FALLBACK_DIFF_THRESHOLD = 18;
|
||
const SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD = 42;
|
||
const SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP = 18;
|
||
const SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS = 180;
|
||
const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MAX_RATIO = 0.72;
|
||
const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_RELAXED_MAX_RATIO = 0.84;
|
||
const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MIN_REMOVED_RATIO = 0.14;
|
||
const SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS = 60;
|
||
const SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN = 0.018;
|
||
const SCALE_ADJUST_RECTANGULAR_SEAM_CHANGE_THRESHOLD = 18;
|
||
const SCALE_ADJUST_RECTANGULAR_SEAM_MIN_EDGE_DIFF = 5;
|
||
const SCALE_ADJUST_RECTANGULAR_SEAM_MAX_EDGE_DIFF = 54;
|
||
const SCALE_ADJUST_RECTANGULAR_SEAM_EDGE_INCREASE = 2.5;
|
||
const SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_RUN_RATIO = 0.11;
|
||
const SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_TRANSITION_RATIO = 0.18;
|
||
const SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO = 0.1;
|
||
const SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO = 0.26;
|
||
const SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_RUN_RATIO = 0.55;
|
||
const SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_TRANSITION_RATIO = 0.5;
|
||
|
||
const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
|
||
"gemini",
|
||
"grsai-gpt-image",
|
||
"alibaba-image",
|
||
"bfl-image",
|
||
"ruxa-image",
|
||
"fal-image",
|
||
"snapgen-image",
|
||
"openai-image",
|
||
"stepfun-image",
|
||
"midjourney",
|
||
"runway",
|
||
"krea",
|
||
"volcano-vision",
|
||
]);
|
||
|
||
function summarizeReferenceImageHosts(urls: string[]) {
|
||
return Array.from(
|
||
new Set(
|
||
urls
|
||
.map((value) => {
|
||
try {
|
||
return new URL(value).host;
|
||
} catch {
|
||
return "";
|
||
}
|
||
})
|
||
.filter(Boolean)
|
||
)
|
||
);
|
||
}
|
||
|
||
function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
|
||
const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
|
||
const matches = [...markdown.matchAll(imageRegex)];
|
||
const parsed: ParsedImage[] = [];
|
||
|
||
for (const match of matches) {
|
||
const altText = match[1] || "";
|
||
const thumbnailUrl = match[2];
|
||
let originalUrl = thumbnailUrl;
|
||
|
||
if (altText.includes("thumb|||")) {
|
||
const content = altText.substring(altText.indexOf("thumb|||") + 8);
|
||
const parts = content.split("|||");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
} else if (altText.includes("thumb$$")) {
|
||
const content = altText.substring(altText.indexOf("thumb$$") + 7);
|
||
const parts = content.split("$$");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
} else if (altText.includes("thumb|")) {
|
||
const content = altText.substring(altText.indexOf("thumb|") + 6);
|
||
const parts = content.split("|");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
}
|
||
|
||
parsed.push({ thumbnailUrl, originalUrl });
|
||
}
|
||
|
||
return parsed;
|
||
}
|
||
|
||
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||
return buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
}
|
||
|
||
function clampRatio(value: number) {
|
||
if (!Number.isFinite(value)) return 0;
|
||
return Math.min(1, Math.max(0, value));
|
||
}
|
||
|
||
function expandLocalEditCompositeRegion(region: CanvasLocalEditCompositeRegion) {
|
||
const padX = clampRatio(
|
||
Math.min(
|
||
LOCAL_EDIT_COMPOSITE_MAX_PADDING_RATIO,
|
||
Math.max(
|
||
LOCAL_EDIT_COMPOSITE_MIN_PADDING_RATIO,
|
||
region.width * LOCAL_EDIT_COMPOSITE_PADDING_RATIO
|
||
)
|
||
)
|
||
);
|
||
const padY = clampRatio(
|
||
Math.min(
|
||
LOCAL_EDIT_COMPOSITE_MAX_PADDING_RATIO,
|
||
Math.max(
|
||
LOCAL_EDIT_COMPOSITE_MIN_PADDING_RATIO,
|
||
region.height * LOCAL_EDIT_COMPOSITE_PADDING_RATIO
|
||
)
|
||
)
|
||
);
|
||
const left = clampRatio(region.x - padX);
|
||
const top = clampRatio(region.y - padY);
|
||
const right = clampRatio(region.x + region.width + padX);
|
||
const bottom = clampRatio(region.y + region.height + padY);
|
||
|
||
return {
|
||
x: left,
|
||
y: top,
|
||
width: Math.max(0, right - left),
|
||
height: Math.max(0, bottom - top),
|
||
};
|
||
}
|
||
|
||
function normalizeOptionalNumber(
|
||
value: unknown,
|
||
options: { min: number; max: number }
|
||
) {
|
||
const numeric = Number(value);
|
||
if (!Number.isFinite(numeric)) return undefined;
|
||
return Math.max(options.min, Math.min(options.max, numeric));
|
||
}
|
||
|
||
function normalizeLocalEditCompositeConfig(
|
||
value: CanvasGenerateImageRequest["localEditComposite"]
|
||
): CanvasLocalEditCompositeConfig | null {
|
||
if (!value || typeof value !== "object") return null;
|
||
const sourceImageUrl = String(value.sourceImageUrl || "").trim();
|
||
if (!/^https?:\/\//i.test(sourceImageUrl)) return null;
|
||
if (!Array.isArray(value.regions) || value.regions.length === 0) return null;
|
||
const expandRegions = value.expandRegions !== false;
|
||
const maskMode = value.maskMode === "diff" ? "diff" : "region";
|
||
|
||
const regions = value.regions
|
||
.map((region) => {
|
||
const x = clampRatio(Number(region?.x));
|
||
const y = clampRatio(Number(region?.y));
|
||
const width = clampRatio(Number(region?.width));
|
||
const height = clampRatio(Number(region?.height));
|
||
return {
|
||
x,
|
||
y,
|
||
width: Math.min(width, Math.max(0, 1 - x)),
|
||
height: Math.min(height, Math.max(0, 1 - y)),
|
||
};
|
||
})
|
||
.map((region) => expandRegions ? expandLocalEditCompositeRegion(region) : region)
|
||
.filter((region) => region.width > 0 && region.height > 0);
|
||
|
||
return regions.length > 0
|
||
? {
|
||
sourceImageUrl,
|
||
regions,
|
||
expandRegions,
|
||
maskMode,
|
||
diffHighThreshold: normalizeOptionalNumber(value.diffHighThreshold, {
|
||
max: 128,
|
||
min: 1,
|
||
}),
|
||
diffLowThreshold: normalizeOptionalNumber(value.diffLowThreshold, {
|
||
max: 127,
|
||
min: 0,
|
||
}),
|
||
diffCompareBlurPx: normalizeOptionalNumber(value.diffCompareBlurPx, {
|
||
max: 12,
|
||
min: 0,
|
||
}),
|
||
featherMaxPx: normalizeOptionalNumber(value.featherMaxPx, {
|
||
max: 160,
|
||
min: 0,
|
||
}),
|
||
featherMinPx: normalizeOptionalNumber(value.featherMinPx, {
|
||
max: 80,
|
||
min: 0,
|
||
}),
|
||
featherRatio: normalizeOptionalNumber(value.featherRatio, {
|
||
max: 0.08,
|
||
min: 0,
|
||
}),
|
||
}
|
||
: null;
|
||
}
|
||
|
||
function normalizeScaleAdjustPatchCompositeConfig(
|
||
value: CanvasGenerateImageRequest["scaleAdjustPatchComposite"]
|
||
): CanvasScaleAdjustPatchCompositeConfig | null {
|
||
if (!value || typeof value !== "object") return null;
|
||
const sourceImageUrl = String(value.sourceImageUrl || "").trim();
|
||
if (!/^https?:\/\//i.test(sourceImageUrl)) return null;
|
||
const workspaceRegion = normalizeCanvasRegion(value.workspaceRegion);
|
||
if (!workspaceRegion) return null;
|
||
const regions = Array.isArray(value.regions)
|
||
? value.regions
|
||
.map((region) => normalizeCanvasRegion(region))
|
||
.filter((region): region is CanvasLocalEditCompositeRegion =>
|
||
Boolean(region)
|
||
)
|
||
: [];
|
||
|
||
return {
|
||
...(/^https?:\/\//i.test(String(value.clipMaskImageUrl || "").trim())
|
||
? { clipMaskImageUrl: String(value.clipMaskImageUrl || "").trim() }
|
||
: {}),
|
||
sourceImageUrl,
|
||
workspaceRegion,
|
||
...(regions.length > 0 ? { regions } : {}),
|
||
maskMode: value.maskMode === "diff" ? "diff" : "workspace",
|
||
diffAlphaCutoff: normalizeOptionalNumber(value.diffAlphaCutoff, {
|
||
max: 96,
|
||
min: 0,
|
||
}),
|
||
diffAlphaMultiplier: normalizeOptionalNumber(value.diffAlphaMultiplier, {
|
||
max: 6,
|
||
min: 0.5,
|
||
}),
|
||
diffHighThreshold: normalizeOptionalNumber(value.diffHighThreshold, {
|
||
max: 120,
|
||
min: 1,
|
||
}),
|
||
diffLowThreshold: normalizeOptionalNumber(value.diffLowThreshold, {
|
||
max: 80,
|
||
min: 0,
|
||
}),
|
||
diffCompareBlurPx: normalizeOptionalNumber(value.diffCompareBlurPx, {
|
||
max: 12,
|
||
min: 0,
|
||
}),
|
||
featherMaxPx: normalizeOptionalNumber(value.featherMaxPx, {
|
||
max: 180,
|
||
min: 0,
|
||
}),
|
||
featherMinPx: normalizeOptionalNumber(value.featherMinPx, {
|
||
max: 100,
|
||
min: 0,
|
||
}),
|
||
featherRatio: normalizeOptionalNumber(value.featherRatio, {
|
||
max: 0.12,
|
||
min: 0,
|
||
}),
|
||
};
|
||
}
|
||
|
||
function normalizeCanvasRegion(region: unknown): CanvasLocalEditCompositeRegion | null {
|
||
if (!region || typeof region !== "object") return null;
|
||
const raw = region as Partial<CanvasLocalEditCompositeRegion>;
|
||
const x = clampRatio(Number(raw.x));
|
||
const y = clampRatio(Number(raw.y));
|
||
const width = clampRatio(Number(raw.width));
|
||
const height = clampRatio(Number(raw.height));
|
||
const normalized = {
|
||
x,
|
||
y,
|
||
width: Math.min(width, Math.max(0, 1 - x)),
|
||
height: Math.min(height, Math.max(0, 1 - y)),
|
||
};
|
||
return normalized.width > 0 && normalized.height > 0 ? normalized : null;
|
||
}
|
||
|
||
function normalizeScaleAdjustVerificationConfig(
|
||
value: CanvasGenerateImageRequest["scaleAdjustVerification"]
|
||
): CanvasScaleAdjustVerificationConfig | null {
|
||
if (!value || typeof value !== "object") return null;
|
||
const sourceImageUrl = String(value.sourceImageUrl || "").trim();
|
||
if (!/^https?:\/\//i.test(sourceImageUrl)) return null;
|
||
if (!Array.isArray(value.regions) || value.regions.length === 0) return null;
|
||
const rawScalePercent = Number(value.scalePercent);
|
||
const scalePercent =
|
||
Number.isFinite(rawScalePercent) && Math.abs(rawScalePercent) >= 1
|
||
? rawScalePercent
|
||
: undefined;
|
||
|
||
const regions = value.regions
|
||
.map((region) => normalizeCanvasRegion(region))
|
||
.filter((region): region is CanvasLocalEditCompositeRegion => Boolean(region));
|
||
const targetRegions = Array.isArray((value as { targetRegions?: unknown[] }).targetRegions)
|
||
? (value as { targetRegions?: unknown[] }).targetRegions
|
||
?.map((region) => normalizeCanvasRegion(region))
|
||
.filter((region): region is CanvasLocalEditCompositeRegion =>
|
||
Boolean(region)
|
||
) || []
|
||
: [];
|
||
const protectedRegions = Array.isArray(value.protectedRegions)
|
||
? value.protectedRegions
|
||
.map((region) => normalizeCanvasRegion(region))
|
||
.filter((region): region is CanvasLocalEditCompositeRegion =>
|
||
Boolean(region)
|
||
)
|
||
: [];
|
||
const regionPrompts = Array.isArray(value.regionPrompts)
|
||
? value.regionPrompts
|
||
.map((prompt) => String(prompt || "").trim())
|
||
.slice(0, regions.length)
|
||
: [];
|
||
|
||
return regions.length > 0 &&
|
||
(targetRegions.length > 0 || typeof scalePercent === "number")
|
||
? {
|
||
sourceImageUrl,
|
||
regions,
|
||
allowDeterministicFallback: value.allowDeterministicFallback === true,
|
||
allowOutsideChangedWhenChangedInside:
|
||
value.allowOutsideChangedWhenChangedInside === true,
|
||
rejectFailedCandidates: value.rejectFailedCandidates === true,
|
||
...(regionPrompts.length > 0 ? { regionPrompts } : {}),
|
||
...(targetRegions.length > 0 ? { targetRegions } : {}),
|
||
...(typeof scalePercent === "number" ? { scalePercent } : {}),
|
||
...(protectedRegions.length > 0 ? { protectedRegions } : {}),
|
||
}
|
||
: null;
|
||
}
|
||
|
||
function scaleRegionAroundCenter(
|
||
region: CanvasLocalEditCompositeRegion,
|
||
scalePercent: number
|
||
): CanvasLocalEditCompositeRegion {
|
||
const requestedFactor = Math.max(0.05, 1 + scalePercent / 100);
|
||
const maxUniformFactor = Math.max(
|
||
0.05,
|
||
Math.min(
|
||
region.width > 0 ? 1 / region.width : requestedFactor,
|
||
region.height > 0 ? 1 / region.height : requestedFactor
|
||
)
|
||
);
|
||
const factor = Math.min(requestedFactor, maxUniformFactor);
|
||
let width = clampRatio(region.width * factor);
|
||
let height = clampRatio(region.height * factor);
|
||
const centerX = region.x + region.width / 2;
|
||
const centerY = region.y + region.height / 2;
|
||
let x = centerX - width / 2;
|
||
let y = centerY - height / 2;
|
||
|
||
if (x < 0) x = 0;
|
||
if (y < 0) y = 0;
|
||
if (x + width > 1) x = 1 - width;
|
||
if (y + height > 1) y = 1 - height;
|
||
|
||
x = clampRatio(x);
|
||
y = clampRatio(y);
|
||
width = Math.min(width, Math.max(0, 1 - x));
|
||
height = Math.min(height, Math.max(0, 1 - y));
|
||
|
||
return { x, y, width, height };
|
||
}
|
||
|
||
function expandCanvasRegion(
|
||
region: CanvasLocalEditCompositeRegion,
|
||
margin: number
|
||
): CanvasLocalEditCompositeRegion {
|
||
const x = clampRatio(region.x - margin);
|
||
const y = clampRatio(region.y - margin);
|
||
const right = clampRatio(region.x + region.width + margin);
|
||
const bottom = clampRatio(region.y + region.height + margin);
|
||
return {
|
||
x,
|
||
y,
|
||
width: Math.max(0, right - x),
|
||
height: Math.max(0, bottom - y),
|
||
};
|
||
}
|
||
|
||
function canvasRegionsIntersect(
|
||
left: CanvasLocalEditCompositeRegion,
|
||
right: CanvasLocalEditCompositeRegion
|
||
) {
|
||
return !(
|
||
left.x + left.width <= right.x ||
|
||
right.x + right.width <= left.x ||
|
||
left.y + left.height <= right.y ||
|
||
right.y + right.height <= left.y
|
||
);
|
||
}
|
||
|
||
function getScaleAdjustProtectedRegions(
|
||
verification: CanvasScaleAdjustVerificationConfig
|
||
) {
|
||
if (verification.protectedRegions?.length) {
|
||
return verification.protectedRegions.map((region) =>
|
||
expandCanvasRegion(region, SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN)
|
||
);
|
||
}
|
||
if (verification.targetRegions?.length) {
|
||
return verification.regions.flatMap((region, index) => {
|
||
const targetRegion = verification.targetRegions?.[index];
|
||
return targetRegion
|
||
? [
|
||
expandCanvasRegion(region, SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN),
|
||
expandCanvasRegion(
|
||
targetRegion,
|
||
SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN
|
||
),
|
||
]
|
||
: [expandCanvasRegion(region, SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN)];
|
||
});
|
||
}
|
||
if (typeof verification.scalePercent === "number") {
|
||
return verification.regions.flatMap((region) => [
|
||
expandCanvasRegion(region, SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN),
|
||
expandCanvasRegion(
|
||
scaleRegionAroundCenter(region, verification.scalePercent as number),
|
||
SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN
|
||
),
|
||
]);
|
||
}
|
||
return verification.regions.map((region) =>
|
||
expandCanvasRegion(region, SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN)
|
||
);
|
||
}
|
||
|
||
function getScaleAdjustOutsideSampleRegions(
|
||
verification: CanvasScaleAdjustVerificationConfig
|
||
) {
|
||
const protectedRegions = getScaleAdjustProtectedRegions(verification);
|
||
const fixedCandidates: CanvasLocalEditCompositeRegion[] = [
|
||
{ x: 0.03, y: 0.03, width: 0.2, height: 0.16 },
|
||
{ x: 0.4, y: 0.03, width: 0.2, height: 0.16 },
|
||
{ x: 0.77, y: 0.03, width: 0.2, height: 0.16 },
|
||
{ x: 0.03, y: 0.28, width: 0.2, height: 0.16 },
|
||
{ x: 0.77, y: 0.28, width: 0.2, height: 0.16 },
|
||
{ x: 0.03, y: 0.55, width: 0.2, height: 0.16 },
|
||
{ x: 0.77, y: 0.55, width: 0.2, height: 0.16 },
|
||
{ x: 0.03, y: 0.81, width: 0.2, height: 0.16 },
|
||
{ x: 0.4, y: 0.81, width: 0.2, height: 0.16 },
|
||
{ x: 0.77, y: 0.81, width: 0.2, height: 0.16 },
|
||
{ x: 0.28, y: 0.28, width: 0.18, height: 0.16 },
|
||
{ x: 0.54, y: 0.28, width: 0.18, height: 0.16 },
|
||
{ x: 0.28, y: 0.55, width: 0.18, height: 0.16 },
|
||
{ x: 0.54, y: 0.55, width: 0.18, height: 0.16 },
|
||
];
|
||
const gridCandidates: CanvasLocalEditCompositeRegion[] = [];
|
||
const columns = 7;
|
||
const rows = 7;
|
||
const cellWidth = 0.12;
|
||
const cellHeight = 0.1;
|
||
|
||
for (let row = 0; row < rows; row += 1) {
|
||
for (let column = 0; column < columns; column += 1) {
|
||
gridCandidates.push({
|
||
x: (column / Math.max(1, columns - 1)) * (1 - cellWidth),
|
||
y: (row / Math.max(1, rows - 1)) * (1 - cellHeight),
|
||
width: cellWidth,
|
||
height: cellHeight,
|
||
});
|
||
}
|
||
}
|
||
const candidates = [...fixedCandidates, ...gridCandidates];
|
||
const uniqueCandidates = candidates.filter((candidate, index) => {
|
||
const key = `${candidate.x.toFixed(3)}:${candidate.y.toFixed(3)}:${candidate.width.toFixed(3)}:${candidate.height.toFixed(3)}`;
|
||
return (
|
||
candidates.findIndex(
|
||
(item) =>
|
||
`${item.x.toFixed(3)}:${item.y.toFixed(3)}:${item.width.toFixed(3)}:${item.height.toFixed(3)}` ===
|
||
key
|
||
) === index
|
||
);
|
||
});
|
||
|
||
return uniqueCandidates
|
||
.filter(
|
||
(candidate) =>
|
||
!protectedRegions.some((protectedRegion) =>
|
||
canvasRegionsIntersect(candidate, protectedRegion)
|
||
)
|
||
)
|
||
.slice(0, 28);
|
||
}
|
||
|
||
function rectToPixelBox(params: {
|
||
region: CanvasLocalEditCompositeRegion;
|
||
width: number;
|
||
height: number;
|
||
}) {
|
||
const { region, width, height } = params;
|
||
const left = Math.max(
|
||
0,
|
||
Math.min(width - 1, Math.round(region.x * width))
|
||
);
|
||
const top = Math.max(
|
||
0,
|
||
Math.min(height - 1, Math.round(region.y * height))
|
||
);
|
||
const right = Math.max(
|
||
left + 1,
|
||
Math.min(width, Math.round((region.x + region.width) * width))
|
||
);
|
||
const bottom = Math.max(
|
||
top + 1,
|
||
Math.min(height, Math.round((region.y + region.height) * height))
|
||
);
|
||
|
||
return {
|
||
left,
|
||
top,
|
||
width: Math.max(1, right - left),
|
||
height: Math.max(1, bottom - top),
|
||
};
|
||
}
|
||
|
||
async function getScaleAdjustRegionSample(params: {
|
||
buffer: Buffer;
|
||
sourceWidth: number;
|
||
sourceHeight: number;
|
||
region: CanvasLocalEditCompositeRegion;
|
||
}) {
|
||
const normalizedBuffer = await sharp(params.buffer, { failOn: "none" })
|
||
.rotate()
|
||
.resize(params.sourceWidth, params.sourceHeight, {
|
||
fit: "cover",
|
||
position: "center",
|
||
})
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const box = rectToPixelBox({
|
||
region: params.region,
|
||
width: params.sourceWidth,
|
||
height: params.sourceHeight,
|
||
});
|
||
|
||
return sharp(normalizedBuffer, { failOn: "none" })
|
||
.extract(box)
|
||
.resize(SCALE_ADJUST_DIFF_SAMPLE_SIZE, SCALE_ADJUST_DIFF_SAMPLE_SIZE, {
|
||
fit: "fill",
|
||
})
|
||
.removeAlpha()
|
||
.toColorspace("srgb")
|
||
.raw()
|
||
.toBuffer();
|
||
}
|
||
|
||
function calculateMeanAbsoluteDiff(left: Buffer, right: Buffer) {
|
||
const length = Math.min(left.length, right.length);
|
||
if (length <= 0) return Number.POSITIVE_INFINITY;
|
||
let sum = 0;
|
||
for (let index = 0; index < length; index += 1) {
|
||
sum += Math.abs(left[index] - right[index]);
|
||
}
|
||
return sum / length;
|
||
}
|
||
|
||
async function getScaleAdjustDiffScores(params: {
|
||
generatedBuffer: Buffer;
|
||
sourceBuffer: Buffer;
|
||
sourceWidth: number;
|
||
sourceHeight: number;
|
||
regions: CanvasLocalEditCompositeRegion[];
|
||
}) {
|
||
const scores: number[] = [];
|
||
for (const region of params.regions) {
|
||
const [sourceSample, generatedSample] = await Promise.all([
|
||
getScaleAdjustRegionSample({
|
||
buffer: params.sourceBuffer,
|
||
sourceWidth: params.sourceWidth,
|
||
sourceHeight: params.sourceHeight,
|
||
region,
|
||
}),
|
||
getScaleAdjustRegionSample({
|
||
buffer: params.generatedBuffer,
|
||
sourceWidth: params.sourceWidth,
|
||
sourceHeight: params.sourceHeight,
|
||
region,
|
||
}),
|
||
]);
|
||
scores.push(calculateMeanAbsoluteDiff(sourceSample, generatedSample));
|
||
}
|
||
return scores;
|
||
}
|
||
|
||
function getScaleAdjustEffectiveChangeRegions(
|
||
verification: CanvasScaleAdjustVerificationConfig
|
||
) {
|
||
const regions: CanvasLocalEditCompositeRegion[] = [];
|
||
for (const [index, region] of verification.regions.entries()) {
|
||
regions.push(region);
|
||
const targetRegion = verification.targetRegions?.[index];
|
||
if (targetRegion) {
|
||
regions.push(targetRegion);
|
||
} else if (typeof verification.scalePercent === "number") {
|
||
regions.push(scaleRegionAroundCenter(region, verification.scalePercent));
|
||
}
|
||
}
|
||
|
||
return regions.filter((region, index) => {
|
||
const key = `${region.x.toFixed(4)}:${region.y.toFixed(4)}:${region.width.toFixed(4)}:${region.height.toFixed(4)}`;
|
||
return (
|
||
regions.findIndex(
|
||
(item) =>
|
||
`${item.x.toFixed(4)}:${item.y.toFixed(4)}:${item.width.toFixed(4)}:${item.height.toFixed(4)}` ===
|
||
key
|
||
) === index
|
||
);
|
||
});
|
||
}
|
||
|
||
async function getScaleAdjustOutsideDiffScores(params: {
|
||
generatedBuffer: Buffer;
|
||
sourceBuffer: Buffer;
|
||
sourceWidth: number;
|
||
sourceHeight: number;
|
||
verification: CanvasScaleAdjustVerificationConfig;
|
||
}) {
|
||
const sampleRegions = getScaleAdjustOutsideSampleRegions(params.verification);
|
||
const scores: number[] = [];
|
||
|
||
for (const region of sampleRegions) {
|
||
const [sourceSample, generatedSample] = await Promise.all([
|
||
getScaleAdjustRegionSample({
|
||
buffer: params.sourceBuffer,
|
||
sourceWidth: params.sourceWidth,
|
||
sourceHeight: params.sourceHeight,
|
||
region,
|
||
}),
|
||
getScaleAdjustRegionSample({
|
||
buffer: params.generatedBuffer,
|
||
sourceWidth: params.sourceWidth,
|
||
sourceHeight: params.sourceHeight,
|
||
region,
|
||
}),
|
||
]);
|
||
scores.push(calculateMeanAbsoluteDiff(sourceSample, generatedSample));
|
||
}
|
||
|
||
return scores;
|
||
}
|
||
|
||
function isScaleAdjustOutsideChanged(scores: number[]) {
|
||
if (scores.length < 2) return false;
|
||
const changedCount = scores.filter(
|
||
(score) => score >= SCALE_ADJUST_OUTSIDE_DIFF_THRESHOLD
|
||
).length;
|
||
const requiredCount = Math.max(
|
||
3,
|
||
Math.ceil(scores.length * SCALE_ADJUST_OUTSIDE_DIFF_MIN_CHANGED_RATIO)
|
||
);
|
||
return changedCount >= requiredCount;
|
||
}
|
||
|
||
async function getScaleAdjustOutsidePixelChangeStats(params: {
|
||
generatedBuffer: Buffer;
|
||
sourceBuffer: Buffer;
|
||
sourceWidth: number;
|
||
sourceHeight: number;
|
||
verification: CanvasScaleAdjustVerificationConfig;
|
||
}) {
|
||
const { generatedBuffer, sourceBuffer, sourceHeight, sourceWidth, verification } =
|
||
params;
|
||
const protectedRegions = getScaleAdjustProtectedRegions(verification);
|
||
const normalizeToRaw = (buffer: Buffer) =>
|
||
sharp(buffer, { failOn: "none" })
|
||
.rotate()
|
||
.resize(sourceWidth, sourceHeight, {
|
||
fit: "cover",
|
||
position: "center",
|
||
})
|
||
.removeAlpha()
|
||
.toColorspace("srgb")
|
||
.raw()
|
||
.toBuffer();
|
||
const [sourceRaw, generatedRaw] = await Promise.all([
|
||
normalizeToRaw(sourceBuffer),
|
||
normalizeToRaw(generatedBuffer),
|
||
]);
|
||
const step = Math.max(2, Math.ceil(Math.max(sourceWidth, sourceHeight) / 1200));
|
||
let changedPixelCount = 0;
|
||
let sampledPixelCount = 0;
|
||
let diffSum = 0;
|
||
|
||
for (let y = 0; y < sourceHeight; y += step) {
|
||
const normalizedY = y / sourceHeight;
|
||
for (let x = 0; x < sourceWidth; x += step) {
|
||
const normalizedX = x / sourceWidth;
|
||
if (
|
||
protectedRegions.some(
|
||
(region) =>
|
||
normalizedX >= region.x &&
|
||
normalizedX <= region.x + region.width &&
|
||
normalizedY >= region.y &&
|
||
normalizedY <= region.y + region.height
|
||
)
|
||
) {
|
||
continue;
|
||
}
|
||
const rawIndex = (y * sourceWidth + x) * 3;
|
||
const diff =
|
||
Math.abs(generatedRaw[rawIndex] - sourceRaw[rawIndex]) * 0.299 +
|
||
Math.abs(generatedRaw[rawIndex + 1] - sourceRaw[rawIndex + 1]) * 0.587 +
|
||
Math.abs(generatedRaw[rawIndex + 2] - sourceRaw[rawIndex + 2]) * 0.114;
|
||
diffSum += diff;
|
||
sampledPixelCount += 1;
|
||
if (diff >= SCALE_ADJUST_OUTSIDE_PIXEL_DIFF_THRESHOLD) {
|
||
changedPixelCount += 1;
|
||
}
|
||
}
|
||
}
|
||
|
||
const changedRatio =
|
||
sampledPixelCount > 0 ? changedPixelCount / sampledPixelCount : 0;
|
||
const meanDiff = sampledPixelCount > 0 ? diffSum / sampledPixelCount : 0;
|
||
|
||
return {
|
||
changedPixelCount,
|
||
changedRatio,
|
||
meanDiff,
|
||
sampledPixelCount,
|
||
step,
|
||
};
|
||
}
|
||
|
||
function isScaleAdjustOutsidePixelChanged(params: {
|
||
changedRatio: number;
|
||
meanDiff: number;
|
||
sampledPixelCount: number;
|
||
}) {
|
||
if (params.sampledPixelCount < 100) return false;
|
||
return (
|
||
params.changedRatio >= SCALE_ADJUST_OUTSIDE_PIXEL_CHANGED_RATIO_THRESHOLD &&
|
||
params.meanDiff >= SCALE_ADJUST_OUTSIDE_PIXEL_MEAN_DIFF_THRESHOLD
|
||
);
|
||
}
|
||
|
||
function getScaleAdjustRawRgbDiff(raw: Buffer, leftIndex: number, rightIndex: number) {
|
||
return (
|
||
Math.abs(raw[leftIndex] - raw[rightIndex]) * 0.299 +
|
||
Math.abs(raw[leftIndex + 1] - raw[rightIndex + 1]) * 0.587 +
|
||
Math.abs(raw[leftIndex + 2] - raw[rightIndex + 2]) * 0.114
|
||
);
|
||
}
|
||
|
||
function getScaleAdjustPixelChanged(params: {
|
||
generatedRaw: Buffer;
|
||
index: number;
|
||
sourceRaw: Buffer;
|
||
}) {
|
||
return (
|
||
Math.abs(params.generatedRaw[params.index] - params.sourceRaw[params.index]) *
|
||
0.299 +
|
||
Math.abs(
|
||
params.generatedRaw[params.index + 1] -
|
||
params.sourceRaw[params.index + 1]
|
||
) *
|
||
0.587 +
|
||
Math.abs(
|
||
params.generatedRaw[params.index + 2] -
|
||
params.sourceRaw[params.index + 2]
|
||
) *
|
||
0.114
|
||
) >= SCALE_ADJUST_RECTANGULAR_SEAM_CHANGE_THRESHOLD;
|
||
}
|
||
|
||
function summarizeScaleAdjustSeamLine(params: {
|
||
axis: "horizontal" | "vertical";
|
||
boxIndex: number;
|
||
changeMask: Uint8Array;
|
||
generatedRaw: Buffer;
|
||
height: number;
|
||
position: number;
|
||
rangeEnd: number;
|
||
rangeStart: number;
|
||
sourceRaw: Buffer;
|
||
width: number;
|
||
}) {
|
||
const {
|
||
axis,
|
||
boxIndex,
|
||
changeMask,
|
||
generatedRaw,
|
||
height,
|
||
position,
|
||
rangeEnd,
|
||
rangeStart,
|
||
sourceRaw,
|
||
width,
|
||
} = params;
|
||
let transitionCount = 0;
|
||
let currentRun = 0;
|
||
let maxRun = 0;
|
||
let edgeDiffSum = 0;
|
||
const sampleCount = Math.max(1, rangeEnd - rangeStart);
|
||
|
||
for (let offset = rangeStart; offset < rangeEnd; offset += 1) {
|
||
const leftX = axis === "vertical" ? position - 1 : offset;
|
||
const leftY = axis === "vertical" ? offset : position - 1;
|
||
const rightX = axis === "vertical" ? position : offset;
|
||
const rightY = axis === "vertical" ? offset : position;
|
||
if (
|
||
leftX < 0 ||
|
||
rightX < 0 ||
|
||
leftY < 0 ||
|
||
rightY < 0 ||
|
||
leftX >= width ||
|
||
rightX >= width ||
|
||
leftY >= height ||
|
||
rightY >= height
|
||
) {
|
||
currentRun = 0;
|
||
continue;
|
||
}
|
||
|
||
const leftPixel = leftY * width + leftX;
|
||
const rightPixel = rightY * width + rightX;
|
||
const changedBoundary =
|
||
changeMask[leftPixel] !== changeMask[rightPixel];
|
||
const leftRawIndex = leftPixel * 3;
|
||
const rightRawIndex = rightPixel * 3;
|
||
const generatedEdgeDiff = getScaleAdjustRawRgbDiff(
|
||
generatedRaw,
|
||
leftRawIndex,
|
||
rightRawIndex
|
||
);
|
||
const sourceEdgeDiff = getScaleAdjustRawRgbDiff(
|
||
sourceRaw,
|
||
leftRawIndex,
|
||
rightRawIndex
|
||
);
|
||
const likelyBackgroundSeam =
|
||
generatedEdgeDiff >= SCALE_ADJUST_RECTANGULAR_SEAM_MIN_EDGE_DIFF &&
|
||
generatedEdgeDiff <= SCALE_ADJUST_RECTANGULAR_SEAM_MAX_EDGE_DIFF &&
|
||
generatedEdgeDiff - sourceEdgeDiff >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_EDGE_INCREASE;
|
||
|
||
if (changedBoundary && likelyBackgroundSeam) {
|
||
transitionCount += 1;
|
||
currentRun += 1;
|
||
maxRun = Math.max(maxRun, currentRun);
|
||
edgeDiffSum += generatedEdgeDiff;
|
||
} else {
|
||
currentRun = 0;
|
||
}
|
||
}
|
||
|
||
return {
|
||
axis,
|
||
boxIndex,
|
||
edgeDiffMean: transitionCount > 0 ? edgeDiffSum / transitionCount : 0,
|
||
maxRun,
|
||
maxRunRatio: maxRun / sampleCount,
|
||
position,
|
||
sampleCount,
|
||
transitionCount,
|
||
transitionRatio: transitionCount / sampleCount,
|
||
};
|
||
}
|
||
|
||
async function getScaleAdjustRectangularSeamStats(params: {
|
||
generatedBuffer: Buffer;
|
||
sourceBuffer: Buffer;
|
||
sourceWidth: number;
|
||
sourceHeight: number;
|
||
verification: CanvasScaleAdjustVerificationConfig;
|
||
}) {
|
||
const { generatedBuffer, sourceBuffer, sourceHeight, sourceWidth, verification } =
|
||
params;
|
||
const normalizeToRaw = (buffer: Buffer) =>
|
||
sharp(buffer, { failOn: "none" })
|
||
.rotate()
|
||
.resize(sourceWidth, sourceHeight, {
|
||
fit: "cover",
|
||
position: "center",
|
||
})
|
||
.removeAlpha()
|
||
.toColorspace("srgb")
|
||
.raw()
|
||
.toBuffer();
|
||
const [sourceRaw, generatedRaw] = await Promise.all([
|
||
normalizeToRaw(sourceBuffer),
|
||
normalizeToRaw(generatedBuffer),
|
||
]);
|
||
const changeMask = new Uint8Array(sourceWidth * sourceHeight);
|
||
for (let pixel = 0; pixel < sourceWidth * sourceHeight; pixel += 1) {
|
||
const rawIndex = pixel * 3;
|
||
changeMask[pixel] = getScaleAdjustPixelChanged({
|
||
generatedRaw,
|
||
index: rawIndex,
|
||
sourceRaw,
|
||
})
|
||
? 1
|
||
: 0;
|
||
}
|
||
|
||
const protectedBoxes = getScaleAdjustProtectedRegions(verification).map((region) =>
|
||
rectToPixelBox({
|
||
height: sourceHeight,
|
||
region,
|
||
width: sourceWidth,
|
||
})
|
||
);
|
||
const lineStep = Math.max(
|
||
1,
|
||
Math.floor(Math.min(sourceWidth, sourceHeight) / 1400)
|
||
);
|
||
let bestVertical: ReturnType<typeof summarizeScaleAdjustSeamLine> | null = null;
|
||
let bestHorizontal: ReturnType<typeof summarizeScaleAdjustSeamLine> | null = null;
|
||
|
||
for (const [boxIndex, box] of protectedBoxes.entries()) {
|
||
const left = Math.max(2, box.left + 2);
|
||
const top = Math.max(2, box.top + 2);
|
||
const right = Math.min(sourceWidth - 2, box.left + box.width - 2);
|
||
const bottom = Math.min(sourceHeight - 2, box.top + box.height - 2);
|
||
if (right <= left || bottom <= top) continue;
|
||
|
||
for (let x = left; x <= right; x += lineStep) {
|
||
const stats = summarizeScaleAdjustSeamLine({
|
||
axis: "vertical",
|
||
boxIndex,
|
||
changeMask,
|
||
generatedRaw,
|
||
height: sourceHeight,
|
||
position: x,
|
||
rangeEnd: bottom,
|
||
rangeStart: top,
|
||
sourceRaw,
|
||
width: sourceWidth,
|
||
});
|
||
if (!bestVertical || stats.maxRunRatio > bestVertical.maxRunRatio) {
|
||
bestVertical = stats;
|
||
}
|
||
}
|
||
|
||
for (let y = top; y <= bottom; y += lineStep) {
|
||
const stats = summarizeScaleAdjustSeamLine({
|
||
axis: "horizontal",
|
||
boxIndex,
|
||
changeMask,
|
||
generatedRaw,
|
||
height: sourceHeight,
|
||
position: y,
|
||
rangeEnd: right,
|
||
rangeStart: left,
|
||
sourceRaw,
|
||
width: sourceWidth,
|
||
});
|
||
if (!bestHorizontal || stats.maxRunRatio > bestHorizontal.maxRunRatio) {
|
||
bestHorizontal = stats;
|
||
}
|
||
}
|
||
}
|
||
|
||
const verticalPair =
|
||
bestVertical &&
|
||
bestVertical.maxRunRatio >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_RUN_RATIO &&
|
||
bestVertical.transitionRatio >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_TRANSITION_RATIO;
|
||
const horizontalPair =
|
||
bestHorizontal &&
|
||
bestHorizontal.maxRunRatio >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_RUN_RATIO &&
|
||
bestHorizontal.transitionRatio >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_TRANSITION_RATIO;
|
||
const verticalStrong =
|
||
bestVertical &&
|
||
bestVertical.maxRunRatio >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO &&
|
||
bestVertical.transitionRatio >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO;
|
||
const horizontalStrong =
|
||
bestHorizontal &&
|
||
bestHorizontal.maxRunRatio >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO &&
|
||
bestHorizontal.transitionRatio >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO;
|
||
const verticalVeryStrong =
|
||
bestVertical &&
|
||
bestVertical.maxRunRatio >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_RUN_RATIO &&
|
||
bestVertical.transitionRatio >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_TRANSITION_RATIO;
|
||
const horizontalVeryStrong =
|
||
bestHorizontal &&
|
||
bestHorizontal.maxRunRatio >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_RUN_RATIO &&
|
||
bestHorizontal.transitionRatio >=
|
||
SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_TRANSITION_RATIO;
|
||
|
||
return {
|
||
detected: Boolean(
|
||
(verticalPair && horizontalPair) ||
|
||
(verticalStrong && horizontalStrong) ||
|
||
verticalVeryStrong ||
|
||
horizontalVeryStrong
|
||
),
|
||
horizontal: bestHorizontal,
|
||
lineStep,
|
||
protectedBoxCount: protectedBoxes.length,
|
||
vertical: bestVertical,
|
||
};
|
||
}
|
||
|
||
async function getScaleAdjustFootprintChangeStats(params: {
|
||
generatedBuffer: Buffer;
|
||
sourceBuffer: Buffer;
|
||
sourceWidth: number;
|
||
sourceHeight: number;
|
||
verification: CanvasScaleAdjustVerificationConfig;
|
||
}) {
|
||
const { generatedBuffer, sourceBuffer, sourceHeight, sourceWidth, verification } =
|
||
params;
|
||
const normalizeToRaw = (buffer: Buffer) =>
|
||
sharp(buffer, { failOn: "none" })
|
||
.rotate()
|
||
.resize(sourceWidth, sourceHeight, {
|
||
fit: "cover",
|
||
position: "center",
|
||
})
|
||
.removeAlpha()
|
||
.toColorspace("srgb")
|
||
.raw()
|
||
.toBuffer();
|
||
const [sourceRaw, generatedRaw] = await Promise.all([
|
||
normalizeToRaw(sourceBuffer),
|
||
normalizeToRaw(generatedBuffer),
|
||
]);
|
||
const shrinkStats: Array<{
|
||
generatedOutsidePixels: number;
|
||
outsideRatio: number;
|
||
passed: boolean;
|
||
passReason: "low-source-mask" | "strict-ratio" | "removed-old-footprint" | "failed";
|
||
regionIndex: number;
|
||
removedOutsidePixels: number;
|
||
removedOutsideRatio: number;
|
||
sourceOutsidePixels: number;
|
||
}> = [];
|
||
|
||
for (const [index, region] of verification.regions.entries()) {
|
||
const targetRegion =
|
||
verification.targetRegions?.[index] ||
|
||
(typeof verification.scalePercent === "number"
|
||
? scaleRegionAroundCenter(region, verification.scalePercent)
|
||
: null);
|
||
if (!targetRegion) continue;
|
||
const widthFactor = region.width > 0 ? targetRegion.width / region.width : 1;
|
||
const heightFactor = region.height > 0 ? targetRegion.height / region.height : 1;
|
||
const isShrink = widthFactor < 0.92 || heightFactor < 0.92;
|
||
if (!isShrink) continue;
|
||
|
||
const sourceBox = rectToPixelBox({
|
||
height: sourceHeight,
|
||
region,
|
||
width: sourceWidth,
|
||
});
|
||
const targetBox = rectToPixelBox({
|
||
height: sourceHeight,
|
||
region: expandCanvasRegion(
|
||
targetRegion,
|
||
SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN
|
||
),
|
||
width: sourceWidth,
|
||
});
|
||
const sourceMask = buildScaleAdjustFallbackSubjectMask({
|
||
box: sourceBox,
|
||
raw: sourceRaw,
|
||
width: sourceWidth,
|
||
});
|
||
const generatedMask = buildScaleAdjustFallbackSubjectMask({
|
||
box: sourceBox,
|
||
raw: generatedRaw,
|
||
width: sourceWidth,
|
||
});
|
||
let sourceOutsidePixels = 0;
|
||
let generatedOutsidePixels = 0;
|
||
for (let localY = 0; localY < sourceBox.height; localY += 1) {
|
||
const y = sourceBox.top + localY;
|
||
for (let localX = 0; localX < sourceBox.width; localX += 1) {
|
||
const x = sourceBox.left + localX;
|
||
const insideTarget =
|
||
x >= targetBox.left &&
|
||
x < targetBox.left + targetBox.width &&
|
||
y >= targetBox.top &&
|
||
y < targetBox.top + targetBox.height;
|
||
if (insideTarget) continue;
|
||
const maskIndex = localY * sourceBox.width + localX;
|
||
if (sourceMask[maskIndex] > 0) sourceOutsidePixels += 1;
|
||
if (generatedMask[maskIndex] > 0) generatedOutsidePixels += 1;
|
||
}
|
||
}
|
||
const outsideRatio =
|
||
sourceOutsidePixels > 0 ? generatedOutsidePixels / sourceOutsidePixels : 0;
|
||
const removedOutsidePixels = sourceOutsidePixels - generatedOutsidePixels;
|
||
const removedOutsideRatio =
|
||
sourceOutsidePixels > 0 ? removedOutsidePixels / sourceOutsidePixels : 0;
|
||
const minRemovedOutsidePixels = Math.max(
|
||
SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS,
|
||
Math.round(
|
||
sourceOutsidePixels * SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MIN_REMOVED_RATIO
|
||
)
|
||
);
|
||
const lowSourceMask =
|
||
sourceOutsidePixels < SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS;
|
||
const strictRatio =
|
||
outsideRatio <= SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MAX_RATIO;
|
||
const removedOldFootprint =
|
||
outsideRatio <= SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_RELAXED_MAX_RATIO &&
|
||
removedOutsidePixels >= minRemovedOutsidePixels;
|
||
const passed = lowSourceMask || strictRatio || removedOldFootprint;
|
||
shrinkStats.push({
|
||
generatedOutsidePixels,
|
||
outsideRatio,
|
||
passed,
|
||
passReason: lowSourceMask
|
||
? "low-source-mask"
|
||
: strictRatio
|
||
? "strict-ratio"
|
||
: removedOldFootprint
|
||
? "removed-old-footprint"
|
||
: "failed",
|
||
regionIndex: index,
|
||
removedOutsidePixels,
|
||
removedOutsideRatio,
|
||
sourceOutsidePixels,
|
||
});
|
||
}
|
||
|
||
return {
|
||
failedRegionCount: shrinkStats.filter((item) => !item.passed).length,
|
||
passed: shrinkStats.every((item) => item.passed),
|
||
shrinkRegionCount: shrinkStats.length,
|
||
shrinkStats,
|
||
};
|
||
}
|
||
|
||
async function loadScaleAdjustSourceImage(
|
||
verification: CanvasScaleAdjustVerificationConfig
|
||
) {
|
||
const sourceBuffer = await sharp(
|
||
await fetchImageBufferFromUrl(verification.sourceImageUrl),
|
||
{ failOn: "none" }
|
||
)
|
||
.rotate()
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const sourceMetadata = await sharp(sourceBuffer, { failOn: "none" }).metadata();
|
||
const sourceWidth = Number(sourceMetadata.width || 0);
|
||
const sourceHeight = Number(sourceMetadata.height || 0);
|
||
if (sourceWidth <= 0 || sourceHeight <= 0) {
|
||
return null;
|
||
}
|
||
|
||
return {
|
||
sourceBuffer,
|
||
sourceHeight,
|
||
sourceWidth,
|
||
};
|
||
}
|
||
|
||
function isScaleAdjustAnnotationMarkerColor(r: number, g: number, b: number) {
|
||
const redMarker = r >= 175 && g <= 120 && b <= 120 && r >= g + 55 && r >= b + 55;
|
||
const cyanMarker =
|
||
g >= 145 && b >= 145 && r <= 135 && Math.min(g, b) >= r + 45;
|
||
const blueMarker =
|
||
b >= 165 && r <= 125 && b >= r + 55 && b >= g + 20;
|
||
|
||
return redMarker || cyanMarker || blueMarker;
|
||
}
|
||
|
||
function isPointInCanvasRegions(
|
||
normalizedX: number,
|
||
normalizedY: number,
|
||
regions: CanvasLocalEditCompositeRegion[]
|
||
) {
|
||
return regions.some(
|
||
(region) =>
|
||
normalizedX >= region.x &&
|
||
normalizedX <= region.x + region.width &&
|
||
normalizedY >= region.y &&
|
||
normalizedY <= region.y + region.height
|
||
);
|
||
}
|
||
|
||
function dilateBinaryMask(params: {
|
||
height: number;
|
||
mask: Uint8Array;
|
||
radius: number;
|
||
width: number;
|
||
}) {
|
||
const { height, mask, radius, width } = params;
|
||
if (radius <= 0) return mask;
|
||
|
||
const dilated = new Uint8Array(mask.length);
|
||
for (let y = 0; y < height; y += 1) {
|
||
for (let x = 0; x < width; x += 1) {
|
||
if (mask[y * width + x] === 0) continue;
|
||
const left = Math.max(0, x - radius);
|
||
const right = Math.min(width - 1, x + radius);
|
||
const top = Math.max(0, y - radius);
|
||
const bottom = Math.min(height - 1, y + radius);
|
||
for (let yy = top; yy <= bottom; yy += 1) {
|
||
for (let xx = left; xx <= right; xx += 1) {
|
||
dilated[yy * width + xx] = 255;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return dilated;
|
||
}
|
||
|
||
async function removeScaleAdjustAnnotationArtifacts(params: {
|
||
debugLabel: string;
|
||
generatedBuffer: Buffer;
|
||
imageIndex: number;
|
||
sourceBuffer: Buffer;
|
||
sourceHeight: number;
|
||
sourceWidth: number;
|
||
verification: CanvasScaleAdjustVerificationConfig;
|
||
}) {
|
||
const {
|
||
debugLabel,
|
||
generatedBuffer,
|
||
imageIndex,
|
||
sourceBuffer,
|
||
sourceHeight,
|
||
sourceWidth,
|
||
verification,
|
||
} = params;
|
||
const cleanupRegions = getScaleAdjustProtectedRegions(verification).map((region) =>
|
||
expandCanvasRegion(region, SCALE_ADJUST_MARKER_CLEANUP_MARGIN)
|
||
);
|
||
if (cleanupRegions.length === 0) return null;
|
||
|
||
const normalizeToRaw = (buffer: Buffer) =>
|
||
sharp(buffer, { failOn: "none" })
|
||
.rotate()
|
||
.resize(sourceWidth, sourceHeight, {
|
||
fit: "cover",
|
||
position: "center",
|
||
})
|
||
.removeAlpha()
|
||
.toColorspace("srgb")
|
||
.raw()
|
||
.toBuffer();
|
||
const [sourceRaw, generatedRaw] = await Promise.all([
|
||
normalizeToRaw(sourceBuffer),
|
||
normalizeToRaw(generatedBuffer),
|
||
]);
|
||
const markerMask = new Uint8Array(sourceWidth * sourceHeight);
|
||
let markerPixelCount = 0;
|
||
|
||
for (let y = 0; y < sourceHeight; y += 1) {
|
||
const normalizedY = y / sourceHeight;
|
||
for (let x = 0; x < sourceWidth; x += 1) {
|
||
const normalizedX = x / sourceWidth;
|
||
if (!isPointInCanvasRegions(normalizedX, normalizedY, cleanupRegions)) {
|
||
continue;
|
||
}
|
||
const maskIndex = y * sourceWidth + x;
|
||
const rawIndex = maskIndex * 3;
|
||
if (
|
||
isScaleAdjustAnnotationMarkerColor(
|
||
generatedRaw[rawIndex],
|
||
generatedRaw[rawIndex + 1],
|
||
generatedRaw[rawIndex + 2]
|
||
)
|
||
) {
|
||
markerMask[maskIndex] = 255;
|
||
markerPixelCount += 1;
|
||
}
|
||
}
|
||
}
|
||
|
||
if (markerPixelCount < SCALE_ADJUST_MARKER_MIN_PIXEL_COUNT) return null;
|
||
|
||
const cleanupMask = dilateBinaryMask({
|
||
height: sourceHeight,
|
||
mask: markerMask,
|
||
radius: SCALE_ADJUST_MARKER_DILATE_RADIUS,
|
||
width: sourceWidth,
|
||
});
|
||
const cleanedRaw = Buffer.from(generatedRaw);
|
||
let cleanedPixelCount = 0;
|
||
|
||
for (let index = 0; index < cleanupMask.length; index += 1) {
|
||
if (cleanupMask[index] === 0) continue;
|
||
const rawIndex = index * 3;
|
||
cleanedRaw[rawIndex] = sourceRaw[rawIndex];
|
||
cleanedRaw[rawIndex + 1] = sourceRaw[rawIndex + 1];
|
||
cleanedRaw[rawIndex + 2] = sourceRaw[rawIndex + 2];
|
||
cleanedPixelCount += 1;
|
||
}
|
||
|
||
const cleanedBuffer = await sharp(cleanedRaw, {
|
||
raw: { channels: 3, height: sourceHeight, width: sourceWidth },
|
||
})
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const thumbnailBuffer = await sharp(cleanedBuffer, { failOn: "none" })
|
||
.resize(512, 512, { fit: "inside", withoutEnlargement: true })
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const suffix = `${Date.now()}-${imageIndex + 1}-${Math.random()
|
||
.toString(36)
|
||
.slice(2, 8)}`;
|
||
const [originalUpload, thumbnailUpload] = await Promise.all([
|
||
uploadFile(
|
||
`canvas-scale-adjust-marker-cleaned-${suffix}.png`,
|
||
bufferToArrayBuffer(cleanedBuffer)
|
||
),
|
||
uploadFile(
|
||
`canvas-scale-adjust-marker-cleaned-thumb-${suffix}.png`,
|
||
bufferToArrayBuffer(thumbnailBuffer)
|
||
),
|
||
]);
|
||
|
||
console.info("[Canvas Generate Image] Scale adjust annotation artifacts cleaned:", {
|
||
cleanupRegionCount: cleanupRegions.length,
|
||
cleanedPixelCount,
|
||
debugLabel,
|
||
imageIndex,
|
||
markerPixelCount,
|
||
sourceHeight,
|
||
sourceWidth,
|
||
});
|
||
|
||
return {
|
||
originalUrl: originalUpload.url,
|
||
thumbnailUrl: thumbnailUpload.url,
|
||
} satisfies ParsedImage;
|
||
}
|
||
|
||
async function applyScaleAdjustVerificationToImages(params: {
|
||
images: ParsedImage[];
|
||
verification: CanvasScaleAdjustVerificationConfig | null;
|
||
debugLabel: string;
|
||
}) {
|
||
const { images, verification, debugLabel } = params;
|
||
if (!verification || images.length === 0) return images;
|
||
|
||
const sourceImage = await loadScaleAdjustSourceImage(verification);
|
||
if (!sourceImage) {
|
||
console.warn("[Canvas Generate Image] Skip scale adjust result guard: source image unavailable", {
|
||
debugLabel,
|
||
imageCount: images.length,
|
||
regionCount: verification.regions.length,
|
||
scalePercent: verification.scalePercent ?? null,
|
||
targetRegionCount: verification.targetRegions?.length || 0,
|
||
});
|
||
return images;
|
||
}
|
||
|
||
const evaluatedImages: Array<{
|
||
changeDiffScores: number[];
|
||
changedInside: boolean;
|
||
cleanedImage?: ParsedImage;
|
||
diffScores: number[];
|
||
footprintStats: Awaited<ReturnType<typeof getScaleAdjustFootprintChangeStats>>;
|
||
image: ParsedImage;
|
||
index: number;
|
||
outsideChanged: boolean;
|
||
outsideDiffScores: number[];
|
||
outsidePixelStats: Awaited<
|
||
ReturnType<typeof getScaleAdjustOutsidePixelChangeStats>
|
||
>;
|
||
rectangularSeamDetected: boolean;
|
||
rectangularSeamStats: Awaited<
|
||
ReturnType<typeof getScaleAdjustRectangularSeamStats>
|
||
>;
|
||
}> = [];
|
||
for (const [index, image] of images.entries()) {
|
||
const generatedUrl = image.originalUrl || image.thumbnailUrl;
|
||
const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl);
|
||
const changeRegions = getScaleAdjustEffectiveChangeRegions(verification);
|
||
const [
|
||
diffScores,
|
||
changeDiffScores,
|
||
outsideDiffScores,
|
||
outsidePixelStats,
|
||
footprintStats,
|
||
rectangularSeamStats,
|
||
] = await Promise.all([
|
||
getScaleAdjustDiffScores({
|
||
generatedBuffer,
|
||
regions: verification.regions,
|
||
sourceBuffer: sourceImage.sourceBuffer,
|
||
sourceHeight: sourceImage.sourceHeight,
|
||
sourceWidth: sourceImage.sourceWidth,
|
||
}),
|
||
getScaleAdjustDiffScores({
|
||
generatedBuffer,
|
||
regions: changeRegions,
|
||
sourceBuffer: sourceImage.sourceBuffer,
|
||
sourceHeight: sourceImage.sourceHeight,
|
||
sourceWidth: sourceImage.sourceWidth,
|
||
}),
|
||
getScaleAdjustOutsideDiffScores({
|
||
generatedBuffer,
|
||
sourceBuffer: sourceImage.sourceBuffer,
|
||
sourceHeight: sourceImage.sourceHeight,
|
||
sourceWidth: sourceImage.sourceWidth,
|
||
verification,
|
||
}),
|
||
getScaleAdjustOutsidePixelChangeStats({
|
||
generatedBuffer,
|
||
sourceBuffer: sourceImage.sourceBuffer,
|
||
sourceHeight: sourceImage.sourceHeight,
|
||
sourceWidth: sourceImage.sourceWidth,
|
||
verification,
|
||
}),
|
||
getScaleAdjustFootprintChangeStats({
|
||
generatedBuffer,
|
||
sourceBuffer: sourceImage.sourceBuffer,
|
||
sourceHeight: sourceImage.sourceHeight,
|
||
sourceWidth: sourceImage.sourceWidth,
|
||
verification,
|
||
}),
|
||
getScaleAdjustRectangularSeamStats({
|
||
generatedBuffer,
|
||
sourceBuffer: sourceImage.sourceBuffer,
|
||
sourceHeight: sourceImage.sourceHeight,
|
||
sourceWidth: sourceImage.sourceWidth,
|
||
verification,
|
||
}),
|
||
]);
|
||
const visuallyChangedInside = changeDiffScores.some(
|
||
(score) => score >= SCALE_ADJUST_VISUAL_DIFF_THRESHOLD
|
||
);
|
||
const footprintFallbackAccepted =
|
||
visuallyChangedInside &&
|
||
!footprintStats.passed &&
|
||
changeDiffScores.some(
|
||
(score) => score >= SCALE_ADJUST_FOOTPRINT_FALLBACK_DIFF_THRESHOLD
|
||
);
|
||
const changedInside =
|
||
(visuallyChangedInside && footprintStats.passed) ||
|
||
footprintFallbackAccepted;
|
||
const outsideChanged =
|
||
isScaleAdjustOutsideChanged(outsideDiffScores) ||
|
||
isScaleAdjustOutsidePixelChanged(outsidePixelStats);
|
||
const rectangularSeamDetected = rectangularSeamStats.detected;
|
||
|
||
console.info("[Canvas Generate Image] Scale adjust result guard:", {
|
||
debugLabel,
|
||
changedInside,
|
||
footprintFallbackAccepted,
|
||
footprintChanged: footprintStats.passed,
|
||
footprintStats: {
|
||
failedRegionCount: footprintStats.failedRegionCount,
|
||
shrinkRegionCount: footprintStats.shrinkRegionCount,
|
||
shrinkStats: footprintStats.shrinkStats.map((item) => ({
|
||
generatedOutsidePixels: item.generatedOutsidePixels,
|
||
outsideRatio: Number(item.outsideRatio.toFixed(3)),
|
||
passed: item.passed,
|
||
passReason: item.passReason,
|
||
regionIndex: item.regionIndex,
|
||
removedOutsidePixels: item.removedOutsidePixels,
|
||
removedOutsideRatio: Number(item.removedOutsideRatio.toFixed(3)),
|
||
sourceOutsidePixels: item.sourceOutsidePixels,
|
||
})),
|
||
},
|
||
imageIndex: index,
|
||
outsideChanged,
|
||
outsideDiffScores: outsideDiffScores.map((score) => Number(score.toFixed(2))),
|
||
outsidePixelStats: {
|
||
changedPixelCount: outsidePixelStats.changedPixelCount,
|
||
changedRatio: Number(outsidePixelStats.changedRatio.toFixed(4)),
|
||
meanDiff: Number(outsidePixelStats.meanDiff.toFixed(2)),
|
||
sampledPixelCount: outsidePixelStats.sampledPixelCount,
|
||
step: outsidePixelStats.step,
|
||
},
|
||
changeRegionDiffScores: changeDiffScores.map((score) =>
|
||
Number(score.toFixed(2))
|
||
),
|
||
protectedRegionCount: verification.protectedRegions?.length || 0,
|
||
rectangularSeamDetected,
|
||
rectangularSeamStats: {
|
||
horizontal: rectangularSeamStats.horizontal
|
||
? {
|
||
edgeDiffMean: Number(
|
||
rectangularSeamStats.horizontal.edgeDiffMean.toFixed(2)
|
||
),
|
||
maxRunRatio: Number(
|
||
rectangularSeamStats.horizontal.maxRunRatio.toFixed(3)
|
||
),
|
||
position: rectangularSeamStats.horizontal.position,
|
||
transitionRatio: Number(
|
||
rectangularSeamStats.horizontal.transitionRatio.toFixed(3)
|
||
),
|
||
}
|
||
: null,
|
||
lineStep: rectangularSeamStats.lineStep,
|
||
vertical: rectangularSeamStats.vertical
|
||
? {
|
||
edgeDiffMean: Number(
|
||
rectangularSeamStats.vertical.edgeDiffMean.toFixed(2)
|
||
),
|
||
maxRunRatio: Number(
|
||
rectangularSeamStats.vertical.maxRunRatio.toFixed(3)
|
||
),
|
||
position: rectangularSeamStats.vertical.position,
|
||
transitionRatio: Number(
|
||
rectangularSeamStats.vertical.transitionRatio.toFixed(3)
|
||
),
|
||
}
|
||
: null,
|
||
},
|
||
regionCount: verification.regions.length,
|
||
regionDiffScores: diffScores.map((score) => Number(score.toFixed(2))),
|
||
scalePercent: verification.scalePercent ?? null,
|
||
targetRegionCount: verification.targetRegions?.length || 0,
|
||
});
|
||
|
||
const guardWarnings = [
|
||
...(outsideChanged ? ["outside-frame-changed"] : []),
|
||
...(!changedInside ? ["no-effective-scale-change"] : []),
|
||
...(rectangularSeamDetected ? ["rectangular-seam-detected"] : []),
|
||
];
|
||
if (guardWarnings.length > 0) {
|
||
console.warn("[Canvas Generate Image] Scale adjust result guard warning; keeping result for preview:", {
|
||
debugLabel,
|
||
imageIndex: index,
|
||
reasons: guardWarnings,
|
||
changedInside,
|
||
footprintFallbackAccepted,
|
||
footprintChanged: footprintStats.passed,
|
||
footprintStats: {
|
||
failedRegionCount: footprintStats.failedRegionCount,
|
||
shrinkRegionCount: footprintStats.shrinkRegionCount,
|
||
shrinkStats: footprintStats.shrinkStats.map((item) => ({
|
||
generatedOutsidePixels: item.generatedOutsidePixels,
|
||
outsideRatio: Number(item.outsideRatio.toFixed(3)),
|
||
passed: item.passed,
|
||
passReason: item.passReason,
|
||
regionIndex: item.regionIndex,
|
||
removedOutsidePixels: item.removedOutsidePixels,
|
||
removedOutsideRatio: Number(item.removedOutsideRatio.toFixed(3)),
|
||
sourceOutsidePixels: item.sourceOutsidePixels,
|
||
})),
|
||
},
|
||
outsideChanged,
|
||
outsideDiffScores: outsideDiffScores.map((score) => Number(score.toFixed(2))),
|
||
outsidePixelStats: {
|
||
changedPixelCount: outsidePixelStats.changedPixelCount,
|
||
changedRatio: Number(outsidePixelStats.changedRatio.toFixed(4)),
|
||
meanDiff: Number(outsidePixelStats.meanDiff.toFixed(2)),
|
||
sampledPixelCount: outsidePixelStats.sampledPixelCount,
|
||
step: outsidePixelStats.step,
|
||
},
|
||
rectangularSeamDetected,
|
||
rectangularSeamStats: {
|
||
horizontal: rectangularSeamStats.horizontal
|
||
? {
|
||
edgeDiffMean: Number(
|
||
rectangularSeamStats.horizontal.edgeDiffMean.toFixed(2)
|
||
),
|
||
maxRunRatio: Number(
|
||
rectangularSeamStats.horizontal.maxRunRatio.toFixed(3)
|
||
),
|
||
position: rectangularSeamStats.horizontal.position,
|
||
transitionRatio: Number(
|
||
rectangularSeamStats.horizontal.transitionRatio.toFixed(3)
|
||
),
|
||
}
|
||
: null,
|
||
vertical: rectangularSeamStats.vertical
|
||
? {
|
||
edgeDiffMean: Number(
|
||
rectangularSeamStats.vertical.edgeDiffMean.toFixed(2)
|
||
),
|
||
maxRunRatio: Number(
|
||
rectangularSeamStats.vertical.maxRunRatio.toFixed(3)
|
||
),
|
||
position: rectangularSeamStats.vertical.position,
|
||
transitionRatio: Number(
|
||
rectangularSeamStats.vertical.transitionRatio.toFixed(3)
|
||
),
|
||
}
|
||
: null,
|
||
},
|
||
changeRegionDiffScores: changeDiffScores.map((score) =>
|
||
Number(score.toFixed(2))
|
||
),
|
||
regionDiffScores: diffScores.map((score) => Number(score.toFixed(2))),
|
||
scalePercent: verification.scalePercent ?? null,
|
||
targetRegionCount: verification.targetRegions?.length || 0,
|
||
});
|
||
}
|
||
|
||
let cleanedImage: ParsedImage | undefined;
|
||
try {
|
||
cleanedImage = (await removeScaleAdjustAnnotationArtifacts({
|
||
debugLabel,
|
||
generatedBuffer,
|
||
imageIndex: index,
|
||
sourceBuffer: sourceImage.sourceBuffer,
|
||
sourceHeight: sourceImage.sourceHeight,
|
||
sourceWidth: sourceImage.sourceWidth,
|
||
verification,
|
||
})) ?? undefined;
|
||
if (cleanedImage) {
|
||
evaluatedImages.push({
|
||
changeDiffScores,
|
||
changedInside,
|
||
cleanedImage,
|
||
diffScores,
|
||
footprintStats,
|
||
image,
|
||
index,
|
||
outsideChanged,
|
||
outsideDiffScores,
|
||
outsidePixelStats,
|
||
rectangularSeamDetected,
|
||
rectangularSeamStats,
|
||
});
|
||
continue;
|
||
}
|
||
} catch (cleanupError) {
|
||
console.warn("[Canvas Generate Image] Failed to clean scale adjust annotation artifacts:", {
|
||
debugLabel,
|
||
error: cleanupError instanceof Error ? cleanupError.message : String(cleanupError),
|
||
imageIndex: index,
|
||
});
|
||
}
|
||
|
||
evaluatedImages.push({
|
||
changeDiffScores,
|
||
changedInside,
|
||
cleanedImage,
|
||
diffScores,
|
||
footprintStats,
|
||
image,
|
||
index,
|
||
outsideChanged,
|
||
outsideDiffScores,
|
||
outsidePixelStats,
|
||
rectangularSeamDetected,
|
||
rectangularSeamStats,
|
||
});
|
||
}
|
||
|
||
if (evaluatedImages.length === 0) return images;
|
||
|
||
const scoreEvaluation = (item: (typeof evaluatedImages)[number]) => {
|
||
const insideScore = Math.max(...item.changeDiffScores, ...item.diffScores, 0);
|
||
const outsideMean = item.outsidePixelStats.meanDiff;
|
||
const outsidePenalty = item.outsideChanged ? 100 : 0;
|
||
const noChangePenalty = item.changedInside ? 0 : 60;
|
||
const seamPenalty = item.rectangularSeamDetected ? 140 : 0;
|
||
return (
|
||
insideScore -
|
||
outsideMean * 0.35 -
|
||
outsidePenalty -
|
||
noChangePenalty -
|
||
seamPenalty
|
||
);
|
||
};
|
||
const rankedImages = [...evaluatedImages].sort((left, right) => {
|
||
const leftPassed =
|
||
left.changedInside && !left.outsideChanged && !left.rectangularSeamDetected;
|
||
const rightPassed =
|
||
right.changedInside && !right.outsideChanged && !right.rectangularSeamDetected;
|
||
if (leftPassed !== rightPassed) return leftPassed ? -1 : 1;
|
||
return scoreEvaluation(right) - scoreEvaluation(left);
|
||
});
|
||
const selected = rankedImages[0];
|
||
|
||
console.info("[Canvas Generate Image] Scale adjust candidate selected:", {
|
||
debugLabel,
|
||
candidateCount: evaluatedImages.length,
|
||
imageIndex: selected.index,
|
||
passed:
|
||
selected.changedInside &&
|
||
!selected.outsideChanged &&
|
||
!selected.rectangularSeamDetected,
|
||
changedInside: selected.changedInside,
|
||
outsideChanged: selected.outsideChanged,
|
||
rectangularSeamDetected: selected.rectangularSeamDetected,
|
||
changeRegionDiffScores: selected.changeDiffScores.map((score) =>
|
||
Number(score.toFixed(2))
|
||
),
|
||
regionDiffScores: selected.diffScores.map((score) => Number(score.toFixed(2))),
|
||
outsidePixelStats: {
|
||
changedRatio: Number(selected.outsidePixelStats.changedRatio.toFixed(4)),
|
||
meanDiff: Number(selected.outsidePixelStats.meanDiff.toFixed(2)),
|
||
},
|
||
score: Number(scoreEvaluation(selected).toFixed(2)),
|
||
});
|
||
|
||
const outsideChangeAccepted =
|
||
verification.allowOutsideChangedWhenChangedInside &&
|
||
selected.changedInside &&
|
||
selected.outsideChanged;
|
||
if (outsideChangeAccepted) {
|
||
console.warn("[Canvas Generate Image] Scale adjust accepting best AI candidate despite outside-frame diff:", {
|
||
debugLabel,
|
||
imageIndex: selected.index,
|
||
outsidePixelStats: {
|
||
changedRatio: Number(selected.outsidePixelStats.changedRatio.toFixed(4)),
|
||
meanDiff: Number(selected.outsidePixelStats.meanDiff.toFixed(2)),
|
||
},
|
||
changeRegionDiffScores: selected.changeDiffScores.map((score) =>
|
||
Number(score.toFixed(2))
|
||
),
|
||
regionDiffScores: selected.diffScores.map((score) => Number(score.toFixed(2))),
|
||
});
|
||
}
|
||
|
||
if (
|
||
verification.rejectFailedCandidates &&
|
||
(!selected.changedInside ||
|
||
selected.rectangularSeamDetected ||
|
||
(selected.outsideChanged && !outsideChangeAccepted))
|
||
) {
|
||
throw new ScaleAdjustOutputRejectedError({
|
||
reason: selected.rectangularSeamDetected
|
||
? "rectangular-seam-detected"
|
||
: selected.outsideChanged
|
||
? "outside-frame-changed"
|
||
: "no-effective-scale-change",
|
||
diffScores: selected.changeDiffScores,
|
||
images: [selected.cleanedImage || selected.image],
|
||
outsideDiffScores: selected.outsideDiffScores,
|
||
});
|
||
}
|
||
|
||
return [selected.cleanedImage || selected.image];
|
||
}
|
||
|
||
function getRawPixelIndex(width: number, x: number, y: number) {
|
||
return (y * width + x) * 3;
|
||
}
|
||
|
||
function normalizeScaleAdjustFocusText(input: unknown) {
|
||
return String(input || "")
|
||
.trim()
|
||
.toLowerCase()
|
||
.normalize("NFKC")
|
||
.replace(/[「」『』【】[\]()（）{}]/g, " ")
|
||
.replace(/[_/|:：,，.。+]+/g, " ")
|
||
.replace(/\s+/g, " ");
|
||
}
|
||
|
||
function getScaleAdjustFallbackFocusDirection(prompt: unknown) {
|
||
const text = normalizeScaleAdjustFocusText(prompt);
|
||
const compact = text.replace(/[^a-z0-9\u4e00-\u9fa5]+/g, "");
|
||
if (text.includes("left") || compact.includes("左")) return "left";
|
||
if (text.includes("right") || compact.includes("右")) return "right";
|
||
if (text.includes("top") || compact.includes("上")) return "top";
|
||
if (text.includes("bottom") || compact.includes("下")) return "bottom";
|
||
return "";
|
||
}
|
||
|
||
function getScaleAdjustFallbackFocusedSourceBox(params: {
|
||
prompt?: string;
|
||
sourceBox: ReturnType<typeof rectToPixelBox>;
|
||
}) {
|
||
const { prompt, sourceBox } = params;
|
||
const direction = getScaleAdjustFallbackFocusDirection(prompt);
|
||
if (!direction) return sourceBox;
|
||
|
||
const minWidth = Math.max(1, Math.round(sourceBox.width * 0.42));
|
||
const minHeight = Math.max(1, Math.round(sourceBox.height * 0.42));
|
||
let left = sourceBox.left;
|
||
let top = sourceBox.top;
|
||
let width = sourceBox.width;
|
||
let height = sourceBox.height;
|
||
|
||
if (direction === "left") {
|
||
width = Math.max(minWidth, Math.round(sourceBox.width * 0.68));
|
||
} else if (direction === "right") {
|
||
const nextWidth = Math.max(minWidth, Math.round(sourceBox.width * 0.68));
|
||
left = sourceBox.left + sourceBox.width - nextWidth;
|
||
width = nextWidth;
|
||
} else if (direction === "top") {
|
||
height = Math.max(minHeight, Math.round(sourceBox.height * 0.72));
|
||
} else if (direction === "bottom") {
|
||
const nextHeight = Math.max(minHeight, Math.round(sourceBox.height * 0.72));
|
||
top = sourceBox.top + sourceBox.height - nextHeight;
|
||
height = nextHeight;
|
||
}
|
||
|
||
return {
|
||
left,
|
||
top,
|
||
width,
|
||
height,
|
||
};
|
||
}
|
||
|
||
function getLuma(r: number, g: number, b: number) {
|
||
return r * 0.299 + g * 0.587 + b * 0.114;
|
||
}
|
||
|
||
function dilateMask(params: {
|
||
height: number;
|
||
mask: Uint8Array;
|
||
radius: number;
|
||
width: number;
|
||
}) {
|
||
const { height, mask, radius, width } = params;
|
||
if (radius <= 0) return mask;
|
||
const output = new Uint8Array(mask.length);
|
||
for (let y = 0; y < height; y += 1) {
|
||
for (let x = 0; x < width; x += 1) {
|
||
if (mask[y * width + x] === 0) continue;
|
||
const left = Math.max(0, x - radius);
|
||
const right = Math.min(width - 1, x + radius);
|
||
const top = Math.max(0, y - radius);
|
||
const bottom = Math.min(height - 1, y + radius);
|
||
for (let yy = top; yy <= bottom; yy += 1) {
|
||
for (let xx = left; xx <= right; xx += 1) {
|
||
output[yy * width + xx] = 255;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return output;
|
||
}
|
||
|
||
function estimateScaleAdjustFallbackBackground(params: {
|
||
box: ReturnType<typeof rectToPixelBox>;
|
||
raw: Buffer;
|
||
width: number;
|
||
}) {
|
||
const { box, raw, width } = params;
|
||
const samples: Array<{ b: number; g: number; luma: number; r: number }> = [];
|
||
for (let y = box.top; y < box.top + box.height; y += 1) {
|
||
for (let x = box.left; x < box.left + box.width; x += 1) {
|
||
const index = getRawPixelIndex(width, x, y);
|
||
const r = raw[index] || 0;
|
||
const g = raw[index + 1] || 0;
|
||
const b = raw[index + 2] || 0;
|
||
samples.push({ b, g, luma: getLuma(r, g, b), r });
|
||
}
|
||
}
|
||
if (samples.length === 0) {
|
||
return { b: 245, g: 245, luma: 245, r: 245 };
|
||
}
|
||
samples.sort((left, right) => right.luma - left.luma);
|
||
const keepCount = Math.max(16, Math.ceil(samples.length * 0.35));
|
||
const kept = samples.slice(0, keepCount);
|
||
const totals = kept.reduce(
|
||
(acc, sample) => {
|
||
acc.r += sample.r;
|
||
acc.g += sample.g;
|
||
acc.b += sample.b;
|
||
acc.luma += sample.luma;
|
||
return acc;
|
||
},
|
||
{ b: 0, g: 0, luma: 0, r: 0 }
|
||
);
|
||
return {
|
||
b: totals.b / kept.length,
|
||
g: totals.g / kept.length,
|
||
luma: totals.luma / kept.length,
|
||
r: totals.r / kept.length,
|
||
};
|
||
}
|
||
|
||
function buildScaleAdjustFallbackSubjectMask(params: {
|
||
box: ReturnType<typeof rectToPixelBox>;
|
||
raw: Buffer;
|
||
width: number;
|
||
}) {
|
||
const { box, raw, width } = params;
|
||
const background = estimateScaleAdjustFallbackBackground(params);
|
||
const mask = new Uint8Array(box.width * box.height);
|
||
for (let localY = 0; localY < box.height; localY += 1) {
|
||
const y = box.top + localY;
|
||
for (let localX = 0; localX < box.width; localX += 1) {
|
||
if (localX >= box.width * 0.94) continue;
|
||
const x = box.left + localX;
|
||
const rawIndex = getRawPixelIndex(width, x, y);
|
||
const r = raw[rawIndex] || 0;
|
||
const g = raw[rawIndex + 1] || 0;
|
||
const b = raw[rawIndex + 2] || 0;
|
||
const luma = getLuma(r, g, b);
|
||
const colorDistance = Math.sqrt(
|
||
(r - background.r) ** 2 +
|
||
(g - background.g) ** 2 +
|
||
(b - background.b) ** 2
|
||
);
|
||
if (
|
||
(colorDistance >= SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD &&
|
||
luma <= background.luma - SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP) ||
|
||
luma <= Math.min(118, background.luma - 55)
|
||
) {
|
||
mask[localY * box.width + localX] = 255;
|
||
}
|
||
}
|
||
}
|
||
return dilateMask({
|
||
height: box.height,
|
||
mask,
|
||
radius: Math.max(1, Math.round(Math.min(box.width, box.height) * 0.018)),
|
||
width: box.width,
|
||
});
|
||
}
|
||
|
||
function inpaintScaleAdjustMaskedRegion(params: {
|
||
background: { b: number; g: number; r: number };
|
||
box: ReturnType<typeof rectToPixelBox>;
|
||
mask: Uint8Array;
|
||
raw: Buffer;
|
||
width: number;
|
||
}) {
|
||
const { background, box, mask, raw, width } = params;
|
||
const working = Buffer.from(raw);
|
||
const pending = new Uint8Array(mask);
|
||
let remaining = pending.reduce((sum, value) => sum + (value ? 1 : 0), 0);
|
||
const offsets = [
|
||
[-1, 0],
|
||
[1, 0],
|
||
[0, -1],
|
||
[0, 1],
|
||
[-1, -1],
|
||
[1, -1],
|
||
[-1, 1],
|
||
[1, 1],
|
||
];
|
||
|
||
for (
|
||
let iteration = 0;
|
||
iteration < SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS && remaining > 0;
|
||
iteration += 1
|
||
) {
|
||
const nextValues: Array<{ b: number; g: number; index: number; r: number }> = [];
|
||
for (let localY = 0; localY < box.height; localY += 1) {
|
||
for (let localX = 0; localX < box.width; localX += 1) {
|
||
const maskIndex = localY * box.width + localX;
|
||
if (pending[maskIndex] === 0) continue;
|
||
let count = 0;
|
||
let totalB = 0;
|
||
let totalG = 0;
|
||
let totalR = 0;
|
||
for (const [offsetX, offsetY] of offsets) {
|
||
const nx = localX + offsetX;
|
||
const ny = localY + offsetY;
|
||
if (nx < 0 || ny < 0 || nx >= box.width || ny >= box.height) continue;
|
||
if (pending[ny * box.width + nx] !== 0) continue;
|
||
const rawIndex = getRawPixelIndex(width, box.left + nx, box.top + ny);
|
||
totalR += working[rawIndex] || 0;
|
||
totalG += working[rawIndex + 1] || 0;
|
||
totalB += working[rawIndex + 2] || 0;
|
||
count += 1;
|
||
}
|
||
if (count > 0) {
|
||
nextValues.push({
|
||
b: Math.round(totalB / count),
|
||
g: Math.round(totalG / count),
|
||
index: maskIndex,
|
||
r: Math.round(totalR / count),
|
||
});
|
||
}
|
||
}
|
||
}
|
||
if (nextValues.length === 0) break;
|
||
for (const value of nextValues) {
|
||
const localX = value.index % box.width;
|
||
const localY = Math.floor(value.index / box.width);
|
||
const rawIndex = getRawPixelIndex(width, box.left + localX, box.top + localY);
|
||
working[rawIndex] = value.r;
|
||
working[rawIndex + 1] = value.g;
|
||
working[rawIndex + 2] = value.b;
|
||
pending[value.index] = 0;
|
||
remaining -= 1;
|
||
}
|
||
}
|
||
|
||
if (remaining > 0) {
|
||
for (let localY = 0; localY < box.height; localY += 1) {
|
||
for (let localX = 0; localX < box.width; localX += 1) {
|
||
const maskIndex = localY * box.width + localX;
|
||
if (pending[maskIndex] === 0) continue;
|
||
const rawIndex = getRawPixelIndex(width, box.left + localX, box.top + localY);
|
||
working[rawIndex] = Math.round(background.r);
|
||
working[rawIndex + 1] = Math.round(background.g);
|
||
working[rawIndex + 2] = Math.round(background.b);
|
||
}
|
||
}
|
||
}
|
||
|
||
return working;
|
||
}
|
||
|
||
async function buildScaleAdjustFallbackObjectPng(params: {
|
||
box: ReturnType<typeof rectToPixelBox>;
|
||
mask: Uint8Array;
|
||
sourceBuffer: Buffer;
|
||
targetBox: ReturnType<typeof rectToPixelBox>;
|
||
}) {
|
||
const { box, mask, sourceBuffer, targetBox } = params;
|
||
const cropBuffer = await sharp(sourceBuffer, { failOn: "none" })
|
||
.extract(box)
|
||
.removeAlpha()
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const alphaBuffer = await sharp(mask, {
|
||
raw: { channels: 1, height: box.height, width: box.width },
|
||
})
|
||
.blur(Math.max(0.7, Math.min(box.width, box.height) * 0.012))
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
|
||
return sharp(cropBuffer, { failOn: "none" })
|
||
.joinChannel(alphaBuffer)
|
||
.resize(targetBox.width, targetBox.height, { fit: "fill" })
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
}
|
||
|
||
async function buildScaleAdjustDeterministicFallbackImages(params: {
|
||
debugLabel: string;
|
||
verification: CanvasScaleAdjustVerificationConfig;
|
||
}) {
|
||
const { debugLabel, verification } = params;
|
||
const sourceImage = await loadScaleAdjustSourceImage(verification);
|
||
if (!sourceImage) return [];
|
||
let baseRaw = await sharp(sourceImage.sourceBuffer, { failOn: "none" })
|
||
.resize(sourceImage.sourceWidth, sourceImage.sourceHeight, {
|
||
fit: "cover",
|
||
position: "center",
|
||
})
|
||
.removeAlpha()
|
||
.toColorspace("srgb")
|
||
.raw()
|
||
.toBuffer();
|
||
const overlays: Array<{ input: Buffer; left: number; top: number }> = [];
|
||
|
||
for (const [index, region] of verification.regions.entries()) {
|
||
const targetRegion =
|
||
verification.targetRegions?.[index] ||
|
||
(typeof verification.scalePercent === "number"
|
||
? scaleRegionAroundCenter(region, verification.scalePercent)
|
||
: null);
|
||
if (!targetRegion) continue;
|
||
const sourceBox = rectToPixelBox({
|
||
height: sourceImage.sourceHeight,
|
||
region,
|
||
width: sourceImage.sourceWidth,
|
||
});
|
||
const targetBox = rectToPixelBox({
|
||
height: sourceImage.sourceHeight,
|
||
region: targetRegion,
|
||
width: sourceImage.sourceWidth,
|
||
});
|
||
const prompt = verification.regionPrompts?.[index] || "";
|
||
const focusedSourceBox = getScaleAdjustFallbackFocusedSourceBox({
|
||
prompt,
|
||
sourceBox,
|
||
});
|
||
const mask = buildScaleAdjustFallbackSubjectMask({
|
||
box: focusedSourceBox,
|
||
raw: baseRaw,
|
||
width: sourceImage.sourceWidth,
|
||
});
|
||
const maskedPixels = mask.reduce((sum, value) => sum + (value ? 1 : 0), 0);
|
||
if (
|
||
maskedPixels <
|
||
Math.max(24, focusedSourceBox.width * focusedSourceBox.height * 0.015)
|
||
) {
|
||
console.warn("[Canvas Generate Image] Scale adjust deterministic fallback skipped sparse mask:", {
|
||
debugLabel,
|
||
maskedPixels,
|
||
regionIndex: index,
|
||
});
|
||
continue;
|
||
}
|
||
const background = estimateScaleAdjustFallbackBackground({
|
||
box: focusedSourceBox,
|
||
raw: baseRaw,
|
||
width: sourceImage.sourceWidth,
|
||
});
|
||
const objectBuffer = await buildScaleAdjustFallbackObjectPng({
|
||
box: focusedSourceBox,
|
||
mask,
|
||
sourceBuffer: sourceImage.sourceBuffer,
|
||
targetBox,
|
||
});
|
||
baseRaw = inpaintScaleAdjustMaskedRegion({
|
||
background,
|
||
box: focusedSourceBox,
|
||
mask,
|
||
raw: baseRaw,
|
||
width: sourceImage.sourceWidth,
|
||
});
|
||
overlays.push({
|
||
input: objectBuffer,
|
||
left: targetBox.left,
|
||
top: targetBox.top,
|
||
});
|
||
}
|
||
|
||
if (overlays.length === 0) return [];
|
||
const baseBuffer = await sharp(baseRaw, {
|
||
raw: {
|
||
channels: 3,
|
||
height: sourceImage.sourceHeight,
|
||
width: sourceImage.sourceWidth,
|
||
},
|
||
})
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const mergedBuffer = await sharp(baseBuffer, { failOn: "none" })
|
||
.composite(overlays)
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" })
|
||
.resize(512, 512, { fit: "inside", withoutEnlargement: true })
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||
const [originalUpload, thumbnailUpload] = await Promise.all([
|
||
uploadFile(
|
||
`canvas-scale-adjust-deterministic-fallback-${suffix}.png`,
|
||
bufferToArrayBuffer(mergedBuffer)
|
||
),
|
||
uploadFile(
|
||
`canvas-scale-adjust-deterministic-fallback-thumb-${suffix}.png`,
|
||
bufferToArrayBuffer(thumbnailBuffer)
|
||
),
|
||
]);
|
||
console.warn("[Canvas Generate Image] Scale adjust deterministic fallback generated:", {
|
||
debugLabel,
|
||
originalUrl: originalUpload.url,
|
||
regionCount: verification.regions.length,
|
||
thumbnailUrl: thumbnailUpload.url,
|
||
});
|
||
|
||
return [
|
||
{
|
||
originalUrl: originalUpload.url,
|
||
thumbnailUrl: thumbnailUpload.url,
|
||
},
|
||
] satisfies ParsedImage[];
|
||
}
|
||
|
||
function buildLocalEditCompositeMaskSvg(params: {
|
||
width: number;
|
||
height: number;
|
||
regions: CanvasLocalEditCompositeRegion[];
|
||
featherPx?: number;
|
||
}) {
|
||
const { width, height, regions } = params;
|
||
const blur =
|
||
typeof params.featherPx === "number"
|
||
? Math.max(0, Math.round(params.featherPx))
|
||
: Math.max(
|
||
LOCAL_EDIT_COMPOSITE_FEATHER_MIN_PX,
|
||
Math.min(
|
||
LOCAL_EDIT_COMPOSITE_FEATHER_MAX_PX,
|
||
Math.round(Math.min(width, height) * LOCAL_EDIT_COMPOSITE_FEATHER_RATIO)
|
||
)
|
||
);
|
||
const rectEntries = regions
|
||
.map((region) => {
|
||
const left = Math.max(0, Math.round(region.x * width));
|
||
const top = Math.max(0, Math.round(region.y * height));
|
||
const rectWidth = Math.min(
|
||
width - left,
|
||
Math.max(1, Math.round(region.width * width))
|
||
);
|
||
const rectHeight = Math.min(
|
||
height - top,
|
||
Math.max(1, Math.round(region.height * height))
|
||
);
|
||
return { left, top, rectHeight, rectWidth };
|
||
})
|
||
.filter(({ rectHeight, rectWidth }) => rectWidth > 0 && rectHeight > 0);
|
||
const softRects = rectEntries
|
||
.map(
|
||
({ left, top, rectHeight, rectWidth }) =>
|
||
`<rect x="${left}" y="${top}" width="${rectWidth}" height="${rectHeight}" rx="${Math.round(
|
||
blur * 0.9
|
||
)}" fill="#ffffff" />`
|
||
)
|
||
.join("");
|
||
const coreRects = rectEntries
|
||
.map(({ left, top, rectHeight, rectWidth }) => {
|
||
const inset = Math.max(
|
||
0,
|
||
Math.min(
|
||
Math.round(blur * 1.35),
|
||
Math.floor(rectWidth / 3),
|
||
Math.floor(rectHeight / 3)
|
||
)
|
||
);
|
||
const coreWidth = rectWidth - inset * 2;
|
||
const coreHeight = rectHeight - inset * 2;
|
||
if (coreWidth <= 0 || coreHeight <= 0) return "";
|
||
return `<rect x="${left + inset}" y="${top + inset}" width="${coreWidth}" height="${coreHeight}" rx="${Math.round(
|
||
blur * 0.45
|
||
)}" fill="#ffffff" />`;
|
||
})
|
||
.join("");
|
||
|
||
return Buffer.from(`
|
||
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
|
||
<defs>
|
||
<filter id="local-edit-soft-mask" x="-30%" y="-30%" width="160%" height="160%" color-interpolation-filters="sRGB">
|
||
<feGaussianBlur stdDeviation="${blur}" />
|
||
</filter>
|
||
</defs>
|
||
<rect x="0" y="0" width="${width}" height="${height}" fill="rgba(0,0,0,0)" />
|
||
<g filter="url(#local-edit-soft-mask)">
|
||
${softRects}
|
||
</g>
|
||
${coreRects}
|
||
</svg>
|
||
`);
|
||
}
|
||
|
||
function buildScaleAdjustPatchCompositeMaskSvg(params: {
|
||
width: number;
|
||
height: number;
|
||
featherPx: number;
|
||
regions?: Array<{ left: number; top: number; width: number; height: number }>;
|
||
touchesLeft: boolean;
|
||
touchesTop: boolean;
|
||
touchesRight: boolean;
|
||
touchesBottom: boolean;
|
||
}) {
|
||
const { height, width } = params;
|
||
const featherPx = Math.max(0, Math.round(params.featherPx));
|
||
const regions = params.regions?.filter(
|
||
(region) => region.width > 0 && region.height > 0
|
||
);
|
||
if (regions?.length) {
|
||
const blur = Math.max(1, featherPx);
|
||
const softPad = Math.max(2, Math.round(featherPx * 1.35));
|
||
const corePad = Math.max(0, Math.round(featherPx * 0.22));
|
||
const softRects = regions
|
||
.map((region) => {
|
||
const left = Math.max(0, region.left - softPad);
|
||
const top = Math.max(0, region.top - softPad);
|
||
const right = Math.min(width, region.left + region.width + softPad);
|
||
const bottom = Math.min(height, region.top + region.height + softPad);
|
||
return `<rect x="${left}" y="${top}" width="${Math.max(1, right - left)}" height="${Math.max(1, bottom - top)}" fill="#ffffff" filter="url(#scale-adjust-patch-soft-mask)" />`;
|
||
})
|
||
.join("");
|
||
const coreRects = regions
|
||
.map((region) => {
|
||
const left = Math.max(0, region.left - corePad);
|
||
const top = Math.max(0, region.top - corePad);
|
||
const right = Math.min(width, region.left + region.width + corePad);
|
||
const bottom = Math.min(height, region.top + region.height + corePad);
|
||
return `<rect x="${left}" y="${top}" width="${Math.max(1, right - left)}" height="${Math.max(1, bottom - top)}" fill="#ffffff" />`;
|
||
})
|
||
.join("");
|
||
|
||
return Buffer.from(`
|
||
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
|
||
<defs>
|
||
<filter id="scale-adjust-patch-soft-mask" x="-50%" y="-50%" width="200%" height="200%" color-interpolation-filters="sRGB">
|
||
<feGaussianBlur stdDeviation="${blur}" />
|
||
</filter>
|
||
</defs>
|
||
<rect x="0" y="0" width="${width}" height="${height}" fill="rgba(0,0,0,0)" />
|
||
${softRects}
|
||
${coreRects}
|
||
</svg>
|
||
`);
|
||
}
|
||
|
||
if (featherPx <= 0) {
|
||
return Buffer.from(`
|
||
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
|
||
<rect x="0" y="0" width="${width}" height="${height}" fill="#ffffff" />
|
||
</svg>
|
||
`);
|
||
}
|
||
|
||
const spill = Math.max(2, Math.round(featherPx * 2.5));
|
||
const softLeft = params.touchesLeft ? -spill : 0;
|
||
const softTop = params.touchesTop ? -spill : 0;
|
||
const softRight = params.touchesRight ? width + spill : width;
|
||
const softBottom = params.touchesBottom ? height + spill : height;
|
||
const coreInset = Math.max(0, Math.round(featherPx * 1.25));
|
||
const coreLeft = params.touchesLeft ? 0 : Math.min(coreInset, Math.floor(width / 3));
|
||
const coreTop = params.touchesTop ? 0 : Math.min(coreInset, Math.floor(height / 3));
|
||
const coreRight = params.touchesRight
|
||
? width
|
||
: Math.max(coreLeft + 1, width - Math.min(coreInset, Math.floor(width / 3)));
|
||
const coreBottom = params.touchesBottom
|
||
? height
|
||
: Math.max(coreTop + 1, height - Math.min(coreInset, Math.floor(height / 3)));
|
||
|
||
return Buffer.from(`
|
||
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
|
||
<defs>
|
||
<filter id="scale-adjust-patch-soft-mask" x="-50%" y="-50%" width="200%" height="200%" color-interpolation-filters="sRGB">
|
||
<feGaussianBlur stdDeviation="${featherPx}" />
|
||
</filter>
|
||
</defs>
|
||
<rect x="0" y="0" width="${width}" height="${height}" fill="rgba(0,0,0,0)" />
|
||
<rect x="${softLeft}" y="${softTop}" width="${Math.max(1, softRight - softLeft)}" height="${Math.max(1, softBottom - softTop)}" fill="#ffffff" filter="url(#scale-adjust-patch-soft-mask)" />
|
||
<rect x="${coreLeft}" y="${coreTop}" width="${Math.max(1, coreRight - coreLeft)}" height="${Math.max(1, coreBottom - coreTop)}" fill="#ffffff" />
|
||
</svg>
|
||
`);
|
||
}
|
||
|
||
function resolveLocalEditCompositeFeatherPx(params: {
|
||
composite: CanvasLocalEditCompositeConfig;
|
||
width: number;
|
||
height: number;
|
||
}) {
|
||
const { composite, width, height } = params;
|
||
const minPx =
|
||
typeof composite.featherMinPx === "number"
|
||
? composite.featherMinPx
|
||
: LOCAL_EDIT_COMPOSITE_FEATHER_MIN_PX;
|
||
const maxPx =
|
||
typeof composite.featherMaxPx === "number"
|
||
? Math.max(minPx, composite.featherMaxPx)
|
||
: LOCAL_EDIT_COMPOSITE_FEATHER_MAX_PX;
|
||
const ratio =
|
||
typeof composite.featherRatio === "number"
|
||
? composite.featherRatio
|
||
: LOCAL_EDIT_COMPOSITE_FEATHER_RATIO;
|
||
|
||
return Math.max(
|
||
minPx,
|
||
Math.min(maxPx, Math.round(Math.min(width, height) * ratio))
|
||
);
|
||
}
|
||
|
||
function getLocalEditCompositeRectEntries(params: {
|
||
width: number;
|
||
height: number;
|
||
regions: CanvasLocalEditCompositeRegion[];
|
||
}) {
|
||
const { width, height, regions } = params;
|
||
return regions
|
||
.map((region) => {
|
||
const left = Math.max(0, Math.round(region.x * width));
|
||
const top = Math.max(0, Math.round(region.y * height));
|
||
const rectWidth = Math.min(
|
||
width - left,
|
||
Math.max(1, Math.round(region.width * width))
|
||
);
|
||
const rectHeight = Math.min(
|
||
height - top,
|
||
Math.max(1, Math.round(region.height * height))
|
||
);
|
||
return { left, top, rectHeight, rectWidth };
|
||
})
|
||
.filter(({ rectHeight, rectWidth }) => rectWidth > 0 && rectHeight > 0);
|
||
}
|
||
|
||
function getSmoothDiffAlpha(diff: number, lowThreshold: number, highThreshold: number) {
|
||
if (diff <= lowThreshold) return 0;
|
||
if (diff >= highThreshold) return 255;
|
||
const progress = (diff - lowThreshold) / Math.max(1, highThreshold - lowThreshold);
|
||
return Math.round(255 * progress * progress * (3 - 2 * progress));
|
||
}
|
||
|
||
async function buildLocalEditDiffCompositeMask(params: {
|
||
composite: CanvasLocalEditCompositeConfig;
|
||
generatedBuffer: Buffer;
|
||
sourceBuffer: Buffer;
|
||
width: number;
|
||
height: number;
|
||
}) {
|
||
const { composite, generatedBuffer, height, sourceBuffer, width } = params;
|
||
const compareBlurPx =
|
||
typeof composite.diffCompareBlurPx === "number"
|
||
? composite.diffCompareBlurPx
|
||
: 0;
|
||
const buildDiffRaw = (buffer: Buffer) => {
|
||
const pipeline = sharp(buffer, { failOn: "none" })
|
||
.resize(width, height, { fit: "cover", position: "center" })
|
||
.removeAlpha()
|
||
.toColorspace("srgb");
|
||
return (compareBlurPx >= 0.3 ? pipeline.blur(compareBlurPx) : pipeline)
|
||
.raw()
|
||
.toBuffer();
|
||
};
|
||
const [sourceRaw, generatedRaw] = await Promise.all([
|
||
buildDiffRaw(sourceBuffer),
|
||
buildDiffRaw(generatedBuffer),
|
||
]);
|
||
const lowThreshold =
|
||
typeof composite.diffLowThreshold === "number"
|
||
? composite.diffLowThreshold
|
||
: LOCAL_EDIT_COMPOSITE_DIFF_LOW_THRESHOLD;
|
||
const highThreshold = Math.max(
|
||
lowThreshold + 1,
|
||
typeof composite.diffHighThreshold === "number"
|
||
? composite.diffHighThreshold
|
||
: LOCAL_EDIT_COMPOSITE_DIFF_HIGH_THRESHOLD
|
||
);
|
||
const alpha = Buffer.alloc(width * height, 0);
|
||
const rectEntries = getLocalEditCompositeRectEntries({
|
||
height,
|
||
regions: composite.regions,
|
||
width,
|
||
});
|
||
|
||
for (const { left, top, rectHeight, rectWidth } of rectEntries) {
|
||
const right = Math.min(width, left + rectWidth);
|
||
const bottom = Math.min(height, top + rectHeight);
|
||
for (let y = top; y < bottom; y += 1) {
|
||
for (let x = left; x < right; x += 1) {
|
||
const pixelIndex = y * width + x;
|
||
const rawIndex = pixelIndex * 3;
|
||
const diff =
|
||
Math.abs(generatedRaw[rawIndex] - sourceRaw[rawIndex]) * 0.299 +
|
||
Math.abs(generatedRaw[rawIndex + 1] - sourceRaw[rawIndex + 1]) * 0.587 +
|
||
Math.abs(generatedRaw[rawIndex + 2] - sourceRaw[rawIndex + 2]) * 0.114;
|
||
const nextAlpha = getSmoothDiffAlpha(diff, lowThreshold, highThreshold);
|
||
if (nextAlpha > alpha[pixelIndex]) {
|
||
alpha[pixelIndex] = nextAlpha;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
const featherPx = resolveLocalEditCompositeFeatherPx({
|
||
composite,
|
||
height,
|
||
width,
|
||
});
|
||
|
||
return sharp(alpha, { raw: { channels: 1, height, width } })
|
||
.blur(Math.max(0.3, featherPx))
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
}
|
||
|
||
async function normalizeAlphaMaskToRaw(params: {
|
||
maskBuffer: Buffer;
|
||
width: number;
|
||
height: number;
|
||
}) {
|
||
const { height, maskBuffer, width } = params;
|
||
return sharp(maskBuffer, { failOn: "none" })
|
||
.resize(width, height, { fit: "fill" })
|
||
.removeAlpha()
|
||
.extractChannel(0)
|
||
.raw()
|
||
.toBuffer();
|
||
}
|
||
|
||
async function normalizeOpacityMaskToRaw(params: {
|
||
maskBuffer: Buffer;
|
||
width: number;
|
||
height: number;
|
||
}) {
|
||
const { height, maskBuffer, width } = params;
|
||
return sharp(maskBuffer, { failOn: "none" })
|
||
.resize(width, height, { fit: "fill" })
|
||
.ensureAlpha()
|
||
.extractChannel(3)
|
||
.raw()
|
||
.toBuffer();
|
||
}
|
||
|
||
function multiplyAlphaMasks(primaryMask: Buffer, clipMask: Buffer) {
|
||
if (primaryMask.length !== clipMask.length) return primaryMask;
|
||
const multiplied = Buffer.allocUnsafe(primaryMask.length);
|
||
for (let index = 0; index < primaryMask.length; index += 1) {
|
||
multiplied[index] = Math.round((primaryMask[index] * clipMask[index]) / 255);
|
||
}
|
||
return multiplied;
|
||
}
|
||
|
||
function boostAlphaMask(mask: Buffer, cutoff: number, multiplier: number) {
|
||
const boosted = Buffer.allocUnsafe(mask.length);
|
||
for (let index = 0; index < mask.length; index += 1) {
|
||
const value = mask[index];
|
||
boosted[index] =
|
||
value < cutoff ? 0 : Math.min(255, Math.round(value * multiplier));
|
||
}
|
||
return boosted;
|
||
}
|
||
|
||
async function applyRawAlphaToImage(params: {
|
||
imageBuffer: Buffer;
|
||
alphaMaskRaw: Buffer;
|
||
width: number;
|
||
height: number;
|
||
}) {
|
||
const { alphaMaskRaw, height, imageBuffer, width } = params;
|
||
const rgbRaw = await sharp(imageBuffer, { failOn: "none" })
|
||
.resize(width, height, { fit: "fill" })
|
||
.removeAlpha()
|
||
.toColorspace("srgb")
|
||
.raw()
|
||
.toBuffer();
|
||
|
||
return sharp(rgbRaw, { raw: { channels: 3, height, width } })
|
||
.joinChannel(alphaMaskRaw, {
|
||
raw: { channels: 1, height, width },
|
||
})
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
}
|
||
|
||
async function applyLocalEditCompositeToImages(params: {
|
||
images: ParsedImage[];
|
||
composite: CanvasLocalEditCompositeConfig | null;
|
||
debugLabel: string;
|
||
}) {
|
||
const { images, composite, debugLabel } = params;
|
||
if (!composite || images.length === 0) return images;
|
||
|
||
const sourceBuffer = await sharp(
|
||
await fetchImageBufferFromUrl(composite.sourceImageUrl),
|
||
{ failOn: "none" }
|
||
)
|
||
.rotate()
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const sourceMetadata = await sharp(sourceBuffer, { failOn: "none" }).metadata();
|
||
const sourceWidth = Number(sourceMetadata.width || 0);
|
||
const sourceHeight = Number(sourceMetadata.height || 0);
|
||
if (sourceWidth <= 0 || sourceHeight <= 0) {
|
||
throw new Error("局部编辑结果合成失败：无法识别原图尺寸");
|
||
}
|
||
|
||
const featherPx = resolveLocalEditCompositeFeatherPx({
|
||
composite,
|
||
height: sourceHeight,
|
||
width: sourceWidth,
|
||
});
|
||
|
||
const mergedImages: ParsedImage[] = [];
|
||
for (const [index, image] of images.entries()) {
|
||
const generatedUrl = image.originalUrl || image.thumbnailUrl;
|
||
const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl);
|
||
const resizedGeneratedBuffer = await sharp(generatedBuffer, { failOn: "none" })
|
||
.rotate()
|
||
.resize(sourceWidth, sourceHeight, { fit: "cover", position: "center" })
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const overlayBuffer =
|
||
composite.maskMode === "diff"
|
||
? await applyRawAlphaToImage({
|
||
alphaMaskRaw: await normalizeAlphaMaskToRaw({
|
||
height: sourceHeight,
|
||
maskBuffer: await buildLocalEditDiffCompositeMask({
|
||
composite,
|
||
generatedBuffer: resizedGeneratedBuffer,
|
||
height: sourceHeight,
|
||
sourceBuffer,
|
||
width: sourceWidth,
|
||
}),
|
||
width: sourceWidth,
|
||
}),
|
||
height: sourceHeight,
|
||
imageBuffer: resizedGeneratedBuffer,
|
||
width: sourceWidth,
|
||
})
|
||
: await sharp(resizedGeneratedBuffer, { failOn: "none" })
|
||
.ensureAlpha()
|
||
.composite([
|
||
{
|
||
input: buildLocalEditCompositeMaskSvg({
|
||
featherPx,
|
||
height: sourceHeight,
|
||
regions: composite.regions,
|
||
width: sourceWidth,
|
||
}),
|
||
blend: "dest-in",
|
||
},
|
||
])
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const mergedBuffer = await sharp(sourceBuffer, { failOn: "none" })
|
||
.ensureAlpha()
|
||
.composite([{ input: overlayBuffer, left: 0, top: 0 }])
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" })
|
||
.resize(512, 512, { fit: "inside", withoutEnlargement: true })
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const suffix = `${Date.now()}-${index + 1}-${Math.random().toString(36).slice(2, 8)}`;
|
||
const [originalUpload, thumbnailUpload] = await Promise.all([
|
||
uploadFile(
|
||
`canvas-local-edit-composited-${suffix}.png`,
|
||
bufferToArrayBuffer(mergedBuffer)
|
||
),
|
||
uploadFile(
|
||
`canvas-local-edit-composited-thumb-${suffix}.png`,
|
||
bufferToArrayBuffer(thumbnailBuffer)
|
||
),
|
||
]);
|
||
mergedImages.push({
|
||
originalUrl: originalUpload.url,
|
||
thumbnailUrl: thumbnailUpload.url,
|
||
});
|
||
}
|
||
|
||
console.info("[Canvas Generate Image] Local edit composite applied:", {
|
||
debugLabel,
|
||
expandRegions: composite.expandRegions !== false,
|
||
featherPx,
|
||
imageCount: mergedImages.length,
|
||
maskMode: composite.maskMode || "region",
|
||
regionCount: composite.regions.length,
|
||
sourceHeight,
|
||
sourceWidth,
|
||
});
|
||
|
||
return mergedImages;
|
||
}
|
||
|
||
function resolveScaleAdjustPatchCompositeFeatherPx(params: {
|
||
composite: CanvasScaleAdjustPatchCompositeConfig;
|
||
width: number;
|
||
height: number;
|
||
}) {
|
||
const { composite, height, width } = params;
|
||
const minPx =
|
||
typeof composite.featherMinPx === "number" ? composite.featherMinPx : 10;
|
||
const maxPx =
|
||
typeof composite.featherMaxPx === "number"
|
||
? Math.max(minPx, composite.featherMaxPx)
|
||
: 42;
|
||
const ratio =
|
||
typeof composite.featherRatio === "number" ? composite.featherRatio : 0.018;
|
||
|
||
return Math.max(
|
||
minPx,
|
||
Math.min(maxPx, Math.round(Math.min(width, height) * ratio))
|
||
);
|
||
}
|
||
|
||
async function applyScaleAdjustPatchCompositeToImages(params: {
|
||
images: ParsedImage[];
|
||
composite: CanvasScaleAdjustPatchCompositeConfig | null;
|
||
debugLabel: string;
|
||
}) {
|
||
const { images, composite, debugLabel } = params;
|
||
if (!composite || images.length === 0) return images;
|
||
|
||
const sourceBuffer = await sharp(
|
||
await fetchImageBufferFromUrl(composite.sourceImageUrl),
|
||
{ failOn: "none" }
|
||
)
|
||
.rotate()
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const sourceMetadata = await sharp(sourceBuffer, { failOn: "none" }).metadata();
|
||
const sourceWidth = Number(sourceMetadata.width || 0);
|
||
const sourceHeight = Number(sourceMetadata.height || 0);
|
||
if (sourceWidth <= 0 || sourceHeight <= 0) {
|
||
throw new Error("比例调整局部结果合成失败：无法识别原图尺寸");
|
||
}
|
||
|
||
const workspaceBox = rectToPixelBox({
|
||
height: sourceHeight,
|
||
region: composite.workspaceRegion,
|
||
width: sourceWidth,
|
||
});
|
||
const featherPx = resolveScaleAdjustPatchCompositeFeatherPx({
|
||
composite,
|
||
height: workspaceBox.height,
|
||
width: workspaceBox.width,
|
||
});
|
||
const touchesLeft = workspaceBox.left <= 0;
|
||
const touchesTop = workspaceBox.top <= 0;
|
||
const touchesRight = workspaceBox.left + workspaceBox.width >= sourceWidth;
|
||
const touchesBottom = workspaceBox.top + workspaceBox.height >= sourceHeight;
|
||
const maskRegionBoxes = composite.regions
|
||
?.map((region) =>
|
||
rectToPixelBox({
|
||
height: sourceHeight,
|
||
region,
|
||
width: sourceWidth,
|
||
})
|
||
)
|
||
.map((box) => {
|
||
const left = Math.max(0, box.left - workspaceBox.left);
|
||
const top = Math.max(0, box.top - workspaceBox.top);
|
||
const right = Math.min(
|
||
workspaceBox.width,
|
||
box.left + box.width - workspaceBox.left
|
||
);
|
||
const bottom = Math.min(
|
||
workspaceBox.height,
|
||
box.top + box.height - workspaceBox.top
|
||
);
|
||
return {
|
||
left,
|
||
top,
|
||
width: Math.max(0, right - left),
|
||
height: Math.max(0, bottom - top),
|
||
};
|
||
})
|
||
.filter((box) => box.width > 0 && box.height > 0);
|
||
const staticPatchMask =
|
||
composite.maskMode === "diff"
|
||
? null
|
||
: buildScaleAdjustPatchCompositeMaskSvg({
|
||
featherPx,
|
||
height: workspaceBox.height,
|
||
regions: maskRegionBoxes,
|
||
touchesBottom,
|
||
touchesLeft,
|
||
touchesRight,
|
||
touchesTop,
|
||
width: workspaceBox.width,
|
||
});
|
||
const regionClipPatchMask = maskRegionBoxes?.length
|
||
? buildScaleAdjustPatchCompositeMaskSvg({
|
||
featherPx,
|
||
height: workspaceBox.height,
|
||
regions: maskRegionBoxes,
|
||
touchesBottom,
|
||
touchesLeft,
|
||
touchesRight,
|
||
touchesTop,
|
||
width: workspaceBox.width,
|
||
})
|
||
: null;
|
||
const regionClipPatchMaskRaw = regionClipPatchMask
|
||
? await normalizeOpacityMaskToRaw({
|
||
height: workspaceBox.height,
|
||
maskBuffer: regionClipPatchMask,
|
||
width: workspaceBox.width,
|
||
})
|
||
: null;
|
||
const sourcePatchBuffer = await sharp(sourceBuffer, { failOn: "none" })
|
||
.extract(workspaceBox)
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const customClipPatchMaskRaw = composite.clipMaskImageUrl
|
||
? await normalizeOpacityMaskToRaw({
|
||
height: workspaceBox.height,
|
||
maskBuffer: await sharp(await fetchImageBufferFromUrl(composite.clipMaskImageUrl), {
|
||
failOn: "none",
|
||
})
|
||
.rotate()
|
||
.resize(sourceWidth, sourceHeight, { fit: "fill" })
|
||
.extract(workspaceBox)
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer(),
|
||
width: workspaceBox.width,
|
||
})
|
||
: null;
|
||
const activeClipPatchMaskRaw = customClipPatchMaskRaw || regionClipPatchMaskRaw;
|
||
const diffRegions = (maskRegionBoxes?.length
|
||
? maskRegionBoxes
|
||
: [{ left: 0, top: 0, width: workspaceBox.width, height: workspaceBox.height }]
|
||
).map((box) => ({
|
||
x: box.left / Math.max(1, workspaceBox.width),
|
||
y: box.top / Math.max(1, workspaceBox.height),
|
||
width: box.width / Math.max(1, workspaceBox.width),
|
||
height: box.height / Math.max(1, workspaceBox.height),
|
||
}));
|
||
|
||
const mergedImages: ParsedImage[] = [];
|
||
for (const [index, image] of images.entries()) {
|
||
const generatedUrl = image.originalUrl || image.thumbnailUrl;
|
||
const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl);
|
||
const resizedGeneratedBuffer = await sharp(generatedBuffer, { failOn: "none" })
|
||
.rotate()
|
||
.resize(workspaceBox.width, workspaceBox.height, {
|
||
fit: "cover",
|
||
position: "center",
|
||
})
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const diffPatchMask =
|
||
composite.maskMode === "diff"
|
||
? await buildLocalEditDiffCompositeMask({
|
||
composite: {
|
||
diffCompareBlurPx: composite.diffCompareBlurPx,
|
||
diffHighThreshold: composite.diffHighThreshold,
|
||
diffLowThreshold: composite.diffLowThreshold,
|
||
featherMaxPx: composite.featherMaxPx,
|
||
featherMinPx: composite.featherMinPx,
|
||
featherRatio: composite.featherRatio,
|
||
maskMode: "diff",
|
||
regions: diffRegions,
|
||
sourceImageUrl: composite.sourceImageUrl,
|
||
},
|
||
generatedBuffer: resizedGeneratedBuffer,
|
||
height: workspaceBox.height,
|
||
sourceBuffer: sourcePatchBuffer,
|
||
width: workspaceBox.width,
|
||
})
|
||
: null;
|
||
const patchMask = diffPatchMask || staticPatchMask;
|
||
if (!patchMask) {
|
||
throw new Error("比例调整局部结果合成失败：无法生成合成蒙版");
|
||
}
|
||
const patchOverlayBuffer = diffPatchMask
|
||
? await applyRawAlphaToImage({
|
||
alphaMaskRaw: boostAlphaMask(
|
||
multiplyAlphaMasks(
|
||
await normalizeAlphaMaskToRaw({
|
||
height: workspaceBox.height,
|
||
maskBuffer: diffPatchMask,
|
||
width: workspaceBox.width,
|
||
}),
|
||
activeClipPatchMaskRaw ||
|
||
Buffer.alloc(workspaceBox.width * workspaceBox.height, 255)
|
||
),
|
||
typeof composite.diffAlphaCutoff === "number"
|
||
? composite.diffAlphaCutoff
|
||
: 8,
|
||
typeof composite.diffAlphaMultiplier === "number"
|
||
? composite.diffAlphaMultiplier
|
||
: 3
|
||
),
|
||
height: workspaceBox.height,
|
||
imageBuffer: resizedGeneratedBuffer,
|
||
width: workspaceBox.width,
|
||
})
|
||
: await sharp(resizedGeneratedBuffer, { failOn: "none" })
|
||
.ensureAlpha()
|
||
.composite([{ input: patchMask, blend: "dest-in" }])
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const mergedBuffer = await sharp(sourceBuffer, { failOn: "none" })
|
||
.ensureAlpha()
|
||
.composite([
|
||
{
|
||
input: patchOverlayBuffer,
|
||
left: workspaceBox.left,
|
||
top: workspaceBox.top,
|
||
},
|
||
])
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" })
|
||
.resize(512, 512, { fit: "inside", withoutEnlargement: true })
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const suffix = `${Date.now()}-${index + 1}-${Math.random()
|
||
.toString(36)
|
||
.slice(2, 8)}`;
|
||
const [originalUpload, thumbnailUpload] = await Promise.all([
|
||
uploadFile(
|
||
`canvas-scale-adjust-ai-patch-composited-${suffix}.png`,
|
||
bufferToArrayBuffer(mergedBuffer)
|
||
),
|
||
uploadFile(
|
||
`canvas-scale-adjust-ai-patch-composited-thumb-${suffix}.png`,
|
||
bufferToArrayBuffer(thumbnailBuffer)
|
||
),
|
||
]);
|
||
mergedImages.push({
|
||
originalUrl: originalUpload.url,
|
||
thumbnailUrl: thumbnailUpload.url,
|
||
});
|
||
}
|
||
|
||
console.info("[Canvas Generate Image] Scale adjust AI patch composite applied:", {
|
||
debugLabel,
|
||
featherPx,
|
||
imageCount: mergedImages.length,
|
||
customClipMaskApplied: Boolean(customClipPatchMaskRaw),
|
||
diffRegionClipApplied: Boolean(regionClipPatchMask),
|
||
maskMode: composite.maskMode || "workspace",
|
||
maskRegionCount: maskRegionBoxes?.length || 0,
|
||
sourceHeight,
|
||
sourceWidth,
|
||
workspaceBox,
|
||
});
|
||
|
||
return mergedImages;
|
||
}
|
||
|
||
function getProviderBatchLimit(providerType: string) {
|
||
switch (providerType) {
|
||
case "ruxa-image":
|
||
case "bfl-image":
|
||
case "openai-image":
|
||
case "stepfun-image":
|
||
case "midjourney":
|
||
case "runway":
|
||
return 1;
|
||
case "snapgen-image":
|
||
return 8;
|
||
case "krea":
|
||
return 4;
|
||
case "gemini":
|
||
case "grsai-gpt-image":
|
||
return 2;
|
||
case "alibaba-image":
|
||
case "fal-image":
|
||
return 6;
|
||
default:
|
||
return 2;
|
||
}
|
||
}
|
||
|
||
type CanvasModelWithProvider = Awaited<
|
||
ReturnType<typeof getEnabledAiModelsWithProvider>
|
||
>[number];
|
||
|
||
type CanvasImageSizeValue = string;
|
||
const CANVAS_FAILOVER_SAME_NAME_THRESHOLD = 0.5;
|
||
|
||
function getCanvasScaleAdjustProviderFailoverPriority(providerType: string) {
|
||
switch (providerType) {
|
||
case "ruxa-image":
|
||
return 10;
|
||
case "snapgen-image":
|
||
return 20;
|
||
case "krea":
|
||
return 30;
|
||
case "gemini":
|
||
case "grsai-gpt-image":
|
||
return 40;
|
||
case "alibaba-image":
|
||
case "bfl-image":
|
||
case "openai-image":
|
||
case "stepfun-image":
|
||
case "volcano-vision":
|
||
return 60;
|
||
case "fal-image":
|
||
return 1000;
|
||
default:
|
||
return 80;
|
||
}
|
||
}
|
||
|
||
function getCanvasScaleAdjustHealthPenalty(
|
||
healthStatus: string | null | undefined
|
||
) {
|
||
if (healthStatus === "degraded") return 25;
|
||
if (healthStatus === "auto_disabled") return 200;
|
||
return 0;
|
||
}
|
||
|
||
function getCanvasScaleAdjustFailoverPriority(item: CanvasModelWithProvider) {
|
||
return (
|
||
getCanvasScaleAdjustProviderFailoverPriority(item.provider.type) +
|
||
getCanvasScaleAdjustHealthPenalty(item.model.healthStatus)
|
||
);
|
||
}
|
||
|
||
interface CanvasImageGenConfig {
|
||
maxImages?: number;
|
||
supports1K?: boolean;
|
||
supports2K?: boolean;
|
||
supports4K?: boolean;
|
||
supportedImageSizes?: string[];
|
||
}
|
||
|
||
function normalizeCanvasImageSizeValue(
|
||
value?: string,
|
||
fallback: CanvasImageSizeValue = "2K"
|
||
): CanvasImageSizeValue {
|
||
const trimmed = String(value || "").trim();
|
||
if (!trimmed) return fallback;
|
||
const kMatch = trimmed.match(/^(\d+(?:\.\d+)?)\s*k$/i);
|
||
if (kMatch?.[1]) {
|
||
return `${Number(kMatch[1]).toString()}K`;
|
||
}
|
||
return trimmed;
|
||
}
|
||
|
||
function isCanvasImageSizeSupported(
|
||
config: CanvasImageGenConfig | null | undefined,
|
||
size: CanvasImageSizeValue
|
||
) {
|
||
if (!config) return true;
|
||
const configuredSizes = Array.isArray(config.supportedImageSizes)
|
||
? config.supportedImageSizes
|
||
.map((item) => normalizeCanvasImageSizeValue(item, ""))
|
||
.filter(Boolean)
|
||
: [];
|
||
if (configuredSizes.length > 0) {
|
||
return configuredSizes.includes(normalizeCanvasImageSizeValue(size, ""));
|
||
}
|
||
if (size === "1K") return config.supports1K !== false;
|
||
if (size === "2K") return config.supports2K !== false;
|
||
if (size === "4K") return config.supports4K !== false;
|
||
return true;
|
||
}
|
||
|
||
function pickSupportedCanvasImageSize(
|
||
config: CanvasImageGenConfig | null | undefined,
|
||
requestedSize: CanvasImageSizeValue
|
||
): CanvasImageSizeValue {
|
||
if (isCanvasImageSizeSupported(config, requestedSize)) return requestedSize;
|
||
|
||
const preferredFallbacks: Record<CanvasImageSizeValue, CanvasImageSizeValue[]> = {
|
||
"4K": ["2K", "1K"],
|
||
"2K": ["1K"],
|
||
"1K": ["2K", "4K"],
|
||
};
|
||
const configuredSizes = Array.isArray(config?.supportedImageSizes)
|
||
? config.supportedImageSizes
|
||
.map((item) => normalizeCanvasImageSizeValue(item, ""))
|
||
.filter(Boolean)
|
||
: [];
|
||
|
||
return (
|
||
(preferredFallbacks[requestedSize] || ["2K", "1K", "4K"]).find((candidate) =>
|
||
isCanvasImageSizeSupported(config, candidate)
|
||
) ||
|
||
configuredSizes.find((candidate) =>
|
||
isCanvasImageSizeSupported(config, candidate)
|
||
) ||
|
||
["1K", "2K", "4K"].find((candidate) =>
|
||
isCanvasImageSizeSupported(config, candidate)
|
||
) ||
|
||
"2K"
|
||
);
|
||
}
|
||
|
||
function normalizeCanvasImageParamsByModel(params: {
|
||
imageCount: number;
|
||
imageSize: string;
|
||
imageGenConfig: CanvasImageGenConfig | null | undefined;
|
||
}) {
|
||
const maxImages =
|
||
typeof params.imageGenConfig?.maxImages === "number" &&
|
||
Number.isFinite(params.imageGenConfig.maxImages)
|
||
? Math.max(1, Math.floor(params.imageGenConfig.maxImages))
|
||
: 12;
|
||
const imageCount = Math.max(1, Math.min(maxImages, Math.floor(params.imageCount)));
|
||
|
||
const requestedSize = normalizeCanvasImageSizeValue(params.imageSize);
|
||
const imageSize = pickSupportedCanvasImageSize(params.imageGenConfig, requestedSize);
|
||
|
||
return { imageCount, imageSize };
|
||
}
|
||
|
||
function getCanvasModelRefKeys(item: CanvasModelWithProvider) {
|
||
return [
|
||
String(item.model.id || "").trim(),
|
||
String(item.model.modelId || "").trim(),
|
||
].filter(Boolean);
|
||
}
|
||
|
||
function normalizeModelNameForSimilarity(value: unknown) {
|
||
return String(value || "")
|
||
.trim()
|
||
.toLowerCase()
|
||
.replace(/[「」『』【】[\]()（）{}]/g, " ")
|
||
.replace(/[_/|:：,，.。+]+/g, "-")
|
||
.replace(/\s+/g, " ");
|
||
}
|
||
|
||
function resolveCanvasScaleAdjustModelFamilyFromText(value: unknown) {
|
||
const normalized = normalizeModelNameForSimilarity(value);
|
||
const compact = normalized.replace(/[^a-z0-9]+/g, "");
|
||
if (!normalized && !compact) return "";
|
||
if (
|
||
normalized.includes("gemini-3-1-flash-image-preview") ||
|
||
normalized.includes("gemini-3.1-flash-image-preview") ||
|
||
normalized.includes("nano-banana-2") ||
|
||
compact.includes("nanobanana2")
|
||
) {
|
||
return "nano-banana-2";
|
||
}
|
||
if (
|
||
normalized.includes("gemini-3-pro-image-preview") ||
|
||
normalized.includes("nano-banana-pro") ||
|
||
compact.includes("nanobananapro")
|
||
) {
|
||
return "nano-banana-pro";
|
||
}
|
||
if (normalized.includes("gpt-image-2") || compact.includes("gptimage2")) {
|
||
return "gpt-image-2";
|
||
}
|
||
return "";
|
||
}
|
||
|
||
function getCanvasScaleAdjustModelFamily(item: CanvasModelWithProvider) {
|
||
return (
|
||
resolveCanvasScaleAdjustModelFamilyFromText(item.model.modelId) ||
|
||
resolveCanvasScaleAdjustModelFamilyFromText(item.model.displayName) ||
|
||
resolveCanvasScaleAdjustModelFamilyFromText(item.model.id)
|
||
);
|
||
}
|
||
|
||
function tokenizeModelNameForSimilarity(value: unknown) {
|
||
const normalized = normalizeModelNameForSimilarity(value);
|
||
const compact = normalized.replace(/[^a-z0-9\u4e00-\u9fa5]+/g, "");
|
||
const tokens = new Set(
|
||
normalized
|
||
.split(/[^a-z0-9\u4e00-\u9fa5]+/g)
|
||
.map((item) => item.trim())
|
||
.filter((item) => item.length >= 2)
|
||
);
|
||
|
||
if (compact.length >= 2) {
|
||
tokens.add(compact);
|
||
for (let index = 0; index < compact.length - 1; index += 1) {
|
||
tokens.add(compact.slice(index, index + 2));
|
||
}
|
||
}
|
||
|
||
return tokens;
|
||
}
|
||
|
||
function scoreModelNameSimilarity(
|
||
source: CanvasModelWithProvider,
|
||
candidate: CanvasModelWithProvider
|
||
) {
|
||
const sourceNames = [
|
||
source.model.displayName,
|
||
source.model.modelId,
|
||
].filter(Boolean);
|
||
const candidateNames = [
|
||
candidate.model.displayName,
|
||
candidate.model.modelId,
|
||
].filter(Boolean);
|
||
|
||
let bestScore = 0;
|
||
for (const sourceName of sourceNames) {
|
||
const sourceTokens = tokenizeModelNameForSimilarity(sourceName);
|
||
if (sourceTokens.size === 0) continue;
|
||
for (const candidateName of candidateNames) {
|
||
const candidateTokens = tokenizeModelNameForSimilarity(candidateName);
|
||
if (candidateTokens.size === 0) continue;
|
||
let overlap = 0;
|
||
sourceTokens.forEach((token) => {
|
||
if (candidateTokens.has(token)) overlap += 1;
|
||
});
|
||
const score = overlap / Math.max(sourceTokens.size, candidateTokens.size);
|
||
bestScore = Math.max(bestScore, score);
|
||
}
|
||
}
|
||
|
||
return bestScore;
|
||
}
|
||
|
||
function findCanvasFailoverModel(params: {
|
||
selectedModel: CanvasModelWithProvider;
|
||
imageModels: CanvasModelWithProvider[];
|
||
failedModelRefs: string[];
|
||
failedProviderTypes?: string[];
|
||
preferLowCostSameNameProviders?: boolean;
|
||
requiresImageInput?: boolean;
|
||
requiresTextToImage?: boolean;
|
||
scaleAdjustModelFamily?: string | null;
|
||
}) {
|
||
const failedRefs = new Set(
|
||
params.failedModelRefs.map((item) => String(item || "").trim()).filter(Boolean)
|
||
);
|
||
const failedProviderTypes = new Set(
|
||
(params.failedProviderTypes || [])
|
||
.map((item) => String(item || "").trim())
|
||
.filter(Boolean)
|
||
);
|
||
const strictScaleAdjustModelFamily =
|
||
params.preferLowCostSameNameProviders &&
|
||
typeof params.scaleAdjustModelFamily === "string" &&
|
||
params.scaleAdjustModelFamily.trim()
|
||
? params.scaleAdjustModelFamily.trim()
|
||
: "";
|
||
const selectedFallbackRefs = [
|
||
params.selectedModel.model.fallbackModelId1,
|
||
params.selectedModel.model.fallbackModelId2,
|
||
]
|
||
.map((item) => String(item || "").trim())
|
||
.filter(Boolean);
|
||
|
||
const isUsableFallback = (item: CanvasModelWithProvider) => {
|
||
const isSameFamilyScaleAdjustFallback = strictScaleAdjustModelFamily
|
||
? getCanvasScaleAdjustModelFamily(item) === strictScaleAdjustModelFamily
|
||
: false;
|
||
const isSameNameScaleAdjustFallback =
|
||
params.preferLowCostSameNameProviders &&
|
||
(isSameFamilyScaleAdjustFallback ||
|
||
scoreModelNameSimilarity(params.selectedModel, item) >=
|
||
CANVAS_FAILOVER_SAME_NAME_THRESHOLD);
|
||
if (strictScaleAdjustModelFamily && !isSameFamilyScaleAdjustFallback) {
|
||
return false;
|
||
}
|
||
if (
|
||
!item.model.isEnabled ||
|
||
item.model.isSubModel ||
|
||
(item.model.excludeFromFailover && !isSameNameScaleAdjustFallback)
|
||
) {
|
||
return false;
|
||
}
|
||
if (failedProviderTypes.has(item.provider.type)) return false;
|
||
if (!item.model.supportsImageGen) return false;
|
||
if (
|
||
params.requiresImageInput &&
|
||
modelLacksReferenceImageInputSupport(item.model, item.provider)
|
||
) {
|
||
return false;
|
||
}
|
||
if (
|
||
params.requiresTextToImage &&
|
||
!modelSupportsTextToImageDescriptor({
|
||
providerType: item.provider.type,
|
||
modelId: item.model.modelId,
|
||
capabilities: item.model.capabilities,
|
||
defaultParams: item.model.defaultParams,
|
||
supportsImageGen: item.model.supportsImageGen,
|
||
})
|
||
) {
|
||
return false;
|
||
}
|
||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(item.provider.type)) return false;
|
||
return getCanvasModelRefKeys(item).every((key) => !failedRefs.has(key));
|
||
};
|
||
|
||
if (!params.preferLowCostSameNameProviders) {
|
||
for (const fallbackRef of selectedFallbackRefs) {
|
||
const configured = params.imageModels.find(
|
||
(item) =>
|
||
isUsableFallback(item) &&
|
||
(item.model.id === fallbackRef || item.model.modelId === fallbackRef)
|
||
);
|
||
if (configured) return configured;
|
||
}
|
||
}
|
||
|
||
const usableModels = params.imageModels.filter(isUsableFallback);
|
||
const scoredModels = usableModels
|
||
.map((item, index) => ({
|
||
item,
|
||
index,
|
||
nameSimilarity: scoreModelNameSimilarity(params.selectedModel, item),
|
||
sameProviderType: item.provider.type === params.selectedModel.provider.type,
|
||
}))
|
||
.sort((left, right) => {
|
||
if (strictScaleAdjustModelFamily) {
|
||
const leftPriority = getCanvasScaleAdjustFailoverPriority(left.item);
|
||
const rightPriority = getCanvasScaleAdjustFailoverPriority(right.item);
|
||
if (leftPriority !== rightPriority) return leftPriority - rightPriority;
|
||
if (right.nameSimilarity !== left.nameSimilarity) {
|
||
return right.nameSimilarity - left.nameSimilarity;
|
||
}
|
||
if (left.sameProviderType !== right.sameProviderType) {
|
||
return left.sameProviderType ? -1 : 1;
|
||
}
|
||
return left.index - right.index;
|
||
}
|
||
const leftSameName =
|
||
left.nameSimilarity >= CANVAS_FAILOVER_SAME_NAME_THRESHOLD;
|
||
const rightSameName =
|
||
right.nameSimilarity >= CANVAS_FAILOVER_SAME_NAME_THRESHOLD;
|
||
if (leftSameName !== rightSameName) return leftSameName ? -1 : 1;
|
||
if (
|
||
params.preferLowCostSameNameProviders &&
|
||
leftSameName &&
|
||
rightSameName
|
||
) {
|
||
const leftPriority = getCanvasScaleAdjustFailoverPriority(left.item);
|
||
const rightPriority = getCanvasScaleAdjustFailoverPriority(right.item);
|
||
if (leftPriority !== rightPriority) return leftPriority - rightPriority;
|
||
if (right.nameSimilarity !== left.nameSimilarity) {
|
||
return right.nameSimilarity - left.nameSimilarity;
|
||
}
|
||
if (left.sameProviderType !== right.sameProviderType) {
|
||
return left.sameProviderType ? -1 : 1;
|
||
}
|
||
return left.index - right.index;
|
||
}
|
||
if (left.sameProviderType !== right.sameProviderType) {
|
||
return left.sameProviderType ? -1 : 1;
|
||
}
|
||
if (right.nameSimilarity !== left.nameSimilarity) {
|
||
return right.nameSimilarity - left.nameSimilarity;
|
||
}
|
||
return left.index - right.index;
|
||
});
|
||
|
||
return scoredModels[0]?.item || null;
|
||
}
|
||
|
||
export async function generateCanvasImage(
|
||
request: CanvasGenerateImageRequest
|
||
): Promise<CanvasGenerateImageResult> {
|
||
const startedAt = Date.now();
|
||
const prompt = String(request.prompt || "").trim();
|
||
if (!prompt) {
|
||
throw new Error("请输入提示词");
|
||
}
|
||
|
||
const negativePrompt =
|
||
typeof request.negativePrompt === "string" ? request.negativePrompt : undefined;
|
||
const model = typeof request.model === "string" ? request.model : undefined;
|
||
const aspectRatio =
|
||
typeof request.aspectRatio === "string" && request.aspectRatio.trim()
|
||
? request.aspectRatio.trim()
|
||
: "auto";
|
||
const requestedImageSize =
|
||
typeof request.imageSize === "string" && request.imageSize.trim()
|
||
? request.imageSize.trim()
|
||
: "2K";
|
||
const requestedImageCount =
|
||
typeof request.imageCount === "number" && Number.isFinite(request.imageCount)
|
||
? Math.max(1, Math.min(12, Math.floor(request.imageCount)))
|
||
: 1;
|
||
const promptExtend =
|
||
typeof request.promptExtend === "boolean" ? request.promptExtend : undefined;
|
||
const providerImageParams =
|
||
request.providerImageParams && typeof request.providerImageParams === "object"
|
||
? request.providerImageParams
|
||
: undefined;
|
||
const ownerKey =
|
||
typeof request.ownerKey === "string" && request.ownerKey.trim()
|
||
? request.ownerKey.trim()
|
||
: undefined;
|
||
const referenceImage =
|
||
typeof request.referenceImage === "string" ? request.referenceImage : undefined;
|
||
const referenceImages = Array.isArray(request.referenceImages)
|
||
? request.referenceImages
|
||
: [];
|
||
const debugLabel = String(request.debugLabel || "").trim() || "canvas-image";
|
||
const debugMeta =
|
||
request.debugMeta && typeof request.debugMeta === "object"
|
||
? request.debugMeta
|
||
: undefined;
|
||
const scaleAdjustModelFamily =
|
||
typeof debugMeta?.scaleAdjustModelFamily === "string"
|
||
? debugMeta.scaleAdjustModelFamily.trim()
|
||
: "";
|
||
const localEditComposite = normalizeLocalEditCompositeConfig(
|
||
request.localEditComposite
|
||
);
|
||
const scaleAdjustPatchComposite = normalizeScaleAdjustPatchCompositeConfig(
|
||
request.scaleAdjustPatchComposite
|
||
);
|
||
const scaleAdjustVerification = normalizeScaleAdjustVerificationConfig(
|
||
request.scaleAdjustVerification
|
||
);
|
||
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const imageModels = allModels.filter((m) => m.model.supportsImageGen);
|
||
|
||
if (!imageModels || imageModels.length === 0) {
|
||
throw new Error("暂无可用的图片生成模型");
|
||
}
|
||
|
||
let selectedModel = model
|
||
? imageModels.find((m) => m.model.modelId === model || m.model.id === model)
|
||
: imageModels.find((m) => m.model.isDefault) || imageModels[0];
|
||
|
||
if (!selectedModel) {
|
||
selectedModel = imageModels[0];
|
||
}
|
||
|
||
const normalizedReferenceImages = Array.from(
|
||
new Set(
|
||
[
|
||
...referenceImages,
|
||
...(referenceImage ? [referenceImage] : []),
|
||
]
|
||
.map((item) => String(item || "").trim())
|
||
.filter((url) => /^https?:\/\//i.test(url))
|
||
)
|
||
).slice(0, 12);
|
||
const requiresImageInput = normalizedReferenceImages.length > 0;
|
||
const requiresTextToImage = normalizedReferenceImages.length === 0;
|
||
|
||
let provider = selectedModel.provider;
|
||
|
||
if (
|
||
requiresImageInput &&
|
||
modelLacksReferenceImageInputSupport(selectedModel.model, selectedModel.provider)
|
||
) {
|
||
if (request.allowFailover === false) {
|
||
throw new Error("当前图片模型不支持参考图/图生图，请选择支持图像输入的生图模型。");
|
||
}
|
||
|
||
const imageInputFallback = findCanvasFailoverModel({
|
||
selectedModel,
|
||
imageModels,
|
||
failedModelRefs: getCanvasModelRefKeys(selectedModel),
|
||
preferLowCostSameNameProviders: Boolean(
|
||
debugMeta?.scaleAdjustPreferSameNameProviderFailover
|
||
),
|
||
requiresImageInput,
|
||
scaleAdjustModelFamily,
|
||
});
|
||
|
||
if (!imageInputFallback) {
|
||
throw new Error("当前任务需要参考图输入，但暂无可用的图生图模型。");
|
||
}
|
||
|
||
console.warn("[Canvas Generate Image] Fallback to reference-image capable model:", {
|
||
debugLabel,
|
||
requestedModel: selectedModel.model.modelId || selectedModel.model.id,
|
||
fallbackModel: imageInputFallback.model.modelId || imageInputFallback.model.id,
|
||
requestedProviderType: provider.type,
|
||
fallbackProviderType: imageInputFallback.provider.type,
|
||
reason: "selected-model-lacks-reference-image-support",
|
||
});
|
||
|
||
selectedModel = imageInputFallback;
|
||
provider = selectedModel.provider;
|
||
}
|
||
|
||
if (
|
||
requiresTextToImage &&
|
||
!modelSupportsTextToImageDescriptor({
|
||
providerType: selectedModel.provider.type,
|
||
modelId: selectedModel.model.modelId,
|
||
capabilities: selectedModel.model.capabilities,
|
||
defaultParams: selectedModel.model.defaultParams,
|
||
supportsImageGen: selectedModel.model.supportsImageGen,
|
||
})
|
||
) {
|
||
if (request.allowFailover === false) {
|
||
throw new Error("当前图片模型不支持纯文生图，请选择支持文生图的生图模型。");
|
||
}
|
||
|
||
const textToImageFallback = findCanvasFailoverModel({
|
||
selectedModel,
|
||
imageModels,
|
||
failedModelRefs: getCanvasModelRefKeys(selectedModel),
|
||
preferLowCostSameNameProviders: Boolean(
|
||
debugMeta?.scaleAdjustPreferSameNameProviderFailover
|
||
),
|
||
requiresTextToImage,
|
||
scaleAdjustModelFamily,
|
||
});
|
||
|
||
if (!textToImageFallback) {
|
||
throw new Error("当前任务需要纯文生图，但暂无可用的文生图模型。");
|
||
}
|
||
|
||
console.warn("[Canvas Generate Image] Fallback to text-to-image capable model:", {
|
||
debugLabel,
|
||
requestedModel: selectedModel.model.modelId || selectedModel.model.id,
|
||
fallbackModel: textToImageFallback.model.modelId || textToImageFallback.model.id,
|
||
requestedProviderType: provider.type,
|
||
fallbackProviderType: textToImageFallback.provider.type,
|
||
reason: "selected-model-lacks-text-to-image-support",
|
||
});
|
||
|
||
selectedModel = textToImageFallback;
|
||
provider = selectedModel.provider;
|
||
}
|
||
|
||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(provider.type)) {
|
||
if (request.allowFailover === false) {
|
||
throw new Error(
|
||
"当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/Grsai GPT Image/火山 Seedream/阿里/BFL/RUXA/FAL/SnapGen/OpenAI/StepFun/Runway/Krea 图片模型。"
|
||
);
|
||
}
|
||
|
||
let fallbackModel = selectedModel;
|
||
let fallbackProvider = provider;
|
||
for (const candidate of imageModels) {
|
||
const candidateProvider = candidate.provider;
|
||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(candidateProvider.type)) continue;
|
||
if (
|
||
requiresImageInput &&
|
||
modelLacksReferenceImageInputSupport(candidate.model, candidate.provider)
|
||
) {
|
||
continue;
|
||
}
|
||
if (
|
||
requiresTextToImage &&
|
||
!modelSupportsTextToImageDescriptor({
|
||
providerType: candidate.provider.type,
|
||
modelId: candidate.model.modelId,
|
||
capabilities: candidate.model.capabilities,
|
||
defaultParams: candidate.model.defaultParams,
|
||
supportsImageGen: candidate.model.supportsImageGen,
|
||
})
|
||
) {
|
||
continue;
|
||
}
|
||
fallbackModel = candidate;
|
||
fallbackProvider = candidateProvider;
|
||
break;
|
||
}
|
||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(fallbackProvider.type)) {
|
||
throw new Error(
|
||
"当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/Grsai GPT Image/火山 Seedream/阿里/BFL/RUXA/FAL/SnapGen/OpenAI/StepFun/Runway/Krea 图片模型。"
|
||
);
|
||
}
|
||
if (fallbackModel.model.id !== selectedModel.model.id) {
|
||
console.warn("[Canvas Generate Image] Fallback to supported image model:", {
|
||
requestedModel: selectedModel.model.modelId || selectedModel.model.id,
|
||
fallbackModel: fallbackModel.model.modelId || fallbackModel.model.id,
|
||
requestedProviderType: provider.type,
|
||
fallbackProviderType: fallbackProvider.type,
|
||
});
|
||
}
|
||
selectedModel = fallbackModel;
|
||
provider = fallbackProvider;
|
||
}
|
||
|
||
const normalizedImageParams = normalizeCanvasImageParamsByModel({
|
||
imageCount: requestedImageCount,
|
||
imageSize: requestedImageSize,
|
||
imageGenConfig: selectedModel.model.imageGenConfig as CanvasImageGenConfig | null | undefined,
|
||
});
|
||
const imageSize = normalizedImageParams.imageSize;
|
||
const imageCount =
|
||
scaleAdjustVerification && provider.type === "fal-image"
|
||
? Math.max(1, Math.min(4, requestedImageCount))
|
||
: normalizedImageParams.imageCount;
|
||
if (imageSize !== requestedImageSize || imageCount !== requestedImageCount) {
|
||
console.info("[Canvas Generate Image] Normalized model image params:", {
|
||
debugLabel,
|
||
requestedImageSize,
|
||
normalizedImageSize: imageSize,
|
||
requestedImageCount,
|
||
normalizedImageCount: imageCount,
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
providerType: provider.type,
|
||
});
|
||
}
|
||
|
||
const { kept: reachableReferenceImages, dropped: unreachableReferenceImages } =
|
||
await filterReachableRemoteUrls(normalizedReferenceImages, { timeoutMs: 3000 });
|
||
const effectiveReferenceImages =
|
||
reachableReferenceImages.length > 0
|
||
? reachableReferenceImages
|
||
: normalizedReferenceImages;
|
||
if (unreachableReferenceImages.length > 0) {
|
||
console.warn("[Canvas Generate Image] Local reachability check failed for some reference images:", {
|
||
count: unreachableReferenceImages.length,
|
||
urls: unreachableReferenceImages.slice(0, 6),
|
||
hosts: summarizeReferenceImageHosts(unreachableReferenceImages),
|
||
});
|
||
}
|
||
if (normalizedReferenceImages.length > 0 && reachableReferenceImages.length === 0) {
|
||
console.warn(
|
||
"[Canvas Generate Image] All reference images failed local reachability check; proceeding with original URLs.",
|
||
{
|
||
count: normalizedReferenceImages.length,
|
||
hosts: summarizeReferenceImageHosts(normalizedReferenceImages),
|
||
debugLabel,
|
||
}
|
||
);
|
||
}
|
||
|
||
console.info("[Canvas Generate Image] Started:", {
|
||
debugLabel,
|
||
promptLength: prompt.length,
|
||
aspectRatio,
|
||
imageSize,
|
||
imageCount,
|
||
requestedReferenceImageCount: normalizedReferenceImages.length,
|
||
reachableReferenceImageCount: reachableReferenceImages.length,
|
||
effectiveReferenceImageCount: effectiveReferenceImages.length,
|
||
referenceImageHosts: summarizeReferenceImageHosts(effectiveReferenceImages),
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
providerType: provider.type,
|
||
...debugMeta,
|
||
hasLocalEditComposite: Boolean(localEditComposite),
|
||
hasScaleAdjustPatchComposite: Boolean(scaleAdjustPatchComposite),
|
||
hasScaleAdjustVerification: Boolean(scaleAdjustVerification),
|
||
});
|
||
|
||
const resultPrompt =
|
||
typeof request.displayPrompt === "string" && request.displayPrompt.trim()
|
||
? request.displayPrompt.trim()
|
||
: prompt;
|
||
|
||
const messages = [
|
||
{
|
||
role: "user" as const,
|
||
content:
|
||
effectiveReferenceImages.length > 0
|
||
? [
|
||
{ type: "text" as const, text: prompt },
|
||
...effectiveReferenceImages.map((url) => ({
|
||
type: "image_url" as const,
|
||
image_url: { url },
|
||
})),
|
||
]
|
||
: prompt,
|
||
},
|
||
];
|
||
|
||
const imageParams = {
|
||
...(providerImageParams || {}),
|
||
aspectRatio,
|
||
safetyTolerance: request.falSafetyTolerance,
|
||
imageSize,
|
||
imageCount: 1,
|
||
negativePrompt,
|
||
promptExtend,
|
||
systemPrompt: request.falSystemPrompt,
|
||
thinkingLevel: request.falThinkingLevel,
|
||
};
|
||
|
||
let executeGenerateBatch:
|
||
| ((batchImageCount: number) => Promise<string>)
|
||
| null = null;
|
||
|
||
if (provider.type === "gemini" || provider.type === "grsai-gpt-image") {
|
||
const geminiProvider = createGeminiImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = geminiProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "alibaba-image") {
|
||
const alibabaProvider = createAlibabaImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = alibabaProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "bfl-image") {
|
||
const bflProvider = createBflImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = bflProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (_batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams,
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "ruxa-image") {
|
||
const ruxaProvider = createRuxaImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const runtimeModelId = resolveModelRuntimeId(selectedModel.model, {
|
||
providerType: provider.type,
|
||
});
|
||
const modelInstance = ruxaProvider.chat(runtimeModelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
imageGenConfig: selectedModel.model.imageGenConfig || null,
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "fal-image") {
|
||
const falProvider = createFalImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = falProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "snapgen-image") {
|
||
const snapgenProvider = createSnapgenImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const runtimeModelId = resolveModelRuntimeId(selectedModel.model, {
|
||
providerType: provider.type,
|
||
});
|
||
const modelInstance = snapgenProvider.chat(runtimeModelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
imageGenConfig: selectedModel.model.imageGenConfig || null,
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "openai-image") {
|
||
const openAIImageProvider = createOpenAIImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = openAIImageProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "stepfun-image") {
|
||
const stepfunProvider = createStepfunImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = stepfunProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (_batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams,
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "midjourney") {
|
||
const midjourneyProvider = createMidjourney({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = midjourneyProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "runway") {
|
||
const runwayProvider = createRunway({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = runwayProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (_batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams,
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "krea") {
|
||
const kreaProvider = createKrea({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = kreaProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "volcano-vision") {
|
||
const volcanoVisionProvider = createVolcanoVision({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = volcanoVisionProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else {
|
||
throw new Error(
|
||
`不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/Grsai GPT Image/火山 Seedream/阿里/BFL/RUXA/FAL/SnapGen/OpenAI/StepFun/Midjourney/Runway/Krea 图片模型`
|
||
);
|
||
}
|
||
|
||
const providerBatchLimit = getProviderBatchLimit(provider.type);
|
||
const targetImageCount = Math.max(1, imageCount);
|
||
const collectedImages: ParsedImage[] = [];
|
||
const maxBatches = Math.max(1, Math.ceil(targetImageCount / providerBatchLimit));
|
||
const resolvedModelKey = normalizeCanvasImageModelKey(
|
||
selectedModel.model.modelId || selectedModel.model.id || model
|
||
);
|
||
const slot = await acquireCanvasImageModelSlot({
|
||
modelKey: resolvedModelKey,
|
||
ownerKey,
|
||
debugLabel,
|
||
debugMeta: {
|
||
selectedModelId: selectedModel.model.modelId,
|
||
selectedModelRef: selectedModel.model.id,
|
||
providerType: provider.type,
|
||
...debugMeta,
|
||
},
|
||
onQueued: request.onQueueStatusChange
|
||
? async (queueStatus) => {
|
||
await request.onQueueStatusChange?.({
|
||
...queueStatus,
|
||
modelName: selectedModel.model.displayName || null,
|
||
modelRef: selectedModel.model.id || null,
|
||
});
|
||
}
|
||
: undefined,
|
||
});
|
||
|
||
if (slot.waitMs > 0 && request.onQueueStatusChange) {
|
||
const runningStatus = await getCanvasImageModelQueueStatus({
|
||
modelKey: resolvedModelKey,
|
||
});
|
||
await request.onQueueStatusChange({
|
||
...runningStatus,
|
||
status: "running",
|
||
queuePosition: 0,
|
||
estimatedWaitMs: 0,
|
||
modelName: selectedModel.model.displayName || null,
|
||
modelRef: selectedModel.model.id || null,
|
||
});
|
||
}
|
||
|
||
let caughtError: unknown = null;
|
||
|
||
try {
|
||
for (let batchIndex = 0; batchIndex < maxBatches; batchIndex += 1) {
|
||
const remaining = targetImageCount - collectedImages.length;
|
||
if (remaining <= 0) break;
|
||
const batchImageCount = Math.min(providerBatchLimit, remaining);
|
||
const batchStartedAt = Date.now();
|
||
console.info("[Canvas Generate Image] Batch started:", {
|
||
debugLabel,
|
||
batchIndex: batchIndex + 1,
|
||
totalBatches: maxBatches,
|
||
batchImageCount,
|
||
currentCollectedCount: collectedImages.length,
|
||
providerType: provider.type,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
...debugMeta,
|
||
});
|
||
const imageMarkdown = await executeGenerateBatch(batchImageCount);
|
||
const parsedImages = parseImagesFromMarkdown(imageMarkdown);
|
||
console.info("[Canvas Generate Image] Batch completed:", {
|
||
debugLabel,
|
||
batchIndex: batchIndex + 1,
|
||
totalBatches: maxBatches,
|
||
batchImageCount,
|
||
parsedImageCount: parsedImages.length,
|
||
elapsedMs: Date.now() - batchStartedAt,
|
||
...debugMeta,
|
||
});
|
||
if (parsedImages.length === 0) {
|
||
if (collectedImages.length > 0) {
|
||
break;
|
||
}
|
||
throw new Error("图片生成失败");
|
||
}
|
||
collectedImages.push(...parsedImages.slice(0, batchImageCount));
|
||
}
|
||
|
||
if (collectedImages.length === 0) {
|
||
throw new Error("图片生成失败");
|
||
}
|
||
|
||
const baseImages = collectedImages.slice(0, targetImageCount);
|
||
const compositedImages = await applyLocalEditCompositeToImages({
|
||
composite: localEditComposite,
|
||
debugLabel,
|
||
images: baseImages,
|
||
});
|
||
const scaleAdjustPatchCompositedImages =
|
||
await applyScaleAdjustPatchCompositeToImages({
|
||
composite: scaleAdjustPatchComposite,
|
||
debugLabel,
|
||
images: compositedImages,
|
||
});
|
||
|
||
const finalImages = await applyScaleAdjustVerificationToImages({
|
||
debugLabel,
|
||
images: scaleAdjustPatchCompositedImages,
|
||
verification: scaleAdjustVerification,
|
||
});
|
||
console.info("[Canvas Generate Image] Completed:", {
|
||
debugLabel,
|
||
elapsedMs: Date.now() - startedAt,
|
||
waitMs: slot.waitMs,
|
||
singleModelConcurrency: slot.maxConcurrent,
|
||
finalImageCount: finalImages.length,
|
||
providerType: provider.type,
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
...debugMeta,
|
||
});
|
||
|
||
await recordAiModelHealthSuccess({
|
||
model: selectedModel.model,
|
||
}).catch((healthError) => {
|
||
console.warn("[Canvas Generate Image] Record model health success failed:", {
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
error: healthError instanceof Error ? healthError.message : String(healthError),
|
||
});
|
||
});
|
||
|
||
recordCanvasImageModelExecutionDuration(
|
||
resolvedModelKey,
|
||
Date.now() - startedAt
|
||
);
|
||
|
||
return {
|
||
images: finalImages,
|
||
model: selectedModel.model.displayName,
|
||
modelRef: selectedModel.model.id,
|
||
prompt: resultPrompt,
|
||
};
|
||
} catch (error) {
|
||
console.error("[Canvas Generate Image] Failed:", {
|
||
debugLabel,
|
||
elapsedMs: Date.now() - startedAt,
|
||
waitMs: slot.waitMs,
|
||
singleModelConcurrency: slot.maxConcurrent,
|
||
providerType: provider.type,
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
error: error instanceof Error ? error.message : String(error),
|
||
...debugMeta,
|
||
});
|
||
caughtError = error;
|
||
} finally {
|
||
await slot.release();
|
||
}
|
||
|
||
const retryVariants = Array.isArray(request.retryVariants)
|
||
? request.retryVariants
|
||
: [];
|
||
const nextRetryVariantIndex =
|
||
typeof request.internalRetryVariantIndex === "number"
|
||
? request.internalRetryVariantIndex + 1
|
||
: 0;
|
||
const retryVariant = retryVariants[nextRetryVariantIndex];
|
||
if (retryVariant) {
|
||
const {
|
||
debugMeta: retryVariantDebugMeta,
|
||
label: retryVariantLabel,
|
||
...retryOverrides
|
||
} = retryVariant;
|
||
const retryDebugMeta = {
|
||
...debugMeta,
|
||
...(retryVariantDebugMeta || {}),
|
||
retryFromError:
|
||
caughtError instanceof Error ? caughtError.message : String(caughtError),
|
||
retryVariantIndex: nextRetryVariantIndex,
|
||
retryVariantLabel: retryVariantLabel || `variant-${nextRetryVariantIndex + 1}`,
|
||
};
|
||
console.warn("[Canvas Generate Image] Retrying with prompt/reference variant:", {
|
||
...retryDebugMeta,
|
||
debugLabel,
|
||
retryVariantIndex: nextRetryVariantIndex,
|
||
retryVariantLabel: retryVariantLabel || null,
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
error: caughtError instanceof Error ? caughtError.message : String(caughtError),
|
||
});
|
||
|
||
return await generateCanvasImage({
|
||
...request,
|
||
...retryOverrides,
|
||
debugMeta: retryDebugMeta,
|
||
internalRetryVariantIndex: nextRetryVariantIndex,
|
||
retryVariants,
|
||
});
|
||
}
|
||
|
||
const scaleAdjustRejection =
|
||
caughtError instanceof ScaleAdjustOutputRejectedError ? caughtError : null;
|
||
const scaleAdjustRejected = Boolean(scaleAdjustRejection);
|
||
|
||
if (scaleAdjustRejected) {
|
||
console.info("[Canvas Generate Image] Skip model health failure for scale adjust rejection:", {
|
||
debugLabel,
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
error: caughtError instanceof Error ? caughtError.message : String(caughtError),
|
||
});
|
||
} else {
|
||
await recordAiModelHealthFailure({
|
||
model: selectedModel.model,
|
||
provider,
|
||
error: caughtError,
|
||
context: debugLabel,
|
||
elapsedMs: Date.now() - startedAt,
|
||
metadata: {
|
||
aspectRatio,
|
||
imageSize,
|
||
imageCount,
|
||
requestedReferenceImageCount: normalizedReferenceImages.length,
|
||
reachableReferenceImageCount: reachableReferenceImages.length,
|
||
effectiveReferenceImageCount: effectiveReferenceImages.length,
|
||
droppedReferenceImageCount: unreachableReferenceImages.length,
|
||
...debugMeta,
|
||
},
|
||
}).catch((healthError) => {
|
||
console.warn("[Canvas Generate Image] Record model health failure failed:", {
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
error: healthError instanceof Error ? healthError.message : String(healthError),
|
||
});
|
||
});
|
||
}
|
||
|
||
const failedModelRefs = Array.from(
|
||
new Set([
|
||
...(Array.isArray(request.internalFailoverTriedModelRefs)
|
||
? request.internalFailoverTriedModelRefs
|
||
: []),
|
||
selectedModel.model.id,
|
||
selectedModel.model.modelId,
|
||
].filter(Boolean))
|
||
);
|
||
const allowFailover = request.allowFailover !== false;
|
||
const shouldFailover =
|
||
shouldFailoverAiModelError(caughtError) ||
|
||
(!scaleAdjustRejected &&
|
||
Boolean(debugMeta?.detailFixedModelApplied) &&
|
||
Boolean(caughtError));
|
||
const failureClassification = classifyAiModelHealthFailure(caughtError);
|
||
const failedProviderTypes = Array.from(
|
||
new Set([
|
||
...(Array.isArray(request.internalFailoverTriedProviderTypes)
|
||
? request.internalFailoverTriedProviderTypes
|
||
: []),
|
||
...(shouldFailover &&
|
||
failureClassification.reason !== "non_provider_failure" &&
|
||
failureClassification.reason !== "user_cancelled"
|
||
? [provider.type]
|
||
: []),
|
||
].filter(Boolean))
|
||
);
|
||
const fallbackModel = allowFailover && shouldFailover
|
||
? findCanvasFailoverModel({
|
||
selectedModel,
|
||
imageModels,
|
||
failedModelRefs,
|
||
failedProviderTypes,
|
||
preferLowCostSameNameProviders: Boolean(
|
||
debugMeta?.scaleAdjustPreferSameNameProviderFailover
|
||
),
|
||
requiresImageInput: normalizedReferenceImages.length > 0,
|
||
requiresTextToImage: normalizedReferenceImages.length === 0,
|
||
scaleAdjustModelFamily,
|
||
})
|
||
: null;
|
||
|
||
if (fallbackModel) {
|
||
console.warn("[Canvas Generate Image] Failover to alternate image model:", {
|
||
debugLabel,
|
||
failedModelRef: selectedModel.model.id,
|
||
failedModelId: selectedModel.model.modelId,
|
||
fallbackModelRef: fallbackModel.model.id,
|
||
fallbackModelId: fallbackModel.model.modelId,
|
||
fallbackProviderType: fallbackModel.provider.type,
|
||
error: caughtError instanceof Error ? caughtError.message : String(caughtError),
|
||
...debugMeta,
|
||
});
|
||
|
||
return await generateCanvasImage({
|
||
...request,
|
||
model: fallbackModel.model.id,
|
||
internalFailoverTriedModelRefs: failedModelRefs,
|
||
internalFailoverTriedProviderTypes: failedProviderTypes,
|
||
debugMeta: {
|
||
...debugMeta,
|
||
failoverFromModelId: selectedModel.model.modelId,
|
||
failoverFromModelRef: selectedModel.model.id,
|
||
},
|
||
});
|
||
}
|
||
|
||
if (scaleAdjustRejection?.images?.length) {
|
||
console.warn("[Canvas Generate Image] Rejecting scale adjust candidate because no effective edit was detected:", {
|
||
debugLabel,
|
||
imageCount: scaleAdjustRejection.images.length,
|
||
rejectedImageUrls: scaleAdjustRejection.images.map((image) => ({
|
||
originalUrl: image.originalUrl,
|
||
thumbnailUrl: image.thumbnailUrl,
|
||
})),
|
||
reason: scaleAdjustRejection.reason,
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
...debugMeta,
|
||
});
|
||
if (scaleAdjustVerification?.allowDeterministicFallback) {
|
||
try {
|
||
const fallbackImages = await buildScaleAdjustDeterministicFallbackImages({
|
||
debugLabel,
|
||
verification: scaleAdjustVerification,
|
||
});
|
||
if (fallbackImages.length > 0) {
|
||
const verifiedFallbackImages = await applyScaleAdjustVerificationToImages({
|
||
debugLabel,
|
||
images: fallbackImages,
|
||
verification: {
|
||
...scaleAdjustVerification,
|
||
rejectFailedCandidates: true,
|
||
},
|
||
});
|
||
console.warn("[Canvas Generate Image] Returning deterministic scale adjust fallback:", {
|
||
debugLabel,
|
||
fallbackImageCount: verifiedFallbackImages.length,
|
||
rejectedReason: scaleAdjustRejection.reason,
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
...debugMeta,
|
||
});
|
||
return {
|
||
images: verifiedFallbackImages,
|
||
model: selectedModel.model.displayName,
|
||
modelRef: selectedModel.model.id,
|
||
prompt: resultPrompt,
|
||
};
|
||
}
|
||
} catch (fallbackError) {
|
||
console.warn("[Canvas Generate Image] Scale adjust deterministic fallback failed:", {
|
||
debugLabel,
|
||
error:
|
||
fallbackError instanceof Error
|
||
? fallbackError.message
|
||
: String(fallbackError),
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
...debugMeta,
|
||
});
|
||
}
|
||
}
|
||
throw caughtError;
|
||
}
|
||
|
||
throw caughtError;
|
||
}
|