import { NextRequest, NextResponse } from "next/server"; import sharp from "sharp"; import { auth } from "@/app/(auth)/auth"; import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import { createCanvasImageTask, enqueueCanvasImageTask, getCanvasImageTaskForUser, } from "@/lib/canvas/image-task-queue"; import type { CanvasGenerateImageRequest } from "@/lib/canvas/generate-image"; import { getEnabledAiModelsWithProvider, getPointsSystemConfig, getSiteSettings, } from "@/lib/db/queries"; import { getImageToolCreditCost } from "@/lib/points/config"; import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; import { uploadFile } from "@/lib/storage/service"; export const runtime = "nodejs"; export const maxDuration = 120; type LocalEditRect = { x?: number; y?: number; width?: number; height?: number; }; type LocalEditRegionInput = { id?: string; index?: number; color?: string; prompt?: string; rect?: LocalEditRect; targetRect?: LocalEditRect; referenceImages?: Array<{ id?: string; url?: string; name?: string; source?: string; }>; }; type LocalEditRequest = { sourceImageUrl?: string; sourceNodeId?: string; regions?: LocalEditRegionInput[]; model?: string; imageSize?: string; aspectRatio?: string; operation?: string; scalePercent?: number; }; type NormalizedLocalEditRegion = { id: string; index: number; color: string; colorName: string; prompt: string; referenceImages: Array<{ id: string; url: string; name?: string; source: "upload" | "canvas"; }>; rect: { x: number; y: number; width: number; height: number; }; targetRect?: { x: number; y: number; width: number; height: number; } | null; pixelRect: { left: number; top: number; width: number; height: number; }; targetPixelRect?: { left: number; top: number; width: number; height: number; } | null; }; const MAX_LOCAL_EDIT_REGIONS = 8; const MAX_REFERENCE_IMAGES_PER_REGION = 3; const MAX_SCALE_ADJUST_CROP_REFERENCES = 4; const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18; const SCALE_ADJUST_CROP_MAX_EDGE = 1536; const MIN_REGION_EDGE_PX = 24; const SCALE_ADJUST_OPERATION = "scale-adjust"; type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION; const MIN_SCALE_ADJUST_PERCENT = -80; const MAX_SCALE_ADJUST_PERCENT = 200; type ScaleAdjustPercent = number; type ScaleAdjustCropReference = { regionIndex: number; url: string; }; type AiModelWithProviderForSelection = Awaited< ReturnType >[number]; const DEFAULT_REGION_COLORS = [ "#ff3b30", "#1677ff", "#00c853", "#ff9500", "#af52de", "#00bcd4", "#ff2d55", "#8d6e63", ]; const SUPPORTED_LOCAL_EDIT_ASPECT_RATIOS = [ "1:1", "1:4", "1:8", "2:3", "3:2", "3:4", "4:1", "4:3", "4:5", "5:4", "8:1", "9:16", "16:9", "21:9", ] as const; function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); } function parseAspectRatioValue(ratio: string) { const [width, height] = ratio.split(":").map((part) => Number(part)); if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { return 1; } return width / height; } function inferClosestAspectRatio(width: number, height: number) { if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { return "1:1"; } const sourceRatio = width / height; let closest: (typeof SUPPORTED_LOCAL_EDIT_ASPECT_RATIOS)[number] = "1:1"; let closestDiff = Number.POSITIVE_INFINITY; for (const candidate of SUPPORTED_LOCAL_EDIT_ASPECT_RATIOS) { const diff = Math.abs(parseAspectRatioValue(candidate) - sourceRatio); if (diff < closestDiff) { closest = candidate; closestDiff = diff; } } return closest; } function resolveLocalEditAspectRatio(input: unknown, sourceWidth: number, sourceHeight: number) { const requested = String(input || "").trim(); if ( requested && requested.toLowerCase() !== "auto" && (SUPPORTED_LOCAL_EDIT_ASPECT_RATIOS as readonly string[]).includes(requested) ) { return requested; } return inferClosestAspectRatio(sourceWidth, sourceHeight); } function toArrayBuffer(buffer: Buffer) { return buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; } function escapeXml(value: string) { return value .replace(/&/g, "&") .replace(//g, ">") .replace(/"/g, """) .replace(/'/g, "'"); } function normalizeHexColor(input: unknown, fallback: string) { const raw = String(input || "").trim(); const expanded = raw.replace(/^#([0-9a-f])([0-9a-f])([0-9a-f])$/i, "#$1$1$2$2$3$3"); return /^#[0-9a-f]{6}$/i.test(expanded) ? expanded.toLowerCase() : fallback; } function findModelSelectionEntry( models: AiModelWithProviderForSelection[], modelRef: string ) { const normalizedRef = String(modelRef || "").trim(); if (!normalizedRef) return null; return ( models.find( (entry) => entry.model.id === normalizedRef || entry.model.modelId === normalizedRef ) || null ); } function isScaleAdjustDedicatedDetailModel( entry: AiModelWithProviderForSelection | null | undefined ) { if (!entry?.model?.supportsImageGen || entry.model.supportsVision === false) { return false; } const label = `${entry.model.displayName || ""} ${entry.model.modelId || ""}` .toLowerCase() .replace(/\s+/g, ""); return label.includes("详情专线"); } function resolveScaleAdjustPreferredModelId(params: { configuredModelId: string; requestedModelId: string; models: AiModelWithProviderForSelection[]; }) { const requestedModel = findModelSelectionEntry( params.models, params.requestedModelId ); if (isScaleAdjustDedicatedDetailModel(requestedModel)) { return requestedModel?.model.id || params.requestedModelId; } const configuredModel = findModelSelectionEntry( params.models, params.configuredModelId ); if (isScaleAdjustDedicatedDetailModel(configuredModel)) { return configuredModel?.model.id || params.configuredModelId; } const autoDetailModel = params.models.find(isScaleAdjustDedicatedDetailModel); if (autoDetailModel) { return autoDetailModel.model.id; } return params.configuredModelId || params.requestedModelId; } function normalizeHttpImageUrl(input: unknown) { const raw = String(input || "").trim(); if (!/^https?:\/\//i.test(raw)) return ""; return raw; } function normalizeRegionReferenceImages( region: LocalEditRegionInput, itemIndex: number ): NormalizedLocalEditRegion["referenceImages"] { if (!Array.isArray(region.referenceImages)) return []; return region.referenceImages .map((reference, referenceIndex) => { const url = normalizeHttpImageUrl(reference?.url); if (!url) return null; const name = typeof reference?.name === "string" && reference.name.trim() ? reference.name.trim().slice(0, 80) : ""; return { id: typeof reference?.id === "string" && reference.id.trim() ? reference.id.trim() : `region-${itemIndex + 1}-reference-${referenceIndex + 1}`, url, ...(name ? { name } : {}), source: reference?.source === "canvas" ? "canvas" as const : "upload" as const, }; }) .filter((item): item is NormalizedLocalEditRegion["referenceImages"][number] => Boolean(item) ) .slice(0, MAX_REFERENCE_IMAGES_PER_REGION); } function resolveLocalEditOperation(input: unknown) { return String(input || "").trim() === SCALE_ADJUST_OPERATION ? SCALE_ADJUST_OPERATION : "local-edit"; } function normalizeScaleAdjustPercent(input: unknown): ScaleAdjustPercent { const value = Math.round(Number(input)); if ( Number.isFinite(value) && value !== 0 && value >= MIN_SCALE_ADJUST_PERCENT && value <= MAX_SCALE_ADJUST_PERCENT ) { return value; } throw new Error("请选择有效的比例调整幅度"); } function buildLocalEditDisplayPrompt(params: { operation: LocalEditOperation; scalePercent: ScaleAdjustPercent | null; }) { if (params.operation === SCALE_ADJUST_OPERATION) { return "比例调整:按目标框调整"; } return "局部编辑处理"; } function getColorName(hex: string) { const normalized = hex.toLowerCase(); const names: Record = { "#ff3b30": "红色", "#1677ff": "蓝色", "#00c853": "绿色", "#ff9500": "橙色", "#af52de": "紫色", "#00bcd4": "青色", "#ff2d55": "玫红色", "#8d6e63": "棕色", }; return names[normalized] || `${normalized} 色`; } function normalizeRect(input: LocalEditRect | undefined) { if (!input) return null; const x = Number(input.x); const y = Number(input.y); const width = Number(input.width); const height = Number(input.height); if (![x, y, width, height].every(Number.isFinite)) return null; const left = clamp(Math.min(x, x + width), 0, 1); const top = clamp(Math.min(y, y + height), 0, 1); const right = clamp(Math.max(x, x + width), 0, 1); const bottom = clamp(Math.max(y, y + height), 0, 1); const normalizedWidth = clamp(right - left, 0, 1 - left); const normalizedHeight = clamp(bottom - top, 0, 1 - top); if (normalizedWidth <= 0 || normalizedHeight <= 0) return null; return { x: left, y: top, width: normalizedWidth, height: normalizedHeight, }; } function rectToPixels( rect: NonNullable>, sourceWidth: number, sourceHeight: number ) { const left = clamp(Math.round(rect.x * sourceWidth), 0, Math.max(0, sourceWidth - 1)); const top = clamp(Math.round(rect.y * sourceHeight), 0, Math.max(0, sourceHeight - 1)); const width = clamp(Math.round(rect.width * sourceWidth), 1, sourceWidth - left); const height = clamp(Math.round(rect.height * sourceHeight), 1, sourceHeight - top); return { left, top, width, height }; } function scaleRectAroundCenter( rect: NormalizedLocalEditRegion["rect"], scalePercent: ScaleAdjustPercent ) { const requestedScale = Math.max(0.1, 1 + scalePercent / 100); const maxUniformScale = Math.max( 0.1, Math.min( rect.width > 0 ? 1 / rect.width : requestedScale, rect.height > 0 ? 1 / rect.height : requestedScale ) ); const scale = Math.min(requestedScale, maxUniformScale); const targetWidth = Math.min(1, rect.width * scale); const targetHeight = Math.min(1, rect.height * scale); const centerX = rect.x + rect.width / 2; const centerY = rect.y + rect.height / 2; const x = clamp(centerX - targetWidth / 2, 0, Math.max(0, 1 - targetWidth)); const y = clamp(centerY - targetHeight / 2, 0, Math.max(0, 1 - targetHeight)); return { x, y, width: targetWidth, height: targetHeight, }; } function getScaleAdjustTargetRect( region: NormalizedLocalEditRegion, scalePercent?: ScaleAdjustPercent | null ) { if (region.targetRect) return region.targetRect; if (typeof scalePercent === "number" && Number.isFinite(scalePercent)) { return scaleRectAroundCenter(region.rect, scalePercent); } return region.rect; } function expandRectWithMargins( rect: NormalizedLocalEditRegion["rect"], marginX: number, marginY: number ) { const left = clamp(rect.x - marginX, 0, 1); const top = clamp(rect.y - marginY, 0, 1); const right = clamp(rect.x + rect.width + marginX, left + 0.001, 1); const bottom = clamp(rect.y + rect.height + marginY, top + 0.001, 1); return { x: left, y: top, width: Math.max(0.001, right - left), height: Math.max(0.001, bottom - top), }; } function getRectUnion( leftRect: NormalizedLocalEditRegion["rect"], rightRect: NormalizedLocalEditRegion["rect"] ) { const left = Math.min(leftRect.x, rightRect.x); const top = Math.min(leftRect.y, rightRect.y); const right = Math.max( leftRect.x + leftRect.width, rightRect.x + rightRect.width ); const bottom = Math.max( leftRect.y + leftRect.height, rightRect.y + rightRect.height ); return { x: clamp(left, 0, 1), y: clamp(top, 0, 1), width: Math.max(0.001, clamp(right, 0, 1) - clamp(left, 0, 1)), height: Math.max(0.001, clamp(bottom, 0, 1) - clamp(top, 0, 1)), }; } function rectsDifferMeaningfully( leftRect: NormalizedLocalEditRegion["rect"], rightRect: NormalizedLocalEditRegion["rect"], epsilon = 0.0005 ) { return ( Math.abs(leftRect.x - rightRect.x) > epsilon || Math.abs(leftRect.y - rightRect.y) > epsilon || Math.abs(leftRect.width - rightRect.width) > epsilon || Math.abs(leftRect.height - rightRect.height) > epsilon ); } function getScaleAdjustWorkspaceRect( sourceRect: NormalizedLocalEditRegion["rect"], targetRect: NormalizedLocalEditRegion["rect"], scalePercent?: ScaleAdjustPercent | null ) { const sourceWidth = Math.max(0.0001, sourceRect.width); const sourceHeight = Math.max(0.0001, sourceRect.height); const widthDeltaRatio = Math.abs(targetRect.width / sourceWidth - 1); const heightDeltaRatio = Math.abs(targetRect.height / sourceHeight - 1); const areaDeltaRatio = Math.abs( Math.sqrt( Math.max(0.0001, targetRect.width * targetRect.height) / Math.max(0.0001, sourceWidth * sourceHeight) ) - 1 ); const sourceCenterX = sourceRect.x + sourceRect.width / 2; const sourceCenterY = sourceRect.y + sourceRect.height / 2; const targetCenterX = targetRect.x + targetRect.width / 2; const targetCenterY = targetRect.y + targetRect.height / 2; const movementRatioX = Math.abs(targetCenterX - sourceCenterX) / sourceWidth; const movementRatioY = Math.abs(targetCenterY - sourceCenterY) / sourceHeight; const explicitChangeRatio = Math.max( widthDeltaRatio, heightDeltaRatio, areaDeltaRatio, movementRatioX, movementRatioY ); const fallbackScaleRatio = typeof scalePercent === "number" && Number.isFinite(scalePercent) ? Math.min(1, Math.abs(scalePercent) / 100) : 0; const changeRatio = Math.max(explicitChangeRatio, fallbackScaleRatio); const unionRect = getRectUnion(sourceRect, targetRect); if (changeRatio <= 0 && !rectsDifferMeaningfully(unionRect, targetRect)) { return targetRect; } const marginX = Math.min( 0.16, Math.max( 0.018, unionRect.width * 0.04, sourceRect.width * changeRatio * 0.24 ) ); const marginY = Math.min( 0.16, Math.max( 0.018, unionRect.height * 0.04, sourceRect.height * changeRatio * 0.24 ) ); return expandRectWithMargins(unionRect, marginX, marginY); } function buildScaleAdjustCompositeRegions( regions: NormalizedLocalEditRegion[], scalePercent?: ScaleAdjustPercent | null ) { return regions.flatMap((region) => { const targetRect = getScaleAdjustTargetRect(region, scalePercent); const workspaceRect = getScaleAdjustWorkspaceRect( region.rect, targetRect, scalePercent ); return rectsDifferMeaningfully(workspaceRect, targetRect) && rectsDifferMeaningfully(workspaceRect, region.rect) ? [region.rect, targetRect, workspaceRect] : [region.rect, targetRect]; }); } function expandPixelRectForCrop(params: { height: number; pixelRect: NormalizedLocalEditRegion["pixelRect"]; width: number; }) { const { height, pixelRect, width } = params; const padX = Math.round(pixelRect.width * SCALE_ADJUST_CROP_PADDING_RATIO); const padY = Math.round(pixelRect.height * SCALE_ADJUST_CROP_PADDING_RATIO); const left = clamp(pixelRect.left - padX, 0, Math.max(0, width - 1)); const top = clamp(pixelRect.top - padY, 0, Math.max(0, height - 1)); const right = clamp( pixelRect.left + pixelRect.width + padX, left + 1, width ); const bottom = clamp( pixelRect.top + pixelRect.height + padY, top + 1, height ); return { left, top, width: Math.max(1, right - left), height: Math.max(1, bottom - top), }; } async function createScaleAdjustCropReferences(params: { regions: NormalizedLocalEditRegion[]; sourceBuffer: Buffer; sourceHeight: number; sourceWidth: number; }): Promise { const references: ScaleAdjustCropReference[] = []; const regionsForCrop = params.regions.slice(0, MAX_SCALE_ADJUST_CROP_REFERENCES); for (const region of regionsForCrop) { const cropBox = expandPixelRectForCrop({ height: params.sourceHeight, pixelRect: region.pixelRect, width: params.sourceWidth, }); const cropBuffer = await sharp(params.sourceBuffer, { failOn: "none" }) .extract(cropBox) .resize({ fit: "inside", height: SCALE_ADJUST_CROP_MAX_EDGE, withoutEnlargement: true, width: SCALE_ADJUST_CROP_MAX_EDGE, }) .png({ compressionLevel: 6 }) .toBuffer(); const upload = await uploadFile( `canvas-scale-adjust-crop-r${region.index}-${Date.now()}-${Math.random() .toString(36) .slice(2, 8)}.png`, toArrayBuffer(cropBuffer) ); references.push({ regionIndex: region.index, url: upload.url, }); } return references; } function wrapText(text: string, maxChars: number, maxLines = 3) { const compact = text.replace(/\s+/g, " ").trim(); if (!compact) return []; const lines: string[] = []; let remaining = compact; while (remaining.length > 0 && lines.length < maxLines) { if (remaining.length <= maxChars) { lines.push(remaining); break; } lines.push(remaining.slice(0, maxChars)); remaining = remaining.slice(maxChars); } if (remaining.length > 0 && lines.length >= maxLines) { lines[lines.length - 1] = `${lines[lines.length - 1].slice(0, Math.max(1, maxChars - 1))}…`; } return lines; } function normalizeRegions( rawRegions: LocalEditRegionInput[] | undefined, sourceWidth: number, sourceHeight: number, options: { requirePrompt?: boolean; fallbackPrompt?: string } = {} ): NormalizedLocalEditRegion[] { if (!Array.isArray(rawRegions) || rawRegions.length === 0) { throw new Error("请先框选至少一个局部编辑区域"); } if (rawRegions.length > MAX_LOCAL_EDIT_REGIONS) { throw new Error(`局部编辑最多支持 ${MAX_LOCAL_EDIT_REGIONS} 个区域`); } return rawRegions.map((region, itemIndex) => { const rect = normalizeRect(region.rect); if (!rect) { throw new Error(`区域 ${itemIndex + 1} 缺少有效框选范围`); } const targetRect = normalizeRect(region.targetRect); const prompt = String(region.prompt || "").trim(); const resolvedPrompt = prompt || String(options.fallbackPrompt || "").trim(); if (options.requirePrompt !== false && !resolvedPrompt) { throw new Error(`区域 ${itemIndex + 1} 请填写局部编辑提示词`); } const pixelRect = rectToPixels(rect, sourceWidth, sourceHeight); const targetPixelRect = targetRect ? rectToPixels(targetRect, sourceWidth, sourceHeight) : null; if (pixelRect.width < MIN_REGION_EDGE_PX || pixelRect.height < MIN_REGION_EDGE_PX) { throw new Error(`区域 ${itemIndex + 1} 过小,请重新框选更大的范围`); } const color = normalizeHexColor( region.color, DEFAULT_REGION_COLORS[itemIndex % DEFAULT_REGION_COLORS.length] ); return { id: String(region.id || `region-${itemIndex + 1}`), index: itemIndex + 1, color, colorName: getColorName(color), prompt: resolvedPrompt.slice(0, 300), referenceImages: normalizeRegionReferenceImages(region, itemIndex), rect, targetRect, pixelRect, targetPixelRect, }; }); } function buildAnnotationSvg(params: { width: number; height: number; regions: NormalizedLocalEditRegion[]; hidePromptText?: boolean; }) { const { width, height, regions } = params; const minSide = Math.max(1, Math.min(width, height)); const strokeWidth = Math.max(4, Math.round(minSide * 0.006)); const fontSize = Math.max(24, Math.round(minSide * 0.026)); const smallFontSize = Math.max(18, Math.round(fontSize * 0.72)); const padX = Math.round(fontSize * 0.55); const padY = Math.round(fontSize * 0.34); const gap = Math.round(fontSize * 0.34); const radius = Math.round(fontSize * 0.5); const regionSvg = regions .map((region) => { const { left, top, width: rectWidth, height: rectHeight } = region.pixelRect; const labelText = params.hidePromptText ? `区域${region.index} 目标主体` : `区域${region.index} ${region.prompt}`; const maxChars = Math.max(8, Math.min(24, Math.floor((width * 0.42) / Math.max(1, fontSize * 0.58)))); const lines = wrapText(labelText, maxChars, 3); const labelWidth = Math.min( Math.max( rectWidth, ...lines.map((line) => Math.ceil(line.length * fontSize * 0.62) + padX * 2) ), Math.max(180, width - 16) ); const labelHeight = lines.length * Math.round(fontSize * 1.24) + padY * 2; const preferBelow = top + rectHeight + gap + labelHeight <= height - 8; const labelX = clamp(left, 8, Math.max(8, width - labelWidth - 8)); const labelY = preferBelow ? top + rectHeight + gap : clamp(top - gap - labelHeight, 8, Math.max(8, height - labelHeight - 8)); const badgeSize = Math.max(34, Math.round(fontSize * 1.35)); const badgeCx = clamp(left, badgeSize / 2 + 6, width - badgeSize / 2 - 6); const badgeCy = clamp(top, badgeSize / 2 + 6, height - badgeSize / 2 - 6); const textLines = lines .map( (line, lineIndex) => `${escapeXml(line)}` ) .join(""); return ` ${region.index} ${textLines} `; }) .join(""); return Buffer.from(` ${regionSvg} `); } function buildScaleAdjustAnnotationSvg(params: { width: number; height: number; regions: NormalizedLocalEditRegion[]; scalePercent?: ScaleAdjustPercent | null; }) { const { width, height, regions, scalePercent } = params; const minSide = Math.max(1, Math.min(width, height)); const strokeWidth = Math.max(3, Math.round(minSide * 0.0048)); const targetColor = "#00d4ff"; const sourceColor = "#ff3b30"; const connectorColor = "#14c8ff"; const dash = Math.max(12, Math.round(strokeWidth * 2.8)); const gap = Math.max(8, Math.round(strokeWidth * 1.8)); const connectorWidth = Math.max(2, Math.round(strokeWidth * 0.74)); const markerSize = Math.max(8, Math.round(strokeWidth * 3.1)); const regionSvg = regions .map((region) => { const source = region.pixelRect; const targetRect = getScaleAdjustTargetRect(region, scalePercent); const target = region.targetPixelRect || rectToPixels(targetRect, width, height); const radius = Math.max(8, Math.round(strokeWidth * 2)); const badgeSize = Math.max(22, Math.round(minSide * 0.022)); const badgeX = clamp( source.left + Math.round(badgeSize * 0.45), badgeSize / 2 + 4, width - badgeSize / 2 - 4 ); const badgeY = clamp( source.top + Math.round(badgeSize * 0.45), badgeSize / 2 + 4, height - badgeSize / 2 - 4 ); const fontSize = Math.max(13, Math.round(badgeSize * 0.52)); const sourceRight = source.left + source.width; const sourceBottom = source.top + source.height; const targetRight = target.left + target.width; const targetBottom = target.top + target.height; const sourceCorners = [ { x: source.left, y: source.top }, { x: sourceRight, y: source.top }, { x: sourceRight, y: sourceBottom }, { x: source.left, y: sourceBottom }, ]; const targetCorners = [ { x: target.left, y: target.top }, { x: targetRight, y: target.top }, { x: targetRight, y: targetBottom }, { x: target.left, y: targetBottom }, ]; const connectorSvg = sourceCorners .map((corner, cornerIndex) => { const targetCorner = targetCorners[cornerIndex]; const distance = Math.hypot(targetCorner.x - corner.x, targetCorner.y - corner.y); if (distance < Math.max(8, strokeWidth * 2.2)) return ""; const dx = (targetCorner.x - corner.x) / distance; const dy = (targetCorner.y - corner.y) / distance; const startInset = Math.min(distance * 0.16, Math.max(5, strokeWidth * 1.8)); const endInset = Math.min(distance * 0.18, Math.max(8, strokeWidth * 2.8)); const x1 = corner.x + dx * startInset; const y1 = corner.y + dy * startInset; const x2 = targetCorner.x - dx * endInset; const y2 = targetCorner.y - dy * endInset; return ``; }) .join(""); const cornerHandleSize = Math.max(7, Math.round(strokeWidth * 2.2)); const sourceCornerSvg = sourceCorners .map( (corner) => `` ) .join(""); const targetCornerSvg = targetCorners .map( (corner) => `` ) .join(""); return ` ${connectorSvg} ${sourceCornerSvg} ${targetCornerSvg} ${region.index} `; }) .join(""); return Buffer.from(` ${regionSvg} `); } function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) { const regionLines = regions .map((region) => { const left = Math.round(region.rect.x * 1000) / 10; const top = Math.round(region.rect.y * 1000) / 10; const width = Math.round(region.rect.width * 1000) / 10; const height = Math.round(region.rect.height * 1000) / 10; const referenceNote = region.referenceImages.length > 0 ? `;该区域有 ${region.referenceImages.length} 张专属参考图,依次为后续输入图片中的“区域 ${region.index} 参考图”,只可用于该区域` : ""; return `区域 ${region.index}(原图位置 x=${left}%,y=${top}%,w=${width}%,h=${height}%${referenceNote}):${region.prompt}`; }) .join("\n"); const referenceLines = regions .flatMap((region) => region.referenceImages.map( (reference, referenceIndex) => `区域 ${region.index} 参考图 ${referenceIndex + 1}:${reference.name || "未命名参考图"},只能影响区域 ${region.index}` ) ) .join("\n"); return [ "输入图片1是原始干净图,也是最终输出必须继承的唯一画面基底。", "系统已根据用户框选区域计算出下方每个区域的原图相对坐标;彩色框只存在于产品界面里,不会作为最终画面内容。", "输入图片2及之后如存在,是各区域专属参考图;专属参考图只服务于对应区域,不得改变其它区域或整张图的主体身份。", "最终结果必须保持输入图片1的原始画幅比例和构图边界,不要把画面压扁、拉伸、加白边或改成方图。", "请不要凭空生成任何彩色框、蓝框、绿框、红框、矩形边框、编号圆点、提示词气泡、边框线、半透明色块或标注文字。", "", "请严格根据每个区域下方的提示词,只在对应坐标区域附近完成局部编辑:", regionLines, referenceLines ? `\n区域专属参考图说明:\n${referenceLines}` : "", "", "新增主体完整性要求:如果区域提示词要求新增动物、人物、商品或其它完整主体,必须把该主体完整放在框选区域的安全内边距内,四周至少预留 8%-12% 缓冲空间;不要让头、身体、四肢、尾巴、产品边缘或阴影贴到选区边界,更不能被边界截断。空间不足时请主动缩小新增主体,而不是裁切主体。", "边缘融合要求:编辑区域与未编辑区域的地面、墙面、桌面、椅脚、阴影、纹理和透视必须连续自然,不得出现错位、断层、硬边、矩形补丁、局部贴片或不同清晰度的边界。", "最终输出必须是干净照片:不得出现任何彩色框、蓝框、绿框、红框、矩形边框、编号、提示词气泡、标注文字或 UI 元素。", "未被列为编辑区域的画面必须逐像素保持原图不变,不允许重绘、改色、补光、换材质、移动物体、改变清晰度或改变背景细节。", "新增内容必须匹配原图的透视、光线、比例、清晰度、色温和真实摄影风格。", "如果某个区域提示词与原图空间冲突,以自然融入原图为最高优先级。", ].join("\n"); } function buildScaleAdjustPrompt( regions: NormalizedLocalEditRegion[], scalePercent: ScaleAdjustPercent | null | undefined, cropReferences: ScaleAdjustCropReference[] = [], sourceSize?: { width: number; height: number } ) { const sourceSizeLine = sourceSize ? `最终输出必须保持与输入图片1完全相同的画幅和像素尺寸:${sourceSize.width} x ${sourceSize.height} px。` : "最终输出必须保持与输入图片1完全相同的画幅和像素尺寸。"; const regionLines = regions .map((region) => { const cropIndex = cropReferences.findIndex( (reference) => reference.regionIndex === region.index ); const targetRect = getScaleAdjustTargetRect(region, scalePercent); const sourceCenterX = region.rect.x + region.rect.width / 2; const sourceCenterY = region.rect.y + region.rect.height / 2; const targetCenterX = targetRect.x + targetRect.width / 2; const targetCenterY = targetRect.y + targetRect.height / 2; const moved = Math.hypot(targetCenterX - sourceCenterX, targetCenterY - sourceCenterY) > 0.01; const targetPositionNote = moved ? "青色虚线框同时指定用户希望的最终目标大小和目标位置,目标主体调整后的视觉中心应移动到青色虚线框中心附近。" : "青色虚线框指定用户希望的最终目标大小,目标主体调整后应围绕当前主体位置自然放置。"; const cropNote = cropIndex >= 0 ? `;输入图片${cropIndex + 3}是该区域的干净局部裁切参考图,只用于识别目标主体的身份、轮廓与原始比例` : ""; return `区域 ${region.index}${cropNote}:请在输入图片1中找到输入图片2红色实线框所标记范围内最主要、可独立调整的商品/物体主体。红色实线框只是“寻找主体”的范围,不是最终边界,也不是要缩放或贴回的矩形补丁。${targetPositionNote}请直接读取输入图片2中青色虚线框相对红色实线框的真实大小和位置关系,并按这个框线关系调整主体;四条蓝色箭头从红框四角指向青框四角,表示主体外接轮廓从当前位置迁移到目标大小和目标位置的方向。不要把框线换算成文字百分比来执行。目标虚线框不是要求主体填满矩形或变成矩形比例,请以主体自身轮廓、原始宽高比和视觉中心为准,整体等比调整主体大小和位置。严禁宽度变窄但高度变高,严禁横向挤压、纵向拉伸或改变主体透视形体。如果红框中只覆盖到主体的一部分,请根据连接关系推断该主体的完整可见结构,只调整该主体,不调整矩形框内背景。`; }) .join("\n"); const cropLines = cropReferences .map( (reference, index) => `输入图片${index + 3}:区域 ${reference.regionIndex} 的原图干净局部裁切参考图,只用于确认要缩放的主体身份、轮廓和宽高比,不能当成输出底图,也不能把裁切图整体贴回。` ) .join("\n"); return [ "这是一项局部主体比例 inpaint/edit 任务,不是整图缩放、整图裁切或重新构图任务。", sourceSizeLine, "输入图片1是原始干净图,也是最终输出必须继承的唯一画面基底。除目标主体尺寸变化与必要背景补全外,输入图片1中的文字、Logo、装饰、排版、背景、光线、构图边界和所有未选区域必须保持不变。", "输入图片2是白底选区参考图,只用于说明输入图片1上的红色实线框、青色目标虚线框和四角箭头关系;它不包含场景内容,不是输出底图,不得学习或复刻它的框线、箭头、编号、角点、半透明遮罩或 UI 标记。", cropLines ? `输入图片3及之后是框选区域的干净局部裁切参考图,作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图,不得被整体放大、贴回或覆盖到原图上。\n${cropLines}` : "", "输入图片2中的红色实线框表示输入图片1里目标主体当前大致位置;它只用于定位主体,不限制调整后的空间。青色虚线框表示用户手画的比例调整后目标大小和目标位置。四条蓝色箭头从红框四角指向青框四角:箭头向外表示放大,箭头向内表示缩小,箭头整体偏移表示移动。最终画面不得出现这些框线、箭头或角点,主体也不需要填满虚线矩形;必须保持主体自身原始宽高比和透视形体。", "如果系统或模型只能读取一张参考图,必须优先以输入图片1为输出基底,不得清理、改写或删除输入图片1中框选区域外的任何文字和图形。", "系统已根据用户框选区域计算出下方每个区域的原图坐标;框选区域只是定位目标主体的搜索范围,不是最终画面的裁切块、贴片、蒙版边界,也不是放大后的空间上限,更不是要整体缩放的矩形图层。", "最终结果必须保持原始干净图的画幅比例、构图边界、镜头透视、光线、色温、材质质感和清晰度;严禁整张图整体缩放、裁切、平移、重新排版或重新生成。", "Highest priority rule: do NOT zoom, crop, resize, reframe, pan, or regenerate the full canvas. Treat image 1 as a locked background plate; edit only the selected subject and the smallest necessary background around its old and new footprint.", "请不要生成任何彩色框、矩形边框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件或半透明色块。", "不要把用户手画的目标框换算成百分比文字执行;必须直接以输入图片2里青色目标框线本身作为大小和位置参考。", "", "任务:先在输入图片1中识别红色实线框范围内最主要、可独立调整的主体物体,再只对这个“主体物体”做局部比例调整。调整后的主体大小和位置必须以输入图片2中对应青色目标框线为准;如果输出看起来和原图一样,就是失败。", regionLines, "", "关键要求:", "1. 只调整目标主体物体本身的整体视觉比例,不得保留原尺寸;主体轮廓宽度和高度必须同步等比变化,宽高比必须与原主体一致,不能为了贴合虚线框而拉宽、压窄、拉高或压扁。", "2. 严禁把整个矩形框选区域、输入图片1整张图或任意裁切图当成图片块整体缩放、平移、复制、裁切或贴回;最终画面不得出现矩形拼接边缘、色差硬边、补丁感、裁切线或局部贴片痕迹。", "2.1 严禁把输入图片1整张图整体放大、缩小、裁切或重新构图;如果画面中的标题、Logo、页面文字、边框、背景家具或非目标主体发生整体位移/缩放/消失,必须视为失败并重新生成。", "3. 如果输入图片2中的青色目标框明显小于红色主体框,目标主体应自然变小,主体缩小后露出的背景、桌面、墙面、阴影、纹理和透视必须被重新生成并无缝补全,选区内应出现明显新增留白或背景。", "4. 如果输入图片2中的青色目标框明显大于红色主体框,目标主体应自然变大;允许主体自然超出用户手画的原始红框,使用周围真实背景完成边缘、阴影和遮挡衔接。宽度和高度都要同步变大,不能出现“高度变高但宽度变窄”的非等比变形;主体边缘与遮挡关系要合理,不能产生突兀裁切。", "5. 如果青色目标框中心与红色主体框中心不同,允许主体同步移动到青色目标框中心附近;如果主体靠近画布边界导致目标空间不足,请用能完整放入画面的最大统一倍率整体调整,绝不能为了塞进画面而只压缩宽度或只拉伸高度。", "6. 框选区域内非目标主体的背景、墙面、桌面、植物、文字和其它物体应保持原场景逻辑,不要跟着主体一起缩放。", "6.1 比例调整后的主体边缘必须和周围环境自然融合,选区边界不能被肉眼看出;不得出现一整块矩形区域清晰度、颜色、光照、纹理或透视突然变化。", "6.2 框选区域通常会包含主体周围的背景留白,背景留白不是目标主体。请锁定这些非主体背景的窗框、文字、地毯纹理、墙面、桌面边缘和透视线,只重建主体缩放后被遮挡或露出的极小必要区域;不要把框选矩形当作整块背景重新生成。", "6.3 如果框选范围内没有清晰、完整、可独立缩放的主体物体,请保持该区域背景结构稳定,不要为了制造变化而重绘窗框、文字、墙面、地面或其它非主体元素。", "7. 未受比例调整影响的其它画面区域必须保持原图不变,尤其是框选区域外的文字、Logo、标题、尺寸标注、装饰条和页面排版不得被清理、改写、模糊或重绘。", "8. 最终输出必须是一张完整、干净、无缝的照片,不得出现任何选区框、说明文字、对比线、比例数字或 UI 元素。", "9. 输出前自检:如果目标主体大小与原图相比没有明显变化,或者主体宽高比发生非等比变形,必须重新生成;禁止返回看起来和原图相同或被拉伸的结果。", ].join("\n"); } export async function POST(request: NextRequest) { try { const session = await auth(); if (!session?.user?.id) { return NextResponse.json({ success: false, error: "未授权,请先登录" }, { status: 401 }); } const body = (await request.json()) as LocalEditRequest; const rawSourceImageUrl = String(body.sourceImageUrl || "").trim(); if (!rawSourceImageUrl) { return NextResponse.json({ success: false, error: "缺少 sourceImageUrl 参数" }, { status: 400 }); } const sourceImageUrl = new URL(rawSourceImageUrl, request.url).toString(); const sourceBuffer = await sharp(await fetchImageBufferFromUrl(sourceImageUrl), { failOn: "none", }) .rotate() .toBuffer(); const sourceMetadata = await sharp(sourceBuffer, { failOn: "none" }).metadata(); const sourceWidth = Number(sourceMetadata.width || 0); const sourceHeight = Number(sourceMetadata.height || 0); if (sourceWidth <= 0 || sourceHeight <= 0) { return NextResponse.json({ success: false, error: "无法识别原图尺寸" }, { status: 400 }); } const operation = resolveLocalEditOperation(body.operation); let scalePercent: ScaleAdjustPercent | null = null; if (operation === SCALE_ADJUST_OPERATION) { const hasRequestedScalePercent = body.scalePercent !== undefined && body.scalePercent !== null; if (hasRequestedScalePercent) { try { scalePercent = normalizeScaleAdjustPercent(body.scalePercent); } catch (validationError) { return NextResponse.json( { success: false, error: validationError instanceof Error ? validationError.message : "比例调整参数无效", }, { status: 400 } ); } } } let regions: NormalizedLocalEditRegion[]; try { regions = normalizeRegions(body.regions, sourceWidth, sourceHeight, { fallbackPrompt: operation === SCALE_ADJUST_OPERATION ? "比例调整:按目标框调整" : undefined, requirePrompt: operation !== SCALE_ADJUST_OPERATION, }); } catch (validationError) { return NextResponse.json( { success: false, error: validationError instanceof Error ? validationError.message : "局部编辑区域参数无效", }, { status: 400 } ); } const hasScaleAdjustTargetRects = operation === SCALE_ADJUST_OPERATION && regions.length > 0 && regions.every((region) => Boolean(region.targetRect)); if ( operation === SCALE_ADJUST_OPERATION && !hasScaleAdjustTargetRects && scalePercent === null ) { return NextResponse.json( { success: false, error: "请为每个区域提供有效的目标框 targetRect,或提供比例调整幅度 scalePercent", }, { status: 400 } ); } const shouldUseScaleAdjustDetailModel = operation === SCALE_ADJUST_OPERATION && (hasScaleAdjustTargetRects || scalePercent !== null); const [pointsConfig, siteSettingsForModel, scaleAdjustModelCandidates] = await Promise.all([ getPointsSystemConfig(), getSiteSettings().catch(() => null), shouldUseScaleAdjustDetailModel ? getEnabledAiModelsWithProvider(false, true).catch((error) => { console.warn( "[Canvas Local Edit] Failed to load scale adjust model candidates:", error ); return [] as AiModelWithProviderForSelection[]; }) : Promise.resolve([] as AiModelWithProviderForSelection[]), ]); const pointsCharge = await prepareFeaturePointsCharge({ userId: session.user.id, unitCost: getImageToolCreditCost(pointsConfig, "repaint"), reason: operation === SCALE_ADJUST_OPERATION ? "image_scale_adjust" : "image_repaint", modelId: operation === SCALE_ADJUST_OPERATION ? "canvas-scale-adjust" : "canvas-local-edit", note: operation === SCALE_ADJUST_OPERATION ? "比例调整消耗积分" : "局部编辑消耗积分", pointsConfig, }); if (!pointsCharge.ok) { return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); } const isScaleAdjustOperation = operation === SCALE_ADJUST_OPERATION && (hasScaleAdjustTargetRects || scalePercent !== null); const displayPrompt = buildLocalEditDisplayPrompt({ operation, scalePercent }); const annotationSvg = isScaleAdjustOperation ? buildScaleAdjustAnnotationSvg({ height: sourceHeight, regions, scalePercent, width: sourceWidth, }) : buildAnnotationSvg({ height: sourceHeight, hidePromptText: false, regions, width: sourceWidth, }); const annotatedBase = isScaleAdjustOperation ? sharp({ create: { background: "#ffffff", channels: 3, height: sourceHeight, width: sourceWidth, }, }) : sharp(sourceBuffer, { failOn: "none" }); const annotatedBuffer = await annotatedBase .composite([{ input: annotationSvg, left: 0, top: 0 }]) .png({ compressionLevel: 6 }) .toBuffer(); const annotatedFilename = `canvas-local-edit-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`; const annotatedUpload = await uploadFile( annotatedFilename, toArrayBuffer(annotatedBuffer) ); const scaleAdjustCropReferences = isScaleAdjustOperation ? await createScaleAdjustCropReferences({ regions, sourceBuffer, sourceHeight, sourceWidth, }) : []; const prompt = isScaleAdjustOperation ? buildScaleAdjustPrompt( regions, scalePercent, scaleAdjustCropReferences, { width: sourceWidth, height: sourceHeight } ) : buildLocalEditPrompt(regions); const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); const scaleAdjustCropReferenceImageUrls = scaleAdjustCropReferences.map( (reference) => reference.url ); const referenceImages = isScaleAdjustOperation ? [ sourceImageUrl, annotatedUpload.url, ...scaleAdjustCropReferenceImageUrls, ...manualRegionReferenceImageUrls, ] : [sourceImageUrl, ...manualRegionReferenceImageUrls]; const resolvedAspectRatio = resolveLocalEditAspectRatio( body.aspectRatio, sourceWidth, sourceHeight ); const requestedModel = typeof body.model === "string" && body.model.trim() ? body.model.trim() : undefined; const configuredDetailModel = typeof siteSettingsForModel?.canvasDetailDefaultModelId === "string" ? siteSettingsForModel.canvasDetailDefaultModelId.trim() : ""; const detailFixedModel = isScaleAdjustOperation ? resolveScaleAdjustPreferredModelId({ configuredModelId: configuredDetailModel, requestedModelId: requestedModel || "", models: scaleAdjustModelCandidates, }) : ""; if ( isScaleAdjustOperation && detailFixedModel && detailFixedModel !== configuredDetailModel ) { console.info("[Canvas Local Edit] Override scale adjust model to detail line:", { configuredDetailModel: configuredDetailModel || null, requestedModel: requestedModel || null, resolvedModel: detailFixedModel, }); } const resolvedModel = detailFixedModel || requestedModel; const scaleAdjustCompositeRegions = isScaleAdjustOperation ? buildScaleAdjustCompositeRegions(regions, scalePercent) : []; const imageRequest: CanvasGenerateImageRequest = { aspectRatio: resolvedAspectRatio, billingFeature: "repaint", debugLabel: operation === SCALE_ADJUST_OPERATION ? "canvas-scale-adjust" : "canvas-local-edit", debugMeta: { annotatedReferenceUrl: annotatedUpload.url, annotatedReferenceSentToModel: isScaleAdjustOperation, annotatedReferenceType: isScaleAdjustOperation ? "white-source-target-arrow-guide" : "source-overlay", cleanSourceReferenceIndex: 1, configuredDetailModelId: configuredDetailModel || null, detailFixedModelApplied: Boolean(detailFixedModel), detailFixedModelId: detailFixedModel || null, displayPrompt, operation, requestedModel: requestedModel || null, resolvedAspectRatio, resolvedModel: resolvedModel || null, manualRegionReferenceImageCount: manualRegionReferenceImageUrls.length, regionReferenceImageCount: manualRegionReferenceImageUrls.length + scaleAdjustCropReferenceImageUrls.length, regionCount: regions.length, scalePercent, scaleAdjustCropReferenceCount: scaleAdjustCropReferenceImageUrls.length, scaleAdjustProtectionMode: hasScaleAdjustTargetRects ? "target-rect" : "scale-percent", scaleAdjustTargetRegionCount: regions.filter((region) => region.targetRect) .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? "locked-plate-local-subject-target-box-arrows-v5" : undefined, scaleAdjustPostCompositeApplied: false, scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation, sourceHeight, sourceNodeId: body.sourceNodeId || null, sourceWidth, }, displayPrompt, imageCount: 1, imageSize: typeof body.imageSize === "string" && body.imageSize.trim() ? body.imageSize.trim() : "2K", model: resolvedModel || undefined, negativePrompt: isScaleAdjustOperation ? "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、整体重绘、删除文字、清理文字、改写文字、文字模糊、Logo消失、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", prompt, promptExtend: false, referenceImages, scaleAdjustVerification: isScaleAdjustOperation ? { protectedRegions: scaleAdjustCompositeRegions, sourceImageUrl, regions: regions.map((region) => region.rect), ...(typeof scalePercent === "number" ? { scalePercent: scalePercent as ScaleAdjustPercent } : {}), ...(hasScaleAdjustTargetRects ? { targetRegions: regions .map((region) => region.targetRect) .filter( ( rect ): rect is NonNullable => Boolean(rect) ), } : {}), } : undefined, localEditComposite: isScaleAdjustOperation ? undefined : { sourceImageUrl, regions: regions.map((region) => region.rect), }, ownerKey: session.user.id, }; const task = await createCanvasImageTask({ request: imageRequest as unknown as Record, userId: session.user.id, }); await enqueueCanvasImageTask(task.id); const currentTask = await getCanvasImageTaskForUser({ taskId: task.id, userId: session.user.id, }); if (pointsCharge.chargeable) { try { await pointsCharge.consume({ note: operation === SCALE_ADJUST_OPERATION ? `比例调整消耗积分(${task.id})` : `局部编辑消耗积分(${task.id})`, }); } catch (pointsError) { console.error("[Points] Failed to consume local edit points:", pointsError); } } return NextResponse.json({ success: true, annotatedReferenceUrl: annotatedUpload.url, aspectRatio: resolvedAspectRatio, displayPrompt, prompt: displayPrompt, regions, sourceHeight, sourceWidth, taskId: task.id, status: currentTask?.status || task.status, queueStatus: currentTask?.queueStatus, }); } catch (error) { console.error("[Canvas Local Edit] Error:", error); return NextResponse.json( { success: false, error: error instanceof Error ? error.message : "局部编辑提交失败", }, { status: 500 } ); } }