Your Name 61da591a5c
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled
fix: improve canvas local edit output
2026-05-21 10:25:19 +08:00

573 lines
20 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { NextRequest, NextResponse } from "next/server";
import sharp from "sharp";
import { auth } from "@/app/(auth)/auth";
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
import {
createCanvasImageTask,
enqueueCanvasImageTask,
getCanvasImageTaskForUser,
} from "@/lib/canvas/image-task-queue";
import type { CanvasGenerateImageRequest } from "@/lib/canvas/generate-image";
import { getPointsSystemConfig } from "@/lib/db/queries";
import { getImageToolCreditCost } from "@/lib/points/config";
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
import { uploadFile } from "@/lib/storage/service";
export const runtime = "nodejs";
export const maxDuration = 120;
type LocalEditRect = {
x?: number;
y?: number;
width?: number;
height?: number;
};
type LocalEditRegionInput = {
id?: string;
index?: number;
color?: string;
prompt?: string;
rect?: LocalEditRect;
referenceImages?: Array<{
id?: string;
url?: string;
name?: string;
source?: string;
}>;
};
type LocalEditRequest = {
sourceImageUrl?: string;
sourceNodeId?: string;
regions?: LocalEditRegionInput[];
model?: string;
imageSize?: string;
aspectRatio?: string;
};
type NormalizedLocalEditRegion = {
id: string;
index: number;
color: string;
colorName: string;
prompt: string;
referenceImages: Array<{
id: string;
url: string;
name?: string;
source: "upload" | "canvas";
}>;
rect: {
x: number;
y: number;
width: number;
height: number;
};
pixelRect: {
left: number;
top: number;
width: number;
height: number;
};
};
const MAX_LOCAL_EDIT_REGIONS = 8;
const MAX_REFERENCE_IMAGES_PER_REGION = 3;
const MIN_REGION_EDGE_PX = 24;
const DEFAULT_REGION_COLORS = [
"#ff3b30",
"#1677ff",
"#00c853",
"#ff9500",
"#af52de",
"#00bcd4",
"#ff2d55",
"#8d6e63",
];
const SUPPORTED_LOCAL_EDIT_ASPECT_RATIOS = [
"1:1",
"1:4",
"1:8",
"2:3",
"3:2",
"3:4",
"4:1",
"4:3",
"4:5",
"5:4",
"8:1",
"9:16",
"16:9",
"21:9",
] as const;
function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(min, value));
}
function parseAspectRatioValue(ratio: string) {
const [width, height] = ratio.split(":").map((part) => Number(part));
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
return 1;
}
return width / height;
}
function inferClosestAspectRatio(width: number, height: number) {
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
return "1:1";
}
const sourceRatio = width / height;
let closest: (typeof SUPPORTED_LOCAL_EDIT_ASPECT_RATIOS)[number] = "1:1";
let closestDiff = Number.POSITIVE_INFINITY;
for (const candidate of SUPPORTED_LOCAL_EDIT_ASPECT_RATIOS) {
const diff = Math.abs(parseAspectRatioValue(candidate) - sourceRatio);
if (diff < closestDiff) {
closest = candidate;
closestDiff = diff;
}
}
return closest;
}
function resolveLocalEditAspectRatio(input: unknown, sourceWidth: number, sourceHeight: number) {
const requested = String(input || "").trim();
if (
requested &&
requested.toLowerCase() !== "auto" &&
(SUPPORTED_LOCAL_EDIT_ASPECT_RATIOS as readonly string[]).includes(requested)
) {
return requested;
}
return inferClosestAspectRatio(sourceWidth, sourceHeight);
}
function toArrayBuffer(buffer: Buffer) {
return buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength
) as ArrayBuffer;
}
function escapeXml(value: string) {
return value
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;")
.replace(/"/g, "&quot;")
.replace(/'/g, "&apos;");
}
function normalizeHexColor(input: unknown, fallback: string) {
const raw = String(input || "").trim();
const expanded = raw.replace(/^#([0-9a-f])([0-9a-f])([0-9a-f])$/i, "#$1$1$2$2$3$3");
return /^#[0-9a-f]{6}$/i.test(expanded) ? expanded.toLowerCase() : fallback;
}
function normalizeHttpImageUrl(input: unknown) {
const raw = String(input || "").trim();
if (!/^https?:\/\//i.test(raw)) return "";
return raw;
}
function normalizeRegionReferenceImages(
region: LocalEditRegionInput,
itemIndex: number
): NormalizedLocalEditRegion["referenceImages"] {
if (!Array.isArray(region.referenceImages)) return [];
return region.referenceImages
.map((reference, referenceIndex) => {
const url = normalizeHttpImageUrl(reference?.url);
if (!url) return null;
const name =
typeof reference?.name === "string" && reference.name.trim()
? reference.name.trim().slice(0, 80)
: "";
return {
id:
typeof reference?.id === "string" && reference.id.trim()
? reference.id.trim()
: `region-${itemIndex + 1}-reference-${referenceIndex + 1}`,
url,
...(name ? { name } : {}),
source: reference?.source === "canvas" ? "canvas" as const : "upload" as const,
};
})
.filter((item): item is NormalizedLocalEditRegion["referenceImages"][number] =>
Boolean(item)
)
.slice(0, MAX_REFERENCE_IMAGES_PER_REGION);
}
function getColorName(hex: string) {
const normalized = hex.toLowerCase();
const names: Record<string, string> = {
"#ff3b30": "红色",
"#1677ff": "蓝色",
"#00c853": "绿色",
"#ff9500": "橙色",
"#af52de": "紫色",
"#00bcd4": "青色",
"#ff2d55": "玫红色",
"#8d6e63": "棕色",
};
return names[normalized] || `${normalized} 色`;
}
function normalizeRect(input: LocalEditRect | undefined) {
if (!input) return null;
const x = Number(input.x);
const y = Number(input.y);
const width = Number(input.width);
const height = Number(input.height);
if (![x, y, width, height].every(Number.isFinite)) return null;
const left = clamp(Math.min(x, x + width), 0, 1);
const top = clamp(Math.min(y, y + height), 0, 1);
const right = clamp(Math.max(x, x + width), 0, 1);
const bottom = clamp(Math.max(y, y + height), 0, 1);
const normalizedWidth = clamp(right - left, 0, 1 - left);
const normalizedHeight = clamp(bottom - top, 0, 1 - top);
if (normalizedWidth <= 0 || normalizedHeight <= 0) return null;
return {
x: left,
y: top,
width: normalizedWidth,
height: normalizedHeight,
};
}
function rectToPixels(
rect: NonNullable<ReturnType<typeof normalizeRect>>,
sourceWidth: number,
sourceHeight: number
) {
const left = clamp(Math.round(rect.x * sourceWidth), 0, Math.max(0, sourceWidth - 1));
const top = clamp(Math.round(rect.y * sourceHeight), 0, Math.max(0, sourceHeight - 1));
const width = clamp(Math.round(rect.width * sourceWidth), 1, sourceWidth - left);
const height = clamp(Math.round(rect.height * sourceHeight), 1, sourceHeight - top);
return { left, top, width, height };
}
function wrapText(text: string, maxChars: number, maxLines = 3) {
const compact = text.replace(/\s+/g, " ").trim();
if (!compact) return [];
const lines: string[] = [];
let remaining = compact;
while (remaining.length > 0 && lines.length < maxLines) {
if (remaining.length <= maxChars) {
lines.push(remaining);
break;
}
lines.push(remaining.slice(0, maxChars));
remaining = remaining.slice(maxChars);
}
if (remaining.length > 0 && lines.length >= maxLines) {
lines[lines.length - 1] = `${lines[lines.length - 1].slice(0, Math.max(1, maxChars - 1))}…`;
}
return lines;
}
function normalizeRegions(
rawRegions: LocalEditRegionInput[] | undefined,
sourceWidth: number,
sourceHeight: number
): NormalizedLocalEditRegion[] {
if (!Array.isArray(rawRegions) || rawRegions.length === 0) {
throw new Error("请先框选至少一个局部编辑区域");
}
if (rawRegions.length > MAX_LOCAL_EDIT_REGIONS) {
throw new Error(`局部编辑最多支持 ${MAX_LOCAL_EDIT_REGIONS} 个区域`);
}
return rawRegions.map((region, itemIndex) => {
const rect = normalizeRect(region.rect);
if (!rect) {
throw new Error(`区域 ${itemIndex + 1} 缺少有效框选范围`);
}
const prompt = String(region.prompt || "").trim();
if (!prompt) {
throw new Error(`区域 ${itemIndex + 1} 请填写局部编辑提示词`);
}
const pixelRect = rectToPixels(rect, sourceWidth, sourceHeight);
if (pixelRect.width < MIN_REGION_EDGE_PX || pixelRect.height < MIN_REGION_EDGE_PX) {
throw new Error(`区域 ${itemIndex + 1} 过小,请重新框选更大的范围`);
}
const color = normalizeHexColor(
region.color,
DEFAULT_REGION_COLORS[itemIndex % DEFAULT_REGION_COLORS.length]
);
return {
id: String(region.id || `region-${itemIndex + 1}`),
index: itemIndex + 1,
color,
colorName: getColorName(color),
prompt: prompt.slice(0, 300),
referenceImages: normalizeRegionReferenceImages(region, itemIndex),
rect,
pixelRect,
};
});
}
function buildAnnotationSvg(params: {
width: number;
height: number;
regions: NormalizedLocalEditRegion[];
}) {
const { width, height, regions } = params;
const minSide = Math.max(1, Math.min(width, height));
const strokeWidth = Math.max(4, Math.round(minSide * 0.006));
const fontSize = Math.max(24, Math.round(minSide * 0.026));
const smallFontSize = Math.max(18, Math.round(fontSize * 0.72));
const padX = Math.round(fontSize * 0.55);
const padY = Math.round(fontSize * 0.34);
const gap = Math.round(fontSize * 0.34);
const radius = Math.round(fontSize * 0.5);
const regionSvg = regions
.map((region) => {
const { left, top, width: rectWidth, height: rectHeight } = region.pixelRect;
const labelText = `区域${region.index} ${region.prompt}`;
const maxChars = Math.max(8, Math.min(24, Math.floor((width * 0.42) / Math.max(1, fontSize * 0.58))));
const lines = wrapText(labelText, maxChars, 3);
const labelWidth = Math.min(
Math.max(
rectWidth,
...lines.map((line) => Math.ceil(line.length * fontSize * 0.62) + padX * 2)
),
Math.max(180, width - 16)
);
const labelHeight = lines.length * Math.round(fontSize * 1.24) + padY * 2;
const preferBelow = top + rectHeight + gap + labelHeight <= height - 8;
const labelX = clamp(left, 8, Math.max(8, width - labelWidth - 8));
const labelY = preferBelow
? top + rectHeight + gap
: clamp(top - gap - labelHeight, 8, Math.max(8, height - labelHeight - 8));
const badgeSize = Math.max(34, Math.round(fontSize * 1.35));
const badgeCx = clamp(left, badgeSize / 2 + 6, width - badgeSize / 2 - 6);
const badgeCy = clamp(top, badgeSize / 2 + 6, height - badgeSize / 2 - 6);
const textLines = lines
.map(
(line, lineIndex) =>
`<text x="${labelX + padX}" y="${labelY + padY + fontSize + lineIndex * Math.round(fontSize * 1.24)}" fill="#ffffff" font-size="${fontSize}" font-family="Arial, Helvetica, sans-serif" font-weight="700">${escapeXml(line)}</text>`
)
.join("");
return `
<rect x="${left}" y="${top}" width="${rectWidth}" height="${rectHeight}" rx="${radius}" fill="none" stroke="${region.color}" stroke-width="${strokeWidth}" />
<rect x="${left}" y="${top}" width="${rectWidth}" height="${rectHeight}" rx="${radius}" fill="${region.color}" opacity="0.08" />
<circle cx="${badgeCx}" cy="${badgeCy}" r="${badgeSize / 2}" fill="${region.color}" stroke="#ffffff" stroke-width="${Math.max(2, Math.round(strokeWidth * 0.45))}" />
<text x="${badgeCx}" y="${badgeCy + smallFontSize * 0.38}" text-anchor="middle" fill="#ffffff" font-size="${smallFontSize}" font-family="Arial, Helvetica, sans-serif" font-weight="800">${region.index}</text>
<rect x="${labelX}" y="${labelY}" width="${labelWidth}" height="${labelHeight}" rx="${radius}" fill="#000000" opacity="0.78" stroke="${region.color}" stroke-width="${Math.max(2, Math.round(strokeWidth * 0.5))}" />
${textLines}
`;
})
.join("");
return Buffer.from(`
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
${regionSvg}
</svg>
`);
}
function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) {
const regionLines = regions
.map((region) => {
const left = Math.round(region.rect.x * 1000) / 10;
const top = Math.round(region.rect.y * 1000) / 10;
const width = Math.round(region.rect.width * 1000) / 10;
const height = Math.round(region.rect.height * 1000) / 10;
const referenceNote =
region.referenceImages.length > 0
? `;该区域有 ${region.referenceImages.length} 张专属参考图,依次为后续输入图片中的“区域 ${region.index} 参考图”,只可用于该区域`
: "";
return `区域 ${region.index}(原图位置 x=${left}%,y=${top}%,w=${width}%,h=${height}%${referenceNote}):${region.prompt}`;
})
.join("\n");
const referenceLines = regions
.flatMap((region) =>
region.referenceImages.map(
(reference, referenceIndex) =>
`区域 ${region.index} 参考图 ${referenceIndex + 1}:${reference.name || "未命名参考图"},只能影响区域 ${region.index}`
)
)
.join("\n");
return [
"输入图片1是原始干净图,也是最终输出必须继承的唯一画面基底。",
"系统已根据用户框选区域计算出下方每个区域的原图相对坐标;彩色框只存在于产品界面里,不会作为最终画面内容。",
"输入图片2及之后如存在,是各区域专属参考图;专属参考图只服务于对应区域,不得改变其它区域或整张图的主体身份。",
"最终结果必须保持输入图片1的原始画幅比例和构图边界,不要把画面压扁、拉伸、加白边或改成方图。",
"请不要凭空生成任何彩色框、蓝框、绿框、红框、矩形边框、编号圆点、提示词气泡、边框线、半透明色块或标注文字。",
"",
"请严格根据每个区域下方的提示词,只在对应坐标区域附近完成局部编辑:",
regionLines,
referenceLines ? `\n区域专属参考图说明:\n${referenceLines}` : "",
"",
"最终输出必须是干净照片:不得出现任何彩色框、蓝框、绿框、红框、矩形边框、编号、提示词气泡、标注文字或 UI 元素。",
"未被列为编辑区域的画面必须尽量保持原图不变。",
"新增内容必须匹配原图的透视、光线、比例、清晰度、色温和真实摄影风格。",
"如果某个区域提示词与原图空间冲突,以自然融入原图为最高优先级。",
].join("\n");
}
export async function POST(request: NextRequest) {
try {
const session = await auth();
if (!session?.user?.id) {
return NextResponse.json({ success: false, error: "未授权,请先登录" }, { status: 401 });
}
const body = (await request.json()) as LocalEditRequest;
const rawSourceImageUrl = String(body.sourceImageUrl || "").trim();
if (!rawSourceImageUrl) {
return NextResponse.json({ success: false, error: "缺少 sourceImageUrl 参数" }, { status: 400 });
}
const sourceImageUrl = new URL(rawSourceImageUrl, request.url).toString();
const sourceBuffer = await sharp(await fetchImageBufferFromUrl(sourceImageUrl), {
failOn: "none",
})
.rotate()
.toBuffer();
const sourceMetadata = await sharp(sourceBuffer, { failOn: "none" }).metadata();
const sourceWidth = Number(sourceMetadata.width || 0);
const sourceHeight = Number(sourceMetadata.height || 0);
if (sourceWidth <= 0 || sourceHeight <= 0) {
return NextResponse.json({ success: false, error: "无法识别原图尺寸" }, { status: 400 });
}
let regions: NormalizedLocalEditRegion[];
try {
regions = normalizeRegions(body.regions, sourceWidth, sourceHeight);
} catch (validationError) {
return NextResponse.json(
{
success: false,
error:
validationError instanceof Error
? validationError.message
: "局部编辑区域参数无效",
},
{ status: 400 }
);
}
const pointsConfig = await getPointsSystemConfig();
const pointsCharge = await prepareFeaturePointsCharge({
userId: session.user.id,
unitCost: getImageToolCreditCost(pointsConfig, "repaint"),
reason: "image_repaint",
modelId: "canvas-local-edit",
note: "局部编辑消耗积分",
pointsConfig,
});
if (!pointsCharge.ok) {
return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status });
}
const annotationSvg = buildAnnotationSvg({
height: sourceHeight,
regions,
width: sourceWidth,
});
const annotatedBuffer = await sharp(sourceBuffer, { failOn: "none" })
.composite([{ input: annotationSvg, left: 0, top: 0 }])
.png({ compressionLevel: 6 })
.toBuffer();
const annotatedFilename = `canvas-local-edit-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`;
const annotatedUpload = await uploadFile(
annotatedFilename,
toArrayBuffer(annotatedBuffer)
);
const prompt = buildLocalEditPrompt(regions);
const regionReferenceImageUrls = regions.flatMap((region) =>
region.referenceImages.map((reference) => reference.url)
);
const resolvedAspectRatio = resolveLocalEditAspectRatio(
body.aspectRatio,
sourceWidth,
sourceHeight
);
const imageRequest: CanvasGenerateImageRequest = {
aspectRatio: resolvedAspectRatio,
billingFeature: "repaint",
debugLabel: "canvas-local-edit",
debugMeta: {
annotatedReferenceUrl: annotatedUpload.url,
annotatedReferenceSentToModel: false,
resolvedAspectRatio,
regionReferenceImageCount: regionReferenceImageUrls.length,
regionCount: regions.length,
sourceHeight,
sourceNodeId: body.sourceNodeId || null,
sourceWidth,
},
imageCount: 1,
imageSize:
typeof body.imageSize === "string" && body.imageSize.trim()
? body.imageSize.trim()
: "2K",
model:
typeof body.model === "string" && body.model.trim()
? body.model.trim()
: undefined,
negativePrompt:
"彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸",
prompt,
promptExtend: false,
referenceImages: [
sourceImageUrl,
...regionReferenceImageUrls,
],
ownerKey: session.user.id,
};
const task = await createCanvasImageTask({
request: imageRequest as unknown as Record<string, unknown>,
userId: session.user.id,
});
await enqueueCanvasImageTask(task.id);
const currentTask = await getCanvasImageTaskForUser({
taskId: task.id,
userId: session.user.id,
});
if (pointsCharge.chargeable) {
try {
await pointsCharge.consume({
note: `局部编辑消耗积分(${task.id})`,
});
} catch (pointsError) {
console.error("[Points] Failed to consume local edit points:", pointsError);
}
}
return NextResponse.json({
success: true,
annotatedReferenceUrl: annotatedUpload.url,
aspectRatio: resolvedAspectRatio,
prompt,
regions,
sourceHeight,
sourceWidth,
taskId: task.id,
status: currentTask?.status || task.status,
queueStatus: currentTask?.queueStatus,
});
} catch (error) {
console.error("[Canvas Local Edit] Error:", error);
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : "局部编辑提交失败",
},
{ status: 500 }
);
}
}