feat: add canvas local image edit
This commit is contained in:
Your Name
parent
5d8146ca11
commit
0b5866bb94
430
app/api/canvas/local-edit/route.ts
Normal file
430
app/api/canvas/local-edit/route.ts
Normal file
@ -0,0 +1,430 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import sharp from "sharp";
|
||||
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||||
import {
|
||||
createCanvasImageTask,
|
||||
enqueueCanvasImageTask,
|
||||
getCanvasImageTaskForUser,
|
||||
} from "@/lib/canvas/image-task-queue";
|
||||
import type { CanvasGenerateImageRequest } from "@/lib/canvas/generate-image";
|
||||
import { getPointsSystemConfig } from "@/lib/db/queries";
|
||||
import { getImageToolCreditCost } from "@/lib/points/config";
|
||||
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 120;
|
||||
|
||||
type LocalEditRect = {
|
||||
x?: number;
|
||||
y?: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
};
|
||||
|
||||
type LocalEditRegionInput = {
|
||||
id?: string;
|
||||
index?: number;
|
||||
color?: string;
|
||||
prompt?: string;
|
||||
rect?: LocalEditRect;
|
||||
};
|
||||
|
||||
type LocalEditRequest = {
|
||||
sourceImageUrl?: string;
|
||||
sourceNodeId?: string;
|
||||
regions?: LocalEditRegionInput[];
|
||||
model?: string;
|
||||
imageSize?: string;
|
||||
aspectRatio?: string;
|
||||
};
|
||||
|
||||
type NormalizedLocalEditRegion = {
|
||||
id: string;
|
||||
index: number;
|
||||
color: string;
|
||||
colorName: string;
|
||||
prompt: string;
|
||||
rect: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
pixelRect: {
|
||||
left: number;
|
||||
top: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
};
|
||||
|
||||
const MAX_LOCAL_EDIT_REGIONS = 8;
|
||||
const MIN_REGION_EDGE_PX = 24;
|
||||
const DEFAULT_REGION_COLORS = [
|
||||
"#ff3b30",
|
||||
"#1677ff",
|
||||
"#00c853",
|
||||
"#ff9500",
|
||||
"#af52de",
|
||||
"#00bcd4",
|
||||
"#ff2d55",
|
||||
"#8d6e63",
|
||||
];
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
function toArrayBuffer(buffer: Buffer) {
|
||||
return buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
}
|
||||
|
||||
function escapeXml(value: string) {
|
||||
return value
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """)
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
function normalizeHexColor(input: unknown, fallback: string) {
|
||||
const raw = String(input || "").trim();
|
||||
const expanded = raw.replace(/^#([0-9a-f])([0-9a-f])([0-9a-f])$/i, "#$1$1$2$2$3$3");
|
||||
return /^#[0-9a-f]{6}$/i.test(expanded) ? expanded.toLowerCase() : fallback;
|
||||
}
|
||||
|
||||
function getColorName(hex: string) {
|
||||
const normalized = hex.toLowerCase();
|
||||
const names: Record<string, string> = {
|
||||
"#ff3b30": "红色",
|
||||
"#1677ff": "蓝色",
|
||||
"#00c853": "绿色",
|
||||
"#ff9500": "橙色",
|
||||
"#af52de": "紫色",
|
||||
"#00bcd4": "青色",
|
||||
"#ff2d55": "玫红色",
|
||||
"#8d6e63": "棕色",
|
||||
};
|
||||
return names[normalized] || `${normalized} 色`;
|
||||
}
|
||||
|
||||
function normalizeRect(input: LocalEditRect | undefined) {
|
||||
if (!input) return null;
|
||||
const x = Number(input.x);
|
||||
const y = Number(input.y);
|
||||
const width = Number(input.width);
|
||||
const height = Number(input.height);
|
||||
if (![x, y, width, height].every(Number.isFinite)) return null;
|
||||
const left = clamp(Math.min(x, x + width), 0, 1);
|
||||
const top = clamp(Math.min(y, y + height), 0, 1);
|
||||
const right = clamp(Math.max(x, x + width), 0, 1);
|
||||
const bottom = clamp(Math.max(y, y + height), 0, 1);
|
||||
const normalizedWidth = clamp(right - left, 0, 1 - left);
|
||||
const normalizedHeight = clamp(bottom - top, 0, 1 - top);
|
||||
if (normalizedWidth <= 0 || normalizedHeight <= 0) return null;
|
||||
return {
|
||||
x: left,
|
||||
y: top,
|
||||
width: normalizedWidth,
|
||||
height: normalizedHeight,
|
||||
};
|
||||
}
|
||||
|
||||
function rectToPixels(
|
||||
rect: NonNullable<ReturnType<typeof normalizeRect>>,
|
||||
sourceWidth: number,
|
||||
sourceHeight: number
|
||||
) {
|
||||
const left = clamp(Math.round(rect.x * sourceWidth), 0, Math.max(0, sourceWidth - 1));
|
||||
const top = clamp(Math.round(rect.y * sourceHeight), 0, Math.max(0, sourceHeight - 1));
|
||||
const width = clamp(Math.round(rect.width * sourceWidth), 1, sourceWidth - left);
|
||||
const height = clamp(Math.round(rect.height * sourceHeight), 1, sourceHeight - top);
|
||||
return { left, top, width, height };
|
||||
}
|
||||
|
||||
function wrapText(text: string, maxChars: number, maxLines = 3) {
|
||||
const compact = text.replace(/\s+/g, " ").trim();
|
||||
if (!compact) return [];
|
||||
const lines: string[] = [];
|
||||
let remaining = compact;
|
||||
while (remaining.length > 0 && lines.length < maxLines) {
|
||||
if (remaining.length <= maxChars) {
|
||||
lines.push(remaining);
|
||||
break;
|
||||
}
|
||||
lines.push(remaining.slice(0, maxChars));
|
||||
remaining = remaining.slice(maxChars);
|
||||
}
|
||||
if (remaining.length > 0 && lines.length >= maxLines) {
|
||||
lines[lines.length - 1] = `${lines[lines.length - 1].slice(0, Math.max(1, maxChars - 1))}…`;
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
function normalizeRegions(
|
||||
rawRegions: LocalEditRegionInput[] | undefined,
|
||||
sourceWidth: number,
|
||||
sourceHeight: number
|
||||
): NormalizedLocalEditRegion[] {
|
||||
if (!Array.isArray(rawRegions) || rawRegions.length === 0) {
|
||||
throw new Error("请先框选至少一个局部编辑区域");
|
||||
}
|
||||
if (rawRegions.length > MAX_LOCAL_EDIT_REGIONS) {
|
||||
throw new Error(`局部编辑最多支持 ${MAX_LOCAL_EDIT_REGIONS} 个区域`);
|
||||
}
|
||||
|
||||
return rawRegions.map((region, itemIndex) => {
|
||||
const rect = normalizeRect(region.rect);
|
||||
if (!rect) {
|
||||
throw new Error(`区域 ${itemIndex + 1} 缺少有效框选范围`);
|
||||
}
|
||||
const prompt = String(region.prompt || "").trim();
|
||||
if (!prompt) {
|
||||
throw new Error(`区域 ${itemIndex + 1} 请填写局部编辑提示词`);
|
||||
}
|
||||
const pixelRect = rectToPixels(rect, sourceWidth, sourceHeight);
|
||||
if (pixelRect.width < MIN_REGION_EDGE_PX || pixelRect.height < MIN_REGION_EDGE_PX) {
|
||||
throw new Error(`区域 ${itemIndex + 1} 过小，请重新框选更大的范围`);
|
||||
}
|
||||
const color = normalizeHexColor(
|
||||
region.color,
|
||||
DEFAULT_REGION_COLORS[itemIndex % DEFAULT_REGION_COLORS.length]
|
||||
);
|
||||
return {
|
||||
id: String(region.id || `region-${itemIndex + 1}`),
|
||||
index: itemIndex + 1,
|
||||
color,
|
||||
colorName: getColorName(color),
|
||||
prompt: prompt.slice(0, 300),
|
||||
rect,
|
||||
pixelRect,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function buildAnnotationSvg(params: {
|
||||
width: number;
|
||||
height: number;
|
||||
regions: NormalizedLocalEditRegion[];
|
||||
}) {
|
||||
const { width, height, regions } = params;
|
||||
const minSide = Math.max(1, Math.min(width, height));
|
||||
const strokeWidth = Math.max(4, Math.round(minSide * 0.006));
|
||||
const fontSize = Math.max(24, Math.round(minSide * 0.026));
|
||||
const smallFontSize = Math.max(18, Math.round(fontSize * 0.72));
|
||||
const padX = Math.round(fontSize * 0.55);
|
||||
const padY = Math.round(fontSize * 0.34);
|
||||
const gap = Math.round(fontSize * 0.34);
|
||||
const radius = Math.round(fontSize * 0.5);
|
||||
|
||||
const regionSvg = regions
|
||||
.map((region) => {
|
||||
const { left, top, width: rectWidth, height: rectHeight } = region.pixelRect;
|
||||
const labelText = `区域${region.index} ${region.prompt}`;
|
||||
const maxChars = Math.max(8, Math.min(24, Math.floor((width * 0.42) / Math.max(1, fontSize * 0.58))));
|
||||
const lines = wrapText(labelText, maxChars, 3);
|
||||
const labelWidth = Math.min(
|
||||
Math.max(
|
||||
rectWidth,
|
||||
...lines.map((line) => Math.ceil(line.length * fontSize * 0.62) + padX * 2)
|
||||
),
|
||||
Math.max(180, width - 16)
|
||||
);
|
||||
const labelHeight = lines.length * Math.round(fontSize * 1.24) + padY * 2;
|
||||
const preferBelow = top + rectHeight + gap + labelHeight <= height - 8;
|
||||
const labelX = clamp(left, 8, Math.max(8, width - labelWidth - 8));
|
||||
const labelY = preferBelow
|
||||
? top + rectHeight + gap
|
||||
: clamp(top - gap - labelHeight, 8, Math.max(8, height - labelHeight - 8));
|
||||
const badgeSize = Math.max(34, Math.round(fontSize * 1.35));
|
||||
const badgeCx = clamp(left, badgeSize / 2 + 6, width - badgeSize / 2 - 6);
|
||||
const badgeCy = clamp(top, badgeSize / 2 + 6, height - badgeSize / 2 - 6);
|
||||
const textLines = lines
|
||||
.map(
|
||||
(line, lineIndex) =>
|
||||
`<text x="${labelX + padX}" y="${labelY + padY + fontSize + lineIndex * Math.round(fontSize * 1.24)}" fill="#ffffff" font-size="${fontSize}" font-family="Arial, Helvetica, sans-serif" font-weight="700">${escapeXml(line)}</text>`
|
||||
)
|
||||
.join("");
|
||||
|
||||
return `
|
||||
<rect x="${left}" y="${top}" width="${rectWidth}" height="${rectHeight}" rx="${radius}" fill="none" stroke="${region.color}" stroke-width="${strokeWidth}" />
|
||||
<rect x="${left}" y="${top}" width="${rectWidth}" height="${rectHeight}" rx="${radius}" fill="${region.color}" opacity="0.08" />
|
||||
<circle cx="${badgeCx}" cy="${badgeCy}" r="${badgeSize / 2}" fill="${region.color}" stroke="#ffffff" stroke-width="${Math.max(2, Math.round(strokeWidth * 0.45))}" />
|
||||
<text x="${badgeCx}" y="${badgeCy + smallFontSize * 0.38}" text-anchor="middle" fill="#ffffff" font-size="${smallFontSize}" font-family="Arial, Helvetica, sans-serif" font-weight="800">${region.index}</text>
|
||||
<rect x="${labelX}" y="${labelY}" width="${labelWidth}" height="${labelHeight}" rx="${radius}" fill="#000000" opacity="0.78" stroke="${region.color}" stroke-width="${Math.max(2, Math.round(strokeWidth * 0.5))}" />
|
||||
${textLines}
|
||||
`;
|
||||
})
|
||||
.join("");
|
||||
|
||||
return Buffer.from(`
|
||||
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
|
||||
${regionSvg}
|
||||
</svg>
|
||||
`);
|
||||
}
|
||||
|
||||
function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) {
|
||||
const regionLines = regions
|
||||
.map((region) => `区域 ${region.index}（${region.colorName}，编号 ${region.index}）：${region.prompt}`)
|
||||
.join("\n");
|
||||
|
||||
return [
|
||||
"输入图片1是一张带彩色标注框和框下提示词的图片编辑参考图；输入图片2是同一张原始干净图，用于保持画面细节与未编辑区域。",
|
||||
"彩色框、编号、提示词气泡只是编辑指令，不是最终画面内容。",
|
||||
"",
|
||||
"请严格根据每个区域下方的提示词，只在对应彩色框附近完成局部编辑：",
|
||||
regionLines,
|
||||
"",
|
||||
"最终输出必须去除所有彩色框、编号、提示词气泡和标注文字。",
|
||||
"未被彩色框标注的区域必须尽量保持原图不变。",
|
||||
"新增内容必须匹配原图的透视、光线、比例、清晰度、色温和真实摄影风格。",
|
||||
"如果某个区域提示词与原图空间冲突，以自然融入原图为最高优先级。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ success: false, error: "未授权，请先登录" }, { status: 401 });
|
||||
}
|
||||
|
||||
const body = (await request.json()) as LocalEditRequest;
|
||||
const rawSourceImageUrl = String(body.sourceImageUrl || "").trim();
|
||||
if (!rawSourceImageUrl) {
|
||||
return NextResponse.json({ success: false, error: "缺少 sourceImageUrl 参数" }, { status: 400 });
|
||||
}
|
||||
|
||||
const sourceImageUrl = new URL(rawSourceImageUrl, request.url).toString();
|
||||
const sourceBuffer = await sharp(await fetchImageBufferFromUrl(sourceImageUrl), {
|
||||
failOn: "none",
|
||||
})
|
||||
.rotate()
|
||||
.toBuffer();
|
||||
const sourceMetadata = await sharp(sourceBuffer, { failOn: "none" }).metadata();
|
||||
const sourceWidth = Number(sourceMetadata.width || 0);
|
||||
const sourceHeight = Number(sourceMetadata.height || 0);
|
||||
if (sourceWidth <= 0 || sourceHeight <= 0) {
|
||||
return NextResponse.json({ success: false, error: "无法识别原图尺寸" }, { status: 400 });
|
||||
}
|
||||
|
||||
let regions: NormalizedLocalEditRegion[];
|
||||
try {
|
||||
regions = normalizeRegions(body.regions, sourceWidth, sourceHeight);
|
||||
} catch (validationError) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error:
|
||||
validationError instanceof Error
|
||||
? validationError.message
|
||||
: "局部编辑区域参数无效",
|
||||
},
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const pointsConfig = await getPointsSystemConfig();
|
||||
const pointsCharge = await prepareFeaturePointsCharge({
|
||||
userId: session.user.id,
|
||||
unitCost: getImageToolCreditCost(pointsConfig, "repaint"),
|
||||
reason: "image_repaint",
|
||||
modelId: "canvas-local-edit",
|
||||
note: "局部编辑消耗积分",
|
||||
pointsConfig,
|
||||
});
|
||||
if (!pointsCharge.ok) {
|
||||
return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status });
|
||||
}
|
||||
|
||||
const annotationSvg = buildAnnotationSvg({
|
||||
height: sourceHeight,
|
||||
regions,
|
||||
width: sourceWidth,
|
||||
});
|
||||
const annotatedBuffer = await sharp(sourceBuffer, { failOn: "none" })
|
||||
.composite([{ input: annotationSvg, left: 0, top: 0 }])
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const annotatedFilename = `canvas-local-edit-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`;
|
||||
const annotatedUpload = await uploadFile(
|
||||
annotatedFilename,
|
||||
toArrayBuffer(annotatedBuffer)
|
||||
);
|
||||
const prompt = buildLocalEditPrompt(regions);
|
||||
|
||||
const imageRequest: CanvasGenerateImageRequest = {
|
||||
aspectRatio:
|
||||
typeof body.aspectRatio === "string" && body.aspectRatio.trim()
|
||||
? body.aspectRatio.trim()
|
||||
: "auto",
|
||||
billingFeature: "repaint",
|
||||
debugLabel: "canvas-local-edit",
|
||||
debugMeta: {
|
||||
annotatedReferenceUrl: annotatedUpload.url,
|
||||
regionCount: regions.length,
|
||||
sourceNodeId: body.sourceNodeId || null,
|
||||
},
|
||||
imageCount: 1,
|
||||
imageSize:
|
||||
typeof body.imageSize === "string" && body.imageSize.trim()
|
||||
? body.imageSize.trim()
|
||||
: "2K",
|
||||
model:
|
||||
typeof body.model === "string" && body.model.trim()
|
||||
? body.model.trim()
|
||||
: undefined,
|
||||
prompt,
|
||||
promptExtend: false,
|
||||
referenceImages: [annotatedUpload.url, sourceImageUrl],
|
||||
ownerKey: session.user.id,
|
||||
};
|
||||
|
||||
const task = await createCanvasImageTask({
|
||||
request: imageRequest as unknown as Record<string, unknown>,
|
||||
userId: session.user.id,
|
||||
});
|
||||
await enqueueCanvasImageTask(task.id);
|
||||
const currentTask = await getCanvasImageTaskForUser({
|
||||
taskId: task.id,
|
||||
userId: session.user.id,
|
||||
});
|
||||
|
||||
if (pointsCharge.chargeable) {
|
||||
try {
|
||||
await pointsCharge.consume({
|
||||
note: `局部编辑消耗积分（${task.id}）`,
|
||||
});
|
||||
} catch (pointsError) {
|
||||
console.error("[Points] Failed to consume local edit points:", pointsError);
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
annotatedReferenceUrl: annotatedUpload.url,
|
||||
prompt,
|
||||
regions,
|
||||
taskId: task.id,
|
||||
status: currentTask?.status || task.status,
|
||||
queueStatus: currentTask?.queueStatus,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Canvas Local Edit] Error:", error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : "局部编辑提交失败",
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -298,6 +298,83 @@ export const imageGenerationService = {
|
||||
}
|
||||
},
|
||||
|
||||
async localEdit(params: {
|
||||
sourceImageUrl: string;
|
||||
sourceNodeId?: string;
|
||||
regions: Array<{
|
||||
id: string;
|
||||
index?: number;
|
||||
color: string;
|
||||
prompt: string;
|
||||
rect: NormalizedCropRect;
|
||||
}>;
|
||||
model?: string;
|
||||
imageSize?: string;
|
||||
aspectRatio?: string;
|
||||
}): Promise<
|
||||
ApiResponse<{
|
||||
annotatedReferenceUrl: string;
|
||||
prompt: string;
|
||||
queueStatus?: unknown;
|
||||
status?: string;
|
||||
taskId: string;
|
||||
}>
|
||||
> {
|
||||
try {
|
||||
const response = await fetch('/api/canvas/local-edit', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(params),
|
||||
});
|
||||
const data = await response.json();
|
||||
if (data.success && data.taskId) {
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
annotatedReferenceUrl: data.annotatedReferenceUrl,
|
||||
prompt: data.prompt,
|
||||
queueStatus: data.queueStatus,
|
||||
status: data.status,
|
||||
taskId: data.taskId,
|
||||
},
|
||||
};
|
||||
}
|
||||
return { success: false, error: data.error || '局部编辑提交失败' };
|
||||
} catch (error) {
|
||||
console.error('[Canvas] 局部编辑提交失败:', error);
|
||||
return { success: false, error: '局部编辑提交失败' };
|
||||
}
|
||||
},
|
||||
|
||||
async getImageTask(taskId: string): Promise<
|
||||
ApiResponse<{
|
||||
task: {
|
||||
id: string;
|
||||
status: string;
|
||||
error?: string | null;
|
||||
result?: {
|
||||
images?: Array<{ thumbnailUrl: string; originalUrl: string }>;
|
||||
model?: string;
|
||||
modelRef?: string;
|
||||
prompt?: string;
|
||||
} | null;
|
||||
queueStatus?: unknown;
|
||||
};
|
||||
}>
|
||||
> {
|
||||
try {
|
||||
const response = await fetch(`/api/canvas/generate/image?taskId=${encodeURIComponent(taskId)}`);
|
||||
const data = await response.json();
|
||||
if (data.success && data.task) {
|
||||
return { success: true, data: { task: data.task } };
|
||||
}
|
||||
return { success: false, error: data.error || '查询生图任务失败' };
|
||||
} catch (error) {
|
||||
console.error('[Canvas] 查询生图任务失败:', error);
|
||||
return { success: false, error: '查询生图任务失败' };
|
||||
}
|
||||
},
|
||||
|
||||
// 图片抠图 - 使用图片URL
|
||||
async removeBackground(imageUrl: string): Promise<ApiResponse<{ url: string }>> {
|
||||
try {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user