fix: stabilize detail text edits and cutline deletion

This commit is contained in:
Your Name 2026-06-01 14:43:34 +08:00
parent e8796c74a6
commit efc77160c2
2 changed files with 166 additions and 38 deletions

View File

@ -16,11 +16,14 @@ import { createKrea } from "@/lib/ai/providers/krea";
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision"; import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
import { isImageGenerationModelDescriptor } from "@/lib/ai/model-classification"; import { isImageGenerationModelDescriptor } from "@/lib/ai/model-classification";
import { createLogger } from "@/lib/observability/logger"; import { createLogger } from "@/lib/observability/logger";
import { uploadFile } from "@/lib/storage/service";
export const runtime = "nodejs"; export const runtime = "nodejs";
export const maxDuration = 120; export const maxDuration = 120;
const logger = createLogger("canvas-text-edit"); const logger = createLogger("canvas-text-edit");
const TEXT_EDIT_SOURCE_ARCHIVE_TIMEOUT_MS = 20_000;
const TEXT_EDIT_MODEL_FALLBACK_LIMIT = 3;
const textItemSchema = z.object({ const textItemSchema = z.object({
id: z.string().min(1), id: z.string().min(1),
@ -50,8 +53,12 @@ const SUPPORTED_TEXT_EDIT_PROVIDER_TYPES = new Set([
"volcano-vision", "volcano-vision",
]); ]);
function pickTextEditModel(models: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>) { type TextEditModelCandidate = Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number];
const candidates = models
function getTextEditModelCandidates(
models: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>
) {
return models
.filter( .filter(
(item) => (item) =>
isImageGenerationModelDescriptor({ isImageGenerationModelDescriptor({
@ -77,8 +84,6 @@ function pickTextEditModel(models: Awaited<ReturnType<typeof getEnabledAiModelsW
(b.model.displayName?.includes("火山") ? 3 : 0); (b.model.displayName?.includes("火山") ? 3 : 0);
return bScore - aScore; return bScore - aScore;
}); });
return candidates[0];
} }
function createTextEditProvider( function createTextEditProvider(
@ -118,6 +123,85 @@ function createTextEditProvider(
} }
} }
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
return buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength
) as ArrayBuffer;
}
function inferImageExtension(contentType: string, imageUrl: string) {
const normalizedContentType = contentType.toLowerCase();
if (normalizedContentType.includes("image/jpeg")) return "jpg";
if (normalizedContentType.includes("image/png")) return "png";
if (normalizedContentType.includes("image/webp")) return "webp";
if (normalizedContentType.includes("image/gif")) return "gif";
const pathname = (() => {
try {
return new URL(imageUrl).pathname;
} catch {
return "";
}
})();
const match = pathname.match(/\.([a-z0-9]{2,5})$/i);
const ext = String(match?.[1] || "").toLowerCase();
return ["jpg", "jpeg", "png", "webp", "gif"].includes(ext) ? ext : "png";
}
async function archiveTextEditSourceImage(imageUrl: string) {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), TEXT_EDIT_SOURCE_ARCHIVE_TIMEOUT_MS);
try {
const response = await fetch(imageUrl, {
cache: "no-store",
signal: controller.signal,
});
if (!response.ok) {
throw new Error(`源图下载失败:${response.status} ${response.statusText}`);
}
const contentType = response.headers.get("content-type") || "";
if (contentType && !contentType.toLowerCase().startsWith("image/")) {
throw new Error(`源图类型异常:${contentType}`);
}
const buffer = Buffer.from(await response.arrayBuffer());
if (buffer.length === 0) {
throw new Error("源图为空文件");
}
const ext = inferImageExtension(contentType, imageUrl);
const uploaded = await uploadFile(
`canvas-text-edit-source-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`,
bufferToArrayBuffer(buffer)
);
return {
bytes: buffer.length,
url: uploaded.url,
};
} finally {
clearTimeout(timeoutId);
}
}
async function resolveTextEditSourceImageUrl(imageUrl: string) {
try {
const archived = await archiveTextEditSourceImage(imageUrl);
logger.info("source_image_archived", {
bytes: archived.bytes,
sourceHost: new URL(imageUrl).host,
targetHost: new URL(archived.url).host,
});
return archived.url;
} catch (error) {
logger.warn("source_image_archive_failed", {
error: error instanceof Error ? error.message : String(error),
});
return imageUrl;
}
}
function normalizeGeneratedImageUrl(value: unknown) { function normalizeGeneratedImageUrl(value: unknown) {
const raw = String(value || "").trim(); const raw = String(value || "").trim();
if (!raw) return ""; if (!raw) return "";
@ -271,6 +355,70 @@ function buildTextEditPrompt(
return sections.join("\n"); return sections.join("\n");
} }
async function generateTextEditImageWithFailover(params: {
candidates: TextEditModelCandidate[];
imageUrl: string;
prompt: string;
}) {
let lastError: unknown = null;
const candidates = params.candidates.slice(0, TEXT_EDIT_MODEL_FALLBACK_LIMIT);
for (const selected of candidates) {
try {
logger.info("selected_model", {
providerType: selected.provider.type,
providerName: selected.provider.name,
modelId: selected.model.modelId,
displayName: selected.model.displayName,
isDefault: selected.model.isDefault,
isDefaultTextEditModel: selected.model.isDefaultTextEditModel,
});
const provider = createTextEditProvider(selected.provider);
const result = await provider.chat(selected.model.modelId).doGenerate({
messages: [
{
role: "user",
content: [
{ type: "image_url", image_url: { url: params.imageUrl } },
{ type: "text", text: params.prompt },
],
},
],
imageParams: {
imageCount: 1,
},
});
const resultText = String(result?.text || "").trim();
const finishReason = String(result?.finishReason || "").toLowerCase();
if (!resultText) {
throw new Error("模型未返回结果");
}
if (finishReason === "error" || resultText.startsWith("图片生成失败")) {
throw new Error(resultText);
}
const parsed = parseImageFromGenerationText(resultText);
return {
...parsed,
model: selected.model.displayName,
};
} catch (error) {
lastError = error;
logger.warn("candidate_failed", {
providerType: selected.provider.type,
providerName: selected.provider.name,
modelId: selected.model.modelId,
displayName: selected.model.displayName,
error: error instanceof Error ? error.message : String(error),
});
}
}
throw lastError instanceof Error ? lastError : new Error("文字替换失败");
}
export async function POST(request: Request) { export async function POST(request: Request) {
try { try {
const session = await auth(); const session = await auth();
@ -327,9 +475,9 @@ export async function POST(request: Request) {
} }
const models = await getEnabledAiModelsWithProvider(); const models = await getEnabledAiModelsWithProvider();
const selected = pickTextEditModel(models); const candidates = getTextEditModelCandidates(models);
if (!selected) { if (candidates.length === 0) {
return NextResponse.json( return NextResponse.json(
{ {
success: false, success: false,
@ -339,46 +487,20 @@ export async function POST(request: Request) {
); );
} }
logger.info("selected_model", {
providerType: selected.provider.type,
providerName: selected.provider.name,
modelId: selected.model.modelId,
displayName: selected.model.displayName,
isDefault: selected.model.isDefault,
isDefaultTextEditModel: selected.model.isDefaultTextEditModel,
});
const provider = createTextEditProvider(selected.provider);
const prompt = buildTextEditPrompt(replacements, deletions); const prompt = buildTextEditPrompt(replacements, deletions);
const generationSourceImageUrl = await resolveTextEditSourceImageUrl(imageUrl);
const result = await provider.chat(selected.model.modelId).doGenerate({ const { originalUrl, thumbnailUrl, model } = await generateTextEditImageWithFailover({
messages: [ candidates,
{ imageUrl: generationSourceImageUrl,
role: "user", prompt,
content: [
{ type: "image_url", image_url: { url: imageUrl } },
{ type: "text", text: prompt },
],
},
],
imageParams: {
imageCount: 1,
},
}); });
if (!result?.text) {
throw new Error("模型未返回结果");
}
const { originalUrl, thumbnailUrl } = parseImageFromGenerationText(result.text);
return NextResponse.json({ return NextResponse.json({
success: true, success: true,
data: { data: {
originalUrl, originalUrl,
thumbnailUrl, thumbnailUrl,
model: selected.model.displayName, model,
}, },
}); });
} catch (error) { } catch (error) {

View File

@ -894,7 +894,13 @@ export function AgentDetailReferenceStitchDialog({
</div> </div>
<button <button
type="button" type="button"
onPointerDown={(event) => {
event.preventDefault();
event.stopPropagation();
setDraggingLineIndex(null);
}}
onClick={(event) => { onClick={(event) => {
event.preventDefault();
event.stopPropagation(); event.stopPropagation();
handleRemoveCutLine(index); handleRemoveCutLine(index);
}} }}