fix: stabilize detail text edits and cutline deletion
This commit is contained in:
Your Name
parent
e8796c74a6
commit
efc77160c2
@ -16,11 +16,14 @@ import { createKrea } from "@/lib/ai/providers/krea";
|
||||
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
|
||||
import { isImageGenerationModelDescriptor } from "@/lib/ai/model-classification";
|
||||
import { createLogger } from "@/lib/observability/logger";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 120;
|
||||
|
||||
const logger = createLogger("canvas-text-edit");
|
||||
const TEXT_EDIT_SOURCE_ARCHIVE_TIMEOUT_MS = 20_000;
|
||||
const TEXT_EDIT_MODEL_FALLBACK_LIMIT = 3;
|
||||
|
||||
const textItemSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
@ -50,8 +53,12 @@ const SUPPORTED_TEXT_EDIT_PROVIDER_TYPES = new Set([
|
||||
"volcano-vision",
|
||||
]);
|
||||
|
||||
function pickTextEditModel(models: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>) {
|
||||
const candidates = models
|
||||
type TextEditModelCandidate = Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number];
|
||||
|
||||
function getTextEditModelCandidates(
|
||||
models: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>
|
||||
) {
|
||||
return models
|
||||
.filter(
|
||||
(item) =>
|
||||
isImageGenerationModelDescriptor({
|
||||
@ -77,8 +84,6 @@ function pickTextEditModel(models: Awaited<ReturnType<typeof getEnabledAiModelsW
|
||||
(b.model.displayName?.includes("火山") ? 3 : 0);
|
||||
return bScore - aScore;
|
||||
});
|
||||
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
function createTextEditProvider(
|
||||
@ -118,6 +123,85 @@ function createTextEditProvider(
|
||||
}
|
||||
}
|
||||
|
||||
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||||
return buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
}
|
||||
|
||||
function inferImageExtension(contentType: string, imageUrl: string) {
|
||||
const normalizedContentType = contentType.toLowerCase();
|
||||
if (normalizedContentType.includes("image/jpeg")) return "jpg";
|
||||
if (normalizedContentType.includes("image/png")) return "png";
|
||||
if (normalizedContentType.includes("image/webp")) return "webp";
|
||||
if (normalizedContentType.includes("image/gif")) return "gif";
|
||||
const pathname = (() => {
|
||||
try {
|
||||
return new URL(imageUrl).pathname;
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
})();
|
||||
const match = pathname.match(/\.([a-z0-9]{2,5})$/i);
|
||||
const ext = String(match?.[1] || "").toLowerCase();
|
||||
return ["jpg", "jpeg", "png", "webp", "gif"].includes(ext) ? ext : "png";
|
||||
}
|
||||
|
||||
async function archiveTextEditSourceImage(imageUrl: string) {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), TEXT_EDIT_SOURCE_ARCHIVE_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const response = await fetch(imageUrl, {
|
||||
cache: "no-store",
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`源图下载失败：${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const contentType = response.headers.get("content-type") || "";
|
||||
if (contentType && !contentType.toLowerCase().startsWith("image/")) {
|
||||
throw new Error(`源图类型异常：${contentType}`);
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
if (buffer.length === 0) {
|
||||
throw new Error("源图为空文件");
|
||||
}
|
||||
|
||||
const ext = inferImageExtension(contentType, imageUrl);
|
||||
const uploaded = await uploadFile(
|
||||
`canvas-text-edit-source-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`,
|
||||
bufferToArrayBuffer(buffer)
|
||||
);
|
||||
return {
|
||||
bytes: buffer.length,
|
||||
url: uploaded.url,
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveTextEditSourceImageUrl(imageUrl: string) {
|
||||
try {
|
||||
const archived = await archiveTextEditSourceImage(imageUrl);
|
||||
logger.info("source_image_archived", {
|
||||
bytes: archived.bytes,
|
||||
sourceHost: new URL(imageUrl).host,
|
||||
targetHost: new URL(archived.url).host,
|
||||
});
|
||||
return archived.url;
|
||||
} catch (error) {
|
||||
logger.warn("source_image_archive_failed", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return imageUrl;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeGeneratedImageUrl(value: unknown) {
|
||||
const raw = String(value || "").trim();
|
||||
if (!raw) return "";
|
||||
@ -271,6 +355,70 @@ function buildTextEditPrompt(
|
||||
return sections.join("\n");
|
||||
}
|
||||
|
||||
async function generateTextEditImageWithFailover(params: {
|
||||
candidates: TextEditModelCandidate[];
|
||||
imageUrl: string;
|
||||
prompt: string;
|
||||
}) {
|
||||
let lastError: unknown = null;
|
||||
const candidates = params.candidates.slice(0, TEXT_EDIT_MODEL_FALLBACK_LIMIT);
|
||||
|
||||
for (const selected of candidates) {
|
||||
try {
|
||||
logger.info("selected_model", {
|
||||
providerType: selected.provider.type,
|
||||
providerName: selected.provider.name,
|
||||
modelId: selected.model.modelId,
|
||||
displayName: selected.model.displayName,
|
||||
isDefault: selected.model.isDefault,
|
||||
isDefaultTextEditModel: selected.model.isDefaultTextEditModel,
|
||||
});
|
||||
|
||||
const provider = createTextEditProvider(selected.provider);
|
||||
const result = await provider.chat(selected.model.modelId).doGenerate({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image_url", image_url: { url: params.imageUrl } },
|
||||
{ type: "text", text: params.prompt },
|
||||
],
|
||||
},
|
||||
],
|
||||
imageParams: {
|
||||
imageCount: 1,
|
||||
},
|
||||
});
|
||||
|
||||
const resultText = String(result?.text || "").trim();
|
||||
const finishReason = String(result?.finishReason || "").toLowerCase();
|
||||
if (!resultText) {
|
||||
throw new Error("模型未返回结果");
|
||||
}
|
||||
if (finishReason === "error" || resultText.startsWith("图片生成失败")) {
|
||||
throw new Error(resultText);
|
||||
}
|
||||
|
||||
const parsed = parseImageFromGenerationText(resultText);
|
||||
return {
|
||||
...parsed,
|
||||
model: selected.model.displayName,
|
||||
};
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
logger.warn("candidate_failed", {
|
||||
providerType: selected.provider.type,
|
||||
providerName: selected.provider.name,
|
||||
modelId: selected.model.modelId,
|
||||
displayName: selected.model.displayName,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError instanceof Error ? lastError : new Error("文字替换失败");
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
const session = await auth();
|
||||
@ -327,9 +475,9 @@ export async function POST(request: Request) {
|
||||
}
|
||||
|
||||
const models = await getEnabledAiModelsWithProvider();
|
||||
const selected = pickTextEditModel(models);
|
||||
const candidates = getTextEditModelCandidates(models);
|
||||
|
||||
if (!selected) {
|
||||
if (candidates.length === 0) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
@ -339,46 +487,20 @@ export async function POST(request: Request) {
|
||||
);
|
||||
}
|
||||
|
||||
logger.info("selected_model", {
|
||||
providerType: selected.provider.type,
|
||||
providerName: selected.provider.name,
|
||||
modelId: selected.model.modelId,
|
||||
displayName: selected.model.displayName,
|
||||
isDefault: selected.model.isDefault,
|
||||
isDefaultTextEditModel: selected.model.isDefaultTextEditModel,
|
||||
});
|
||||
|
||||
const provider = createTextEditProvider(selected.provider);
|
||||
|
||||
const prompt = buildTextEditPrompt(replacements, deletions);
|
||||
|
||||
const result = await provider.chat(selected.model.modelId).doGenerate({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image_url", image_url: { url: imageUrl } },
|
||||
{ type: "text", text: prompt },
|
||||
],
|
||||
},
|
||||
],
|
||||
imageParams: {
|
||||
imageCount: 1,
|
||||
},
|
||||
const generationSourceImageUrl = await resolveTextEditSourceImageUrl(imageUrl);
|
||||
const { originalUrl, thumbnailUrl, model } = await generateTextEditImageWithFailover({
|
||||
candidates,
|
||||
imageUrl: generationSourceImageUrl,
|
||||
prompt,
|
||||
});
|
||||
|
||||
if (!result?.text) {
|
||||
throw new Error("模型未返回结果");
|
||||
}
|
||||
|
||||
const { originalUrl, thumbnailUrl } = parseImageFromGenerationText(result.text);
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
originalUrl,
|
||||
thumbnailUrl,
|
||||
model: selected.model.displayName,
|
||||
model,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
|
||||
@ -894,7 +894,13 @@ export function AgentDetailReferenceStitchDialog({
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onPointerDown={(event) => {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
setDraggingLineIndex(null);
|
||||
}}
|
||||
onClick={(event) => {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
handleRemoveCutLine(index);
|
||||
}}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user