fix: stabilize detail text edits and cutline deletion
This commit is contained in:
parent
e8796c74a6
commit
efc77160c2
@ -16,11 +16,14 @@ import { createKrea } from "@/lib/ai/providers/krea";
|
|||||||
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
|
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
|
||||||
import { isImageGenerationModelDescriptor } from "@/lib/ai/model-classification";
|
import { isImageGenerationModelDescriptor } from "@/lib/ai/model-classification";
|
||||||
import { createLogger } from "@/lib/observability/logger";
|
import { createLogger } from "@/lib/observability/logger";
|
||||||
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
|
||||||
export const runtime = "nodejs";
|
export const runtime = "nodejs";
|
||||||
export const maxDuration = 120;
|
export const maxDuration = 120;
|
||||||
|
|
||||||
const logger = createLogger("canvas-text-edit");
|
const logger = createLogger("canvas-text-edit");
|
||||||
|
const TEXT_EDIT_SOURCE_ARCHIVE_TIMEOUT_MS = 20_000;
|
||||||
|
const TEXT_EDIT_MODEL_FALLBACK_LIMIT = 3;
|
||||||
|
|
||||||
const textItemSchema = z.object({
|
const textItemSchema = z.object({
|
||||||
id: z.string().min(1),
|
id: z.string().min(1),
|
||||||
@ -50,8 +53,12 @@ const SUPPORTED_TEXT_EDIT_PROVIDER_TYPES = new Set([
|
|||||||
"volcano-vision",
|
"volcano-vision",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
function pickTextEditModel(models: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>) {
|
type TextEditModelCandidate = Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number];
|
||||||
const candidates = models
|
|
||||||
|
function getTextEditModelCandidates(
|
||||||
|
models: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>
|
||||||
|
) {
|
||||||
|
return models
|
||||||
.filter(
|
.filter(
|
||||||
(item) =>
|
(item) =>
|
||||||
isImageGenerationModelDescriptor({
|
isImageGenerationModelDescriptor({
|
||||||
@ -77,8 +84,6 @@ function pickTextEditModel(models: Awaited<ReturnType<typeof getEnabledAiModelsW
|
|||||||
(b.model.displayName?.includes("火山") ? 3 : 0);
|
(b.model.displayName?.includes("火山") ? 3 : 0);
|
||||||
return bScore - aScore;
|
return bScore - aScore;
|
||||||
});
|
});
|
||||||
|
|
||||||
return candidates[0];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function createTextEditProvider(
|
function createTextEditProvider(
|
||||||
@ -118,6 +123,85 @@ function createTextEditProvider(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||||||
|
return buffer.buffer.slice(
|
||||||
|
buffer.byteOffset,
|
||||||
|
buffer.byteOffset + buffer.byteLength
|
||||||
|
) as ArrayBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
function inferImageExtension(contentType: string, imageUrl: string) {
|
||||||
|
const normalizedContentType = contentType.toLowerCase();
|
||||||
|
if (normalizedContentType.includes("image/jpeg")) return "jpg";
|
||||||
|
if (normalizedContentType.includes("image/png")) return "png";
|
||||||
|
if (normalizedContentType.includes("image/webp")) return "webp";
|
||||||
|
if (normalizedContentType.includes("image/gif")) return "gif";
|
||||||
|
const pathname = (() => {
|
||||||
|
try {
|
||||||
|
return new URL(imageUrl).pathname;
|
||||||
|
} catch {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
const match = pathname.match(/\.([a-z0-9]{2,5})$/i);
|
||||||
|
const ext = String(match?.[1] || "").toLowerCase();
|
||||||
|
return ["jpg", "jpeg", "png", "webp", "gif"].includes(ext) ? ext : "png";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function archiveTextEditSourceImage(imageUrl: string) {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timeoutId = setTimeout(() => controller.abort(), TEXT_EDIT_SOURCE_ARCHIVE_TIMEOUT_MS);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(imageUrl, {
|
||||||
|
cache: "no-store",
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`源图下载失败:${response.status} ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentType = response.headers.get("content-type") || "";
|
||||||
|
if (contentType && !contentType.toLowerCase().startsWith("image/")) {
|
||||||
|
throw new Error(`源图类型异常:${contentType}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const buffer = Buffer.from(await response.arrayBuffer());
|
||||||
|
if (buffer.length === 0) {
|
||||||
|
throw new Error("源图为空文件");
|
||||||
|
}
|
||||||
|
|
||||||
|
const ext = inferImageExtension(contentType, imageUrl);
|
||||||
|
const uploaded = await uploadFile(
|
||||||
|
`canvas-text-edit-source-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`,
|
||||||
|
bufferToArrayBuffer(buffer)
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
bytes: buffer.length,
|
||||||
|
url: uploaded.url,
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timeoutId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveTextEditSourceImageUrl(imageUrl: string) {
|
||||||
|
try {
|
||||||
|
const archived = await archiveTextEditSourceImage(imageUrl);
|
||||||
|
logger.info("source_image_archived", {
|
||||||
|
bytes: archived.bytes,
|
||||||
|
sourceHost: new URL(imageUrl).host,
|
||||||
|
targetHost: new URL(archived.url).host,
|
||||||
|
});
|
||||||
|
return archived.url;
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn("source_image_archive_failed", {
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
return imageUrl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeGeneratedImageUrl(value: unknown) {
|
function normalizeGeneratedImageUrl(value: unknown) {
|
||||||
const raw = String(value || "").trim();
|
const raw = String(value || "").trim();
|
||||||
if (!raw) return "";
|
if (!raw) return "";
|
||||||
@ -271,6 +355,70 @@ function buildTextEditPrompt(
|
|||||||
return sections.join("\n");
|
return sections.join("\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function generateTextEditImageWithFailover(params: {
|
||||||
|
candidates: TextEditModelCandidate[];
|
||||||
|
imageUrl: string;
|
||||||
|
prompt: string;
|
||||||
|
}) {
|
||||||
|
let lastError: unknown = null;
|
||||||
|
const candidates = params.candidates.slice(0, TEXT_EDIT_MODEL_FALLBACK_LIMIT);
|
||||||
|
|
||||||
|
for (const selected of candidates) {
|
||||||
|
try {
|
||||||
|
logger.info("selected_model", {
|
||||||
|
providerType: selected.provider.type,
|
||||||
|
providerName: selected.provider.name,
|
||||||
|
modelId: selected.model.modelId,
|
||||||
|
displayName: selected.model.displayName,
|
||||||
|
isDefault: selected.model.isDefault,
|
||||||
|
isDefaultTextEditModel: selected.model.isDefaultTextEditModel,
|
||||||
|
});
|
||||||
|
|
||||||
|
const provider = createTextEditProvider(selected.provider);
|
||||||
|
const result = await provider.chat(selected.model.modelId).doGenerate({
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: [
|
||||||
|
{ type: "image_url", image_url: { url: params.imageUrl } },
|
||||||
|
{ type: "text", text: params.prompt },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
imageParams: {
|
||||||
|
imageCount: 1,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const resultText = String(result?.text || "").trim();
|
||||||
|
const finishReason = String(result?.finishReason || "").toLowerCase();
|
||||||
|
if (!resultText) {
|
||||||
|
throw new Error("模型未返回结果");
|
||||||
|
}
|
||||||
|
if (finishReason === "error" || resultText.startsWith("图片生成失败")) {
|
||||||
|
throw new Error(resultText);
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = parseImageFromGenerationText(resultText);
|
||||||
|
return {
|
||||||
|
...parsed,
|
||||||
|
model: selected.model.displayName,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error;
|
||||||
|
logger.warn("candidate_failed", {
|
||||||
|
providerType: selected.provider.type,
|
||||||
|
providerName: selected.provider.name,
|
||||||
|
modelId: selected.model.modelId,
|
||||||
|
displayName: selected.model.displayName,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw lastError instanceof Error ? lastError : new Error("文字替换失败");
|
||||||
|
}
|
||||||
|
|
||||||
export async function POST(request: Request) {
|
export async function POST(request: Request) {
|
||||||
try {
|
try {
|
||||||
const session = await auth();
|
const session = await auth();
|
||||||
@ -327,9 +475,9 @@ export async function POST(request: Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const models = await getEnabledAiModelsWithProvider();
|
const models = await getEnabledAiModelsWithProvider();
|
||||||
const selected = pickTextEditModel(models);
|
const candidates = getTextEditModelCandidates(models);
|
||||||
|
|
||||||
if (!selected) {
|
if (candidates.length === 0) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{
|
{
|
||||||
success: false,
|
success: false,
|
||||||
@ -339,46 +487,20 @@ export async function POST(request: Request) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info("selected_model", {
|
|
||||||
providerType: selected.provider.type,
|
|
||||||
providerName: selected.provider.name,
|
|
||||||
modelId: selected.model.modelId,
|
|
||||||
displayName: selected.model.displayName,
|
|
||||||
isDefault: selected.model.isDefault,
|
|
||||||
isDefaultTextEditModel: selected.model.isDefaultTextEditModel,
|
|
||||||
});
|
|
||||||
|
|
||||||
const provider = createTextEditProvider(selected.provider);
|
|
||||||
|
|
||||||
const prompt = buildTextEditPrompt(replacements, deletions);
|
const prompt = buildTextEditPrompt(replacements, deletions);
|
||||||
|
const generationSourceImageUrl = await resolveTextEditSourceImageUrl(imageUrl);
|
||||||
const result = await provider.chat(selected.model.modelId).doGenerate({
|
const { originalUrl, thumbnailUrl, model } = await generateTextEditImageWithFailover({
|
||||||
messages: [
|
candidates,
|
||||||
{
|
imageUrl: generationSourceImageUrl,
|
||||||
role: "user",
|
prompt,
|
||||||
content: [
|
|
||||||
{ type: "image_url", image_url: { url: imageUrl } },
|
|
||||||
{ type: "text", text: prompt },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
imageParams: {
|
|
||||||
imageCount: 1,
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!result?.text) {
|
|
||||||
throw new Error("模型未返回结果");
|
|
||||||
}
|
|
||||||
|
|
||||||
const { originalUrl, thumbnailUrl } = parseImageFromGenerationText(result.text);
|
|
||||||
|
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
success: true,
|
success: true,
|
||||||
data: {
|
data: {
|
||||||
originalUrl,
|
originalUrl,
|
||||||
thumbnailUrl,
|
thumbnailUrl,
|
||||||
model: selected.model.displayName,
|
model,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@ -894,7 +894,13 @@ export function AgentDetailReferenceStitchDialog({
|
|||||||
</div>
|
</div>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
|
onPointerDown={(event) => {
|
||||||
|
event.preventDefault();
|
||||||
|
event.stopPropagation();
|
||||||
|
setDraggingLineIndex(null);
|
||||||
|
}}
|
||||||
onClick={(event) => {
|
onClick={(event) => {
|
||||||
|
event.preventDefault();
|
||||||
event.stopPropagation();
|
event.stopPropagation();
|
||||||
handleRemoveCutLine(index);
|
handleRemoveCutLine(index);
|
||||||
}}
|
}}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user