diff --git a/.gitignore b/.gitignore index b4e644c00..bdc5ded58 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,10 @@ public/vendor/zoo-modeling-app/ # Drizzle migrations must be versioned (override the global *.sql ignore above) !lib/db/migrations/*.sql + +# Private smart edit runtime and media/model caches +.venv-smart-edit/ +.smart-edit-models/ +.cache/smart-edit/ + +services/video-render/dist/ diff --git a/app/api/angle-edit/route.ts b/app/api/angle-edit/route.ts index 403c2a204..6a4ab34b0 100644 --- a/app/api/angle-edit/route.ts +++ b/app/api/angle-edit/route.ts @@ -1,385 +1,10 @@ -import { NextRequest, NextResponse } from "next/server"; -import { - createMediaAssetRelation, - getMediaAssetsByUserAndUrls, - getPointsSystemConfig, - getSystemSetting, - upsertMediaAsset, -} from "@/lib/db/queries"; +import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; -import { uploadFile } from "@/lib/storage/service"; -import { db } from "@/lib/db"; -import { generatedImage } from "@/lib/db/schema"; -import { createLogger } from "@/lib/observability/logger"; -import { getImageToolCreditCost } from "@/lib/points/config"; -import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; - -const logger = createLogger("angle-edit-api"); - -// 角度编辑API配置类型 -type AngleEditConfig = { - enabled?: boolean; - falApiKey?: string; - loraScale?: number; - inferenceSteps?: number; - guidanceScale?: number; - maxImageSize?: number; - timeout?: number; -}; - -// 设置API路由最大执行时间为3分钟(180秒) +import { executeAngleEditPOST } from "@/lib/canvas/operations/angle-edit"; +export const runtime = "nodejs"; export const maxDuration = 180; - -/** - * 压缩图片生成缩略图 - */ -async function createThumbnail(buffer: Buffer): Promise { - try { - const sharp = await import("sharp"); - - const thumbnailBuffer = await sharp.default(buffer) - .resize(500, null, { - fit: "inside", - withoutEnlargement: true, - }) - .jpeg({ quality: 80 }) - .toBuffer(); - - logger.debug("thumbnail_created", { - originalBytes: buffer.length, - thumbnailBytes: thumbnailBuffer.length, - }); - return thumbnailBuffer; - } catch (error) { - logger.warn("thumbnail_creation_failed", error); - return buffer; - } -} - -/** - * 将 base64 图片上传到 S3 - */ -async function uploadBase64ToS3(base64Data: string, prefix: string = "angle-edit"): Promise<{ thumbnail: string; original: string }> { - try { - const buffer = Buffer.from(base64Data, "base64"); - const timestamp = Date.now(); - const randomStr = Math.random().toString(36).slice(2, 8); - const thumbnailFilename = `${prefix}-thumb-${timestamp}-${randomStr}.png`; - const originalFilename = `${prefix}-orig-${timestamp}-${randomStr}.png`; - - logger.debug("source_image_prepared", { originalBytes: buffer.length }); - - const thumbnailBuffer = await createThumbnail(buffer); - - const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( - thumbnailBuffer.byteOffset, - thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength - ) as ArrayBuffer; - const originalArrayBuffer = buffer.buffer.slice( - buffer.byteOffset, - buffer.byteOffset + buffer.byteLength - ) as ArrayBuffer; - - logger.debug("storage_upload_started", { - hasOriginal: originalArrayBuffer.byteLength > 0, - hasThumbnail: thumbnailArrayBuffer.byteLength > 0, - prefix, - }); - const [thumbnailResult, originalResult] = await Promise.all([ - uploadFile(thumbnailFilename, thumbnailArrayBuffer), - uploadFile(originalFilename, originalArrayBuffer), - ]); - - logger.info("storage_upload_completed", { - originalUrl: originalResult.url, - thumbnailUrl: thumbnailResult.url, - }); - - return { - thumbnail: thumbnailResult.url, - original: originalResult.url, - }; - } catch (error) { - logger.error("storage_upload_failed", error); - throw new Error(`图片上传失败: ${error instanceof Error ? error.message : "未知错误"}`); - } -} - -export async function POST(request: NextRequest) { - const startTime = Date.now(); - - try { - // 1. 验证用户登录 - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json({ error: "未登录" }, { status: 401 }); - } - - logger.info("request_received", { authenticated: true }); - - // 2. 从数据库获取配置 - const angleEditConfig = await getSystemSetting({ key: "angle_edit_config" }); - const config = (angleEditConfig?.value as AngleEditConfig) || {}; - - // 验证功能是否启用 - if (!config.enabled) { - return NextResponse.json( - { error: "角度编辑功能未启用,请联系管理员在系统设置中开启" }, - { status: 400 } - ); - } - - // 验证 API Key - if (!config.falApiKey) { - return NextResponse.json( - { error: "fal.ai API Key 未配置,请联系管理员配置" }, - { status: 500 } - ); - } - - logger.debug("config_loaded", { - enabled: Boolean(config.enabled), - hasApiKey: Boolean(config.falApiKey), - loraScale: config.loraScale || 1, - }); - - // 3. 解析请求 - const body = await request.json(); - const { - image_url, - image_base64, - // 数值参数 (官方 API 格式) - horizontal_angle, // 0-360 度 - vertical_angle, // -30 到 90 度 - zoom, // 0-10 - lora_scale, - } = body; - - if (!image_url && !image_base64) { - return NextResponse.json({ error: "缺少图片数据" }, { status: 400 }); - } - - // 使用请求中的参数或配置中的默认值 - const finalLoraScale = lora_scale ?? config.loraScale ?? 1; - const finalHorizontalAngle = horizontal_angle ?? 0; - const finalVerticalAngle = vertical_angle ?? 0; - const finalZoom = zoom ?? 5; - - logger.info("request_parameters_normalized", { - hasBase64Input: Boolean(image_base64), - hasUrlInput: Boolean(image_url), - horizontalAngle: finalHorizontalAngle, - loraScale: finalLoraScale, - verticalAngle: finalVerticalAngle, - zoom: finalZoom, - }); - - const pointsConfig = await getPointsSystemConfig(); - const pointsCharge = await prepareFeaturePointsCharge({ - userId: session.user.id, - unitCost: getImageToolCreditCost(pointsConfig, "angle-edit"), - reason: "image_angle_edit", - modelId: "qwen-image-edit-2511-multiple-angles", - note: "多角度编辑消耗积分", - pointsConfig, - }); - if (!pointsCharge.ok) { - return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); - } - - // 4. 准备图片输入 - let imageInput: string; - if (image_url) { - imageInput = image_url; - } else { - const mimeType = image_base64.startsWith("/9j/") ? "image/jpeg" : "image/png"; - imageInput = `data:${mimeType};base64,${image_base64}`; - } - - // 5. 动态导入 fal.ai SDK 并调用官方多角度 API - logger.info("provider_call_started", { - modelId: "fal-ai/qwen-image-edit-2511-multiple-angles", - }); - - const fal = await import("@fal-ai/serverless-client"); - - fal.config({ - credentials: config.falApiKey, - }); - - // 使用官方多角度 API 端点 - // 文档: https://fal.ai/models/fal-ai/qwen-image-edit-2511-multiple-angles - const result = await fal.subscribe("fal-ai/qwen-image-edit-2511-multiple-angles", { - input: { - image_urls: [imageInput], // 官方使用 image_urls (数组) - horizontal_angle: finalHorizontalAngle, // 水平旋转角度 0-360 - vertical_angle: finalVerticalAngle, // 垂直角度 -30 到 90 - zoom: finalZoom, // 缩放 0-10 - lora_scale: finalLoraScale, // LoRA 强度 - num_inference_steps: config.inferenceSteps || 28, - guidance_scale: config.guidanceScale || 4.5, // 官方默认 4.5 - enable_safety_checker: true, - output_format: "png", - }, - logs: true, - onQueueUpdate: (update) => { - if (update.status === "IN_PROGRESS") { - logger.debug("provider_progress", { status: update.status }); - } - }, - }); - - // 6. 处理结果 - const apiResult = result as { images?: Array<{ url?: string }>; prompt?: string }; - const outputImage = apiResult.images?.[0]; - if (!outputImage?.url) { - throw new Error("API 未返回有效图片"); - } - - const outputImageUrl = outputImage.url; - logger.info("provider_call_completed", { - hasGeneratedPrompt: Boolean(apiResult.prompt), - imageCount: apiResult.images?.length ?? 0, - outputImageUrl, - }); - - // 7. 下载图片并上传到我们的 S3 - const imageResponse = await fetch(outputImageUrl); - if (!imageResponse.ok) { - throw new Error(`下载输出图片失败: ${imageResponse.status}`); - } - const imageBuffer = Buffer.from(await imageResponse.arrayBuffer()); - const imageBase64Result = imageBuffer.toString("base64"); - - const s3Result = await uploadBase64ToS3(imageBase64Result, "angle-edit"); - logger.info("result_image_persisted", { - originalUrl: s3Result.original, - thumbnailUrl: s3Result.thumbnail, - }); - - // 8. 保存到数据库 - const processingTime = (Date.now() - startTime) / 1000; - - const [savedImage] = await db - .insert(generatedImage) - .values({ - userId: session.user.id, - thumbnailUrl: s3Result.thumbnail, - originalUrl: s3Result.original, - prompt: `角度编辑: H${finalHorizontalAngle}° V${finalVerticalAngle}° Z${finalZoom}`, - modelId: "qwen-image-edit-2511-multiple-angles", - createdAt: new Date(), - }) - .returning(); - - if (pointsCharge.chargeable) { - try { - await pointsCharge.consume({ - note: `多角度编辑消耗积分(${savedImage.id})`, - }); - } catch (pointsError) { - logger.error("points_consume_failed", pointsError, { - imageId: savedImage.id, - userId: session.user.id, - }); - } - } - - try { - const sourceAssets = - typeof image_url === "string" && image_url.trim() - ? await getMediaAssetsByUserAndUrls({ - userId: session.user.id, - urls: [image_url], - }) - : []; - const media = await upsertMediaAsset({ - userId: session.user.id, - mediaType: "image", - sourceType: "edited", - status: "ready", - originalUrl: s3Result.original, - previewUrl: s3Result.original, - thumbnailUrl: s3Result.thumbnail, - displayName: `角度编辑 ${finalHorizontalAngle}/${finalVerticalAngle}`.slice(0, 80), - prompt: `角度编辑: H${finalHorizontalAngle}° V${finalVerticalAngle}° Z${finalZoom}`, - modelId: "qwen-image-edit-2511-multiple-angles", - sourceTable: "GeneratedImage", - sourceRowId: savedImage.id, - primaryParentAssetId: sourceAssets[0]?.id ?? null, - metadata: { - horizontalAngle: finalHorizontalAngle, - verticalAngle: finalVerticalAngle, - zoom: finalZoom, - }, - }); - if (media) { - for (const sourceAsset of sourceAssets) { - await createMediaAssetRelation({ - userId: session.user.id, - fromAssetId: sourceAsset.id, - toAssetId: media.id, - relationType: "edited_from", - metadata: { - horizontalAngle: finalHorizontalAngle, - verticalAngle: finalVerticalAngle, - zoom: finalZoom, - }, - }); - } - } - } catch (assetError) { - logger.warn("media_asset_registration_failed", assetError); - } - - logger.info("request_completed", { - processingSeconds: Number(processingTime.toFixed(2)), - }); - - // 9. 返回结果 - return NextResponse.json({ - success: true, - image_url: s3Result.original, - thumbnail_url: s3Result.thumbnail, - image_base64: imageBase64Result, - processing_time: processingTime, - angles: { - horizontal_angle: finalHorizontalAngle, - vertical_angle: finalVerticalAngle, - zoom: finalZoom, - }, - prompt: apiResult.prompt, - }); - - } catch (error) { - logger.error("request_failed", error); - - // 输出详细的错误信息 - if (error && typeof error === "object" && "status" in error) { - const apiError = error as { status?: number; body?: unknown; message?: string }; - logger.warn("provider_error_summary", { - bodyType: typeof apiError.body, - hasBody: apiError.body != null, - message: apiError.message, - status: apiError.status, - }); - - if (apiError.status === 403) { - return NextResponse.json( - { - error: "API 访问被拒绝 (403 Forbidden)。请检查 API Key 是否正确,以及是否有权限访问该模型。", - details: apiError.body, - }, - { status: 403 } - ); - } - } - - return NextResponse.json( - { - error: error instanceof Error ? error.message : "角度编辑处理失败", - }, - { status: 500 } - ); - } +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + return executeAngleEditPOST(request, session.user.id); } diff --git a/app/api/canvas/3d/model-proxy/route.ts b/app/api/canvas/3d/model-proxy/route.ts index f7be306e6..c9a0db29a 100644 --- a/app/api/canvas/3d/model-proxy/route.ts +++ b/app/api/canvas/3d/model-proxy/route.ts @@ -14,6 +14,7 @@ const ALLOWED_MODEL_HOST_DOMAINS = [ "worldlabs.ai", "s3.1080p.hk", "img.yydd.ai", + "scqny.jiansun.vip", "blob.vercel-storage.com", "aliyuncs.com", "amazonaws.com", diff --git a/app/api/canvas/[id]/route.ts b/app/api/canvas/[id]/route.ts index bb8f5a7dc..3d8063edf 100644 --- a/app/api/canvas/[id]/route.ts +++ b/app/api/canvas/[id]/route.ts @@ -102,7 +102,7 @@ export async function GET( // 更新画布(支持 upsert:如果不存在则创建) export async function PUT( request: NextRequest, - { params }: { params: Promise<{ id: string }> } + { params }: { params: Promise<{ id: string }> }, ) { try { const { id } = await params; @@ -116,29 +116,33 @@ export async function PUT( const session = await auth(); if (!session?.user?.id) { - return NextResponse.json({ success: false, error: '未登录' }, { status: 401 }); + return NextResponse.json( + { success: false, error: "未登录" }, + { status: 401 }, + ); } const { title, description, thumbnail, canvasData } = body; - const baseCanvasRevision = body?.baseCanvasUpdatedAt || body?.baseCanvasRevision; + const baseCanvasRevision = + body?.baseCanvasUpdatedAt || body?.baseCanvasRevision; const workspaceType = body && Object.prototype.hasOwnProperty.call(body, "workspaceType") ? resolveWorkspaceType(body.workspaceType) : undefined; - // 先检查画布是否存在 - const [existingProject] = await db - .select() - .from(canvasProject) - .where( - and( - eq(canvasProject.id, id), - eq(canvasProject.userId, session.user.id) + return await db.transaction(async (tx) => { + const [existingProject] = await tx + .select() + .from(canvasProject) + .where( + and( + eq(canvasProject.id, id), + eq(canvasProject.userId, session.user.id), + ), ) - ); - - if (existingProject) { + .for("update"); if ( + existingProject && canvasData !== undefined && baseCanvasRevision && isStaleCanvasSaveBaseRevision({ @@ -149,101 +153,64 @@ export async function PUT( ) { return NextResponse.json( buildStaleCanvasSaveConflictPayload(existingProject.updatedAt), - { status: 409 } + { status: 409 }, ); } - + if (!existingProject) { + // The parent must exist before node/asset foreign keys are synchronized. + const [created] = await tx + .insert(canvasProject) + .values({ + id, + userId: session.user.id, + title: title || getDefaultWorkspaceTitle(workspaceType || "canvas"), + workspaceType: workspaceType || "canvas", + canvasData: { + nodes: [], + edges: [], + viewport: { x: 0, y: 0, zoom: 1 }, + }, + }) + .onConflictDoNothing({ target: canvasProject.id }) + .returning(); + if (!created) + return NextResponse.json( + { success: false, error: "画布已被创建,请重新加载后保存" }, + { status: 409 }, + ); + } const syncedCanvasData = canvasData !== undefined - ? await syncCanvasNodesForProject({ - projectId: id, - userId: session.user.id, - canvasData, - }) + ? await syncCanvasNodesForProject( + { projectId: id, userId: session.user.id, canvasData }, + tx, + ) : undefined; - - // 更新现有画布 - const updateData: any = { updatedAt: new Date() }; + const updateData: any = { updatedAt: new Date(Math.max(Date.now(), (existingProject?.updatedAt.getTime() || 0) + 1)) }; if (title !== undefined) updateData.title = title; if (description !== undefined) updateData.description = description; if (thumbnail !== undefined) updateData.thumbnail = thumbnail; if (workspaceType !== undefined) updateData.workspaceType = workspaceType; - if (syncedCanvasData !== undefined) updateData.canvasData = syncedCanvasData; - - const [updatedProject] = await db + if (syncedCanvasData !== undefined) + updateData.canvasData = syncedCanvasData; + const [saved] = await tx .update(canvasProject) .set(updateData) .where( and( eq(canvasProject.id, id), - eq(canvasProject.userId, session.user.id) - ) + eq(canvasProject.userId, session.user.id), + ), ) .returning(); - - return NextResponse.json({ success: true, data: updatedProject }); - } else { - const syncedCanvasData = - canvasData !== undefined - ? await syncCanvasNodesForProject({ - projectId: id, - userId: session.user.id, - canvasData, - }) - : undefined; - - const nextWorkspaceType = workspaceType || "canvas"; - const nextCanvasData = syncedCanvasData || { - nodes: [], - edges: [], - viewport: { x: 0, y: 0, zoom: 1 }, - }; - const nextTitle = title || getDefaultWorkspaceTitle(nextWorkspaceType); - const insertData: any = { - id, // 使用指定的 ID - userId: session.user.id, - title: nextTitle, - workspaceType: nextWorkspaceType, - canvasData: nextCanvasData, - }; - const conflictUpdateData: any = { - title: nextTitle, - workspaceType: nextWorkspaceType, - canvasData: nextCanvasData, - updatedAt: new Date(), - }; - if (description !== undefined) { - insertData.description = description; - conflictUpdateData.description = description; - } - if (thumbnail !== undefined) { - insertData.thumbnail = thumbnail; - conflictUpdateData.thumbnail = thumbnail; - } - - // 创建新画布(使用指定 ID),并兼容同一用户并发创建同一画布的场景。 - const [newProject] = await db - .insert(canvasProject) - .values(insertData) - .onConflictDoUpdate({ - target: canvasProject.id, - set: conflictUpdateData, - where: eq(canvasProject.userId, session.user.id), - }) - .returning(); - - if (!newProject) { - return NextResponse.json( - { success: false, error: '画布ID已被占用' }, - { status: 409 } - ); - } - - return NextResponse.json({ success: true, data: newProject }); - } + return NextResponse.json({ success: true, data: saved }); + }); } catch (error) { - console.error('更新/创建画布失败:', error); - return NextResponse.json({ success: false, error: '更新画布失败' }, { status: 500 }); + console.error("更新/创建画布失败:", error); + return NextResponse.json( + { success: false, error: "更新画布失败" }, + { status: 500 }, + ); } } diff --git a/app/api/canvas/audio/models/route.ts b/app/api/canvas/audio/models/route.ts new file mode 100644 index 000000000..92418a3dc --- /dev/null +++ b/app/api/canvas/audio/models/route.ts @@ -0,0 +1,24 @@ +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { listCanvasAudioModels } from "@/lib/video-canvas/audio-generation"; +import { audioContract } from "@/lib/video-canvas/audio-contract"; +export async function GET() { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const models = await listCanvasAudioModels(); + return NextResponse.json( + { + models: models.map(({ model }) => ({ + id: model.id, + modelId: model.modelId, + name: model.displayName, + ...audioContract(model.modelId), + })), + }, + { headers: { "Cache-Control": "no-store" } }, + ); + } catch { + return NextResponse.json({ error: "音频模型读取失败,请稍后重试" }, { status: 503 }); + } +} diff --git a/app/api/canvas/director/upload-model/route.ts b/app/api/canvas/director/upload-model/route.ts new file mode 100644 index 000000000..03600c800 --- /dev/null +++ b/app/api/canvas/director/upload-model/route.ts @@ -0,0 +1,45 @@ +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { uploadFile } from "@/lib/storage/service"; +import { + DIRECTOR_MAX_GLB_BYTES, + validateDirectorGlb, +} from "@/lib/video-canvas/director-glb"; +export const runtime = "nodejs"; +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const form = await request.formData(); + const file = form.get("file"); + if ( + !(file instanceof File) || + !/\.glb$/i.test(file.name) || + file.size > DIRECTOR_MAX_GLB_BYTES + ) + return NextResponse.json( + { error: "请上传 25 MiB 以内的 GLB 模型" }, + { status: 400 }, + ); + const buffer = await file.arrayBuffer(); + try { + validateDirectorGlb(buffer); + } catch (error) { + return NextResponse.json( + { error: error instanceof Error ? error.message : "无效 GLB" }, + { status: 400 }, + ); + } + const result = await uploadFile( + `director-${crypto.randomUUID()}.glb`, + buffer, + ); + return NextResponse.json({ success: true, data: result }); + } catch { + return NextResponse.json( + { error: "模型上传失败,请重试" }, + { status: 500 }, + ); + } +} diff --git a/app/api/canvas/generate/image/route.ts b/app/api/canvas/generate/image/route.ts index f31a24191..5060327d3 100644 --- a/app/api/canvas/generate/image/route.ts +++ b/app/api/canvas/generate/image/route.ts @@ -1,63 +1,10 @@ +import { executeCanvasImageResponse } from "@/lib/canvas/generate-image-response"; import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; -import { - createCanvasImageTask, - enqueueCanvasImageTask, - getCanvasImageTaskForUser, -} from "@/lib/canvas/image-task-queue"; -import { - generateCanvasImage, - type CanvasGenerateImageRequest, -} from "@/lib/canvas/generate-image"; -import { getPointsSystemConfig } from "@/lib/db/queries"; -import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; -import { - getImageToolCreditCost, - type ImageToolCreditFeature, -} from "@/lib/points/config"; +import { getCanvasImageTaskForUser } from "@/lib/canvas/image-task-queue"; export const maxDuration = 1200; -interface GenerateImageRequest extends CanvasGenerateImageRequest { - async?: boolean; -} - -function normalizeBillingFeature( - value: unknown -): Extract | null { - if (value === "multi-view" || value === "repaint") return value; - return null; -} - -function toGenerateImagePayload(body: GenerateImageRequest): CanvasGenerateImageRequest { - return { - allowFailover: typeof body.allowFailover === "boolean" ? body.allowFailover : undefined, - prompt: String(body.prompt || "").trim(), - negativePrompt: - typeof body.negativePrompt === "string" ? body.negativePrompt : undefined, - model: typeof body.model === "string" ? body.model : undefined, - aspectRatio: - typeof body.aspectRatio === "string" && body.aspectRatio.trim() - ? body.aspectRatio.trim() - : "auto", - imageSize: - typeof body.imageSize === "string" && body.imageSize.trim() - ? body.imageSize.trim() - : undefined, - imageCount: - typeof body.imageCount === "number" && Number.isFinite(body.imageCount) - ? body.imageCount - : 1, - promptExtend: - typeof body.promptExtend === "boolean" ? body.promptExtend : undefined, - referenceImage: - typeof body.referenceImage === "string" ? body.referenceImage : undefined, - referenceImages: Array.isArray(body.referenceImages) - ? body.referenceImages.map((item) => String(item || "").trim()).filter(Boolean) - : [], - }; -} - export async function GET(request: Request) { try { const session = await auth(); @@ -68,10 +15,7 @@ export async function GET(request: Request) { const { searchParams } = new URL(request.url); const taskId = String(searchParams.get("taskId") || "").trim(); if (!taskId) { - return NextResponse.json( - { success: false, error: "缺少 taskId 参数" }, - { status: 400 } - ); + return NextResponse.json({ success: false, error: "缺少 taskId 参数" }, { status: 400 }); } const task = await getCanvasImageTaskForUser({ taskId, userId: session.user.id }); @@ -100,105 +44,15 @@ export async function GET(request: Request) { success: false, error: error instanceof Error ? error.message : "查询任务失败", }, - { status: 500 } + { status: 500 }, ); } } export async function POST(request: Request) { - try { - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json( - { success: false, error: "未登录" }, - { status: 401 } - ); - } - - const body: GenerateImageRequest = await request.json(); - const payload: CanvasGenerateImageRequest = { - ...toGenerateImagePayload(body), - ownerKey: session.user.id, - }; - if (!payload.prompt) { - return NextResponse.json( - { success: false, error: "请输入提示词" }, - { status: 400 } - ); - } - - const billingFeature = normalizeBillingFeature(body.billingFeature); - const pointsCharge = billingFeature - ? await (async () => { - const pointsConfig = await getPointsSystemConfig(); - return prepareFeaturePointsCharge({ - userId: session.user.id, - unitCost: getImageToolCreditCost(pointsConfig, billingFeature), - reason: - billingFeature === "multi-view" - ? "image_multi_view" - : "image_repaint", - modelId: "canvas-generate-image", - note: - billingFeature === "multi-view" - ? "图片多视角生成消耗积分" - : "图片重绘消耗积分", - pointsConfig, - }); - })() - : null; - if (pointsCharge && !pointsCharge.ok) { - return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); - } - - const asyncMode = body.async === true; - if (asyncMode) { - const task = await createCanvasImageTask({ - userId: session.user.id, - request: payload as unknown as Record, - }); - await enqueueCanvasImageTask(task.id); - const currentTask = await getCanvasImageTaskForUser({ - taskId: task.id, - userId: session.user.id, - }); - if (pointsCharge?.chargeable) { - try { - await pointsCharge.consume({ - note: - billingFeature === "multi-view" - ? `图片多视角生成消耗积分(${task.id})` - : `图片重绘消耗积分(${task.id})`, - }); - } catch (pointsError) { - console.error("[Points] Failed to consume canvas image task points:", pointsError); - } - } - return NextResponse.json({ - success: true, - taskId: task.id, - status: currentTask?.status || task.status, - queueStatus: currentTask?.queueStatus, - }); - } - - const result = await generateCanvasImage(payload); - if (pointsCharge?.chargeable) { - try { - await pointsCharge.consume(); - } catch (pointsError) { - console.error("[Points] Failed to consume canvas image points:", pointsError); - } - } - return NextResponse.json({ - success: true, - data: result, - }); - } catch (error) { - console.error("[Canvas Generate Image] Error:", error); - return NextResponse.json( - { success: false, error: error instanceof Error ? error.message : "生成失败" }, - { status: 500 } - ); - } + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const body = await request.json().catch(() => ({})); + return executeCanvasImageResponse(body, session.user.id); } diff --git a/app/api/canvas/generate/video/route.ts b/app/api/canvas/generate/video/route.ts index f36270cfb..860817982 100644 --- a/app/api/canvas/generate/video/route.ts +++ b/app/api/canvas/generate/video/route.ts @@ -1,7 +1,8 @@ import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; import { executeDirectVideoGeneration } from "@/lib/ai/tools/generate-video"; -import { filterReachableRemoteUrls } from "@/lib/net/url-access"; +import { probeRemoteUrl } from "@/lib/net/url-access"; +import { validateVideoReferences } from "@/lib/video-canvas/reference-input"; import type { VideoGenerationMode, VideoRatio, VideoResolution } from "@/types/ai"; export const maxDuration = 300; @@ -48,18 +49,11 @@ export async function POST(request: Request) { ); } - const inputImages = Array.isArray(body.images) - ? body.images.map((item) => String(item || "").trim()).filter(Boolean).slice(0, 8) - : []; - const { kept: reachableImages, dropped: unreachableImages } = - await filterReachableRemoteUrls(inputImages, { timeoutMs: 3000 }); - - if (unreachableImages.length > 0) { - return NextResponse.json( - { success: false, error: "参考图片链接不可访问,请重新上传后再试" }, - { status: 400 } - ); + const referenceCheck = await validateVideoReferences(body.images, url => probeRemoteUrl(url, { timeoutMs: 8000 })); + if (referenceCheck.error) { + return NextResponse.json({ success: false, error: referenceCheck.error }, { status: referenceCheck.status }); } + const reachableImages = referenceCheck.images; const mode = body.mode || inferMode(reachableImages); const result = await executeDirectVideoGeneration({ @@ -94,13 +88,13 @@ export async function POST(request: Request) { success: true, data: { url: result.videoUrl, - metadata: result.metadata || null, + metadata: { ...result.metadata, temporaryVideoUrl: result.videoUrl, storageStatus: result.metadata?.permanentVideoUrl === result.videoUrl ? "archived" : "temporary" }, modelDisplayName: result.modelDisplayName, modelId: result.modelId, modelRef: result.modelRef, mode, images: reachableImages, - droppedImages: unreachableImages, + droppedImages: [], }, }); } catch (error) { diff --git a/app/api/canvas/generation-tasks/route.ts b/app/api/canvas/generation-tasks/route.ts new file mode 100644 index 000000000..056ac2819 --- /dev/null +++ b/app/api/canvas/generation-tasks/route.ts @@ -0,0 +1,125 @@ +import { smartEditRequestSchema } from "@/lib/video-canvas/smart-edit/contract"; +import { audioRequestSchema } from "@/lib/video-canvas/audio-contract"; +import { canResumeCanvasTask } from "@/lib/video-canvas/task-recovery"; +import { imageOperationSchema } from "@/lib/video-canvas/image-operation"; +import { after, NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { z } from "zod"; +import { canvasTaskRequestSchema } from "@/lib/video-canvas/task-contract"; +import { + cancelCanvasGenerationTask, + createCanvasGenerationTask, + listCanvasGenerationTasks, + ownsTaskProject, +} from "@/lib/video-canvas/task-store"; +import { runCanvasGenerationQueue } from "@/lib/video-canvas/task-worker"; +export const runtime = "nodejs"; +export const maxDuration = 900; + +async function owner(projectId: unknown) { + const session = await auth(); + if (!session?.user?.id) throw Object.assign(Error("未登录"), { status: 401 }); + if (!z.string().uuid().safeParse(projectId).success) + throw Object.assign(Error("画布 ID 无效"), { status: 400 }); + if (!(await ownsTaskProject(session.user.id, projectId as string))) + throw Object.assign(Error("画布不存在"), { status: 404 }); + return session.user.id; +} +const fail = (error: unknown) => + NextResponse.json( + { success: false, error: error instanceof Error ? error.message : "任务操作失败" }, + { status: Number((error as any)?.status) || (error instanceof SyntaxError ? 400 : 500) }, + ); +const kick = (userId: string, projectId: string) => + after(() => + runCanvasGenerationQueue(userId, projectId).catch((error) => + console.error("[Canvas tasks]", error), + ), + ); +export async function POST(request: Request) { + try { + const text = await request.text(); + if (new TextEncoder().encode(text).byteLength > 262144) + throw Object.assign(Error("任务数据过大,请使用已上传的素材链接"), { status: 413 }); + const parsed = canvasTaskRequestSchema.safeParse(JSON.parse(text)); + if (!parsed.success) throw Object.assign(Error("任务参数不合法"), { status: 400 }); + if ( + parsed.data.kind === "image-operation" && + !imageOperationSchema.safeParse(parsed.data.params).success + ) + throw Object.assign(Error("图片处理参数不合法"), { status: 400 }); + if ( + parsed.data.attachment && + ((parsed.data.attachment.type === "story-asset" && parsed.data.kind !== "image") || + (parsed.data.attachment.type === "story-prompts" && parsed.data.kind !== "analysis")) + ) + throw Object.assign(Error("脚本结果类型不匹配"), { status: 400 }); + if (parsed.data.kind === "audio" && !audioRequestSchema.safeParse(parsed.data.params).success) + throw Object.assign(Error("音频参数不合法"), { status: 400 }); + if ( + parsed.data.kind === "smart-edit" && + !smartEditRequestSchema.safeParse(parsed.data.params).success + ) + throw Object.assign(Error("剪辑参数无效"), { status: 400 }); + const userId = await owner(parsed.data.projectId); + const task = await createCanvasGenerationTask(userId, parsed.data); + kick(userId, parsed.data.projectId); + return NextResponse.json({ success: true, task }, { status: 202 }); + } catch (error) { + return fail(error); + } +} +export async function GET(request: Request) { + try { + const projectId = new URL(request.url).searchParams.get("projectId") || ""; + const userId = await owner(projectId); + const tasks = await listCanvasGenerationTasks(userId, projectId); + if (tasks.some((task) => task.status === "pending" || canResumeCanvasTask(task))) + kick(userId, projectId); + return NextResponse.json( + { success: true, tasks }, + { headers: { "Cache-Control": "no-store" } }, + ); + } catch (error) { + return fail(error); + } +} +export async function DELETE(request: Request) { + try { + const { projectId, taskId } = await request.json(); + const userId = await owner(projectId); + if (!z.string().uuid().safeParse(taskId).success) + throw Object.assign(Error("任务 ID 无效"), { status: 400 }); + const result = await cancelCanvasGenerationTask(userId, projectId, taskId); + if (!result.task) + return NextResponse.json({ success: false, error: "任务不存在" }, { status: 404 }); + if (!result.ok) + return NextResponse.json( + { + success: false, + error: + result.task.status === "running" + ? "任务已提交供应商,当前线路不支持中止。结果会继续保存,请勿重复生成。" + : "任务已结束或状态待核实,请先刷新任务状态。", + task: result.task, + }, + { status: 409 }, + ); + return NextResponse.json({ success: true, task: result.task }); + } catch (error) { + return fail(error); + } +} +export async function PATCH(request: Request) { + try { + const { projectId, taskId, nodeId } = await request.json(); + const userId = await owner(projectId); + if (!z.string().uuid().safeParse(taskId).success || typeof nodeId !== "string") + return NextResponse.json({ success: false }, { status: 400 }); + const { hideCanvasGenerationNode } = await import("@/lib/video-canvas/task-store"); + const success = await hideCanvasGenerationNode(userId, projectId, taskId, nodeId); + return NextResponse.json({ success }, { status: success ? 200 : 404 }); + } catch (error) { + return fail(error); + } +} diff --git a/app/api/canvas/media/upload/route.ts b/app/api/canvas/media/upload/route.ts new file mode 100644 index 000000000..6cd31303d --- /dev/null +++ b/app/api/canvas/media/upload/route.ts @@ -0,0 +1,50 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { uploadFile } from "@/lib/storage/service"; + +export const runtime = "nodejs"; +export const maxDuration = 180; +const probe = promisify(execFile); +const running = new Set(); +const formats: Record = { + "video/mp4": "mp4", "video/quicktime": "mov", "video/webm": "webm", + "audio/mpeg": "mp3", "audio/mp3": "mp3", "audio/mp4": "m4a", "audio/x-m4a": "m4a", + "audio/wav": "wav", "audio/x-wav": "wav", "audio/ogg": "ogg", "audio/flac": "flac", "audio/webm": "webm", +}; + +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + if (running.has(session.user.id) || running.size >= 4) return NextResponse.json({ success: false, error: "正在上传,请稍后重试" }, { status: 429 }); + running.add(session.user.id); + let dir = ""; + try { + const form = await request.formData(); + const file = form.get("file"); + if (!(file instanceof File) || !formats[file.type]) return NextResponse.json({ success: false, error: "请上传 MP4、MOV、WebM 视频,或 MP3、M4A、WAV、OGG、FLAC 音频" }, { status: 400 }); + if (!file.size || file.size > 125 * 1024 * 1024) return NextResponse.json({ success: false, error: "素材必须非空且不超过 125 MiB" }, { status: 413 }); + dir = await mkdtemp(join(tmpdir(), "canvas-upload-")); + const input = join(dir, "input"); + const buffer = await file.arrayBuffer(); + await writeFile(input, Buffer.from(buffer)); + const { stdout } = await probe("ffprobe", ["-v", "error", "-protocol_whitelist", "file", "-format_whitelist", "mov,matroska,webm,mp3,wav,ogg,flac", "-show_streams", "-show_format", "-of", "json", input], { timeout: 15000, maxBuffer: 1024 * 1024 }); + const info = JSON.parse(stdout); + const kind = file.type.startsWith("video/") ? "video" : "audio"; + const stream = info.streams?.find((item: { codec_type: string }) => item.codec_type === kind); + const duration = Number(info.format?.duration); + if (!stream || !Number.isFinite(duration) || duration <= 0 || duration > 600) return NextResponse.json({ success: false, error: "请上传内容完整、10 分钟以内的音视频" }, { status: 400 }); + const uploaded = await uploadFile(`canvas-${randomUUID()}.${formats[file.type]}`, buffer); + return NextResponse.json({ success: true, data: { url: uploaded.url, duration, width: stream.width, height: stream.height } }); + } catch { + return NextResponse.json({ success: false, error: "素材上传失败,请确认文件完整或稍后重试" }, { status: 400 }); + } finally { + running.delete(session.user.id); + if (dir) await rm(dir, { recursive: true, force: true }); + } +} diff --git a/app/api/canvas/prompt-tools/route.ts b/app/api/canvas/prompt-tools/route.ts new file mode 100644 index 000000000..38d9476b4 --- /dev/null +++ b/app/api/canvas/prompt-tools/route.ts @@ -0,0 +1,61 @@ +import { generateText } from "ai"; +import { createLogger } from "@/lib/observability/logger"; +const logger = createLogger("canvas-prompt-tools"); +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; +import { getEnabledAiModelsWithProvider, getSystemSetting } from "@/lib/db/queries"; +import { VIDEO_CANVAS_FUNCTION_CONFIG_KEY, normalizeVideoCanvasFunctionConfig } from "@/lib/video-canvas/config"; +import { promptToolErrorMessage, promptToolFunctionKey, promptToolSchema, selectPromptToolModel } from "@/lib/video-canvas/prompt-tools"; + +export const runtime = "nodejs"; +export const maxDuration = 120; +const running = new Set(); + +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const parsed = promptToolSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) return NextResponse.json({ success: false, error: parsed.error.issues[0]?.message || "参数无效" }, { status: 400 }); + if (running.has(session.user.id)) return NextResponse.json({ success: false, error: "提示词处理中,请稍候" }, { status: 429 }); + running.add(session.user.id); + try { + const [setting, available] = await Promise.all([ + getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }), + getEnabledAiModelsWithProvider(false), + ]); + const config = normalizeVideoCanvasFunctionConfig(setting?.value); + const definition = config.functions[promptToolFunctionKey(parsed.data.mode)]; + if (!config.enabled || !definition?.enabled) return NextResponse.json({ success: false, error: "此提示词功能已停用" }, { status: 403 }); + const requested = parsed.data.model && parsed.data.model !== "auto" + ? parsed.data.model : definition.model && definition.model !== "auto" + ? definition.model : config.defaultMultimodalModel; + let selected; + try { + selected = selectPromptToolModel(available + .filter(item => item.provider.isEnabled && item.model.isEnabled && !item.model.isSubModel) + .map(item => ({ ...item.model, providerType: item.provider.type })), requested); + } catch (error) { + return NextResponse.json({ success: false, error: (error as Error).message }, { status: 400 }); + } + const model = await getLanguageModelById(selected.id); + if (!model) return NextResponse.json({ success: false, error: "提示词模型初始化失败" }, { status: 502 }); + const { text } = await generateText({ + model, + system: definition.prompt, + prompt: parsed.data.prompt, + temperature: parsed.data.mode === "translate" ? 0.1 : 0.4, + maxOutputTokens: 4000, + maxRetries: 0, + abortSignal: AbortSignal.any([request.signal, AbortSignal.timeout(90_000)]), + }); + if (!text.trim()) throw new Error("empty model output"); + return NextResponse.json({ success: true, data: { text: text.trim(), model: selected.id, modelName: selected.displayName || selected.modelId } }); + } catch (error) { + const detail = error as { name?: string; statusCode?: number; cause?: { code?: string }; data?: { error?: { code?: string } } }; + logger.warn("Prompt tool request failed", { name: detail?.name, statusCode: detail?.statusCode, code: detail?.data?.error?.code || detail?.cause?.code }); + return NextResponse.json({ success: false, error: promptToolErrorMessage(error) }, { status: 502 }); + } finally { + running.delete(session.user.id); + } +} diff --git a/app/api/canvas/smart-edit/capabilities/route.ts b/app/api/canvas/smart-edit/capabilities/route.ts new file mode 100644 index 000000000..b67185b4b --- /dev/null +++ b/app/api/canvas/smart-edit/capabilities/route.ts @@ -0,0 +1,47 @@ +import { auth } from "@/app/(auth)/auth"; +import { NextResponse } from "next/server"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import path from "node:path"; +import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; +import { isCanvasVisionModel } from "@/lib/video-canvas/model-matching"; +import { remoteRenderConfig } from "@/lib/video-canvas/smart-edit/remote-render"; +export async function GET() { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + let capabilities = { scenes: false, beats: false, asr: false, ffmpeg: false, ffprobe: false }; + try { + const { stdout } = await promisify(execFile)( + process.env.SMART_EDIT_PYTHON || "python3", + [path.resolve("services/smart-edit/analyze.py"), "--check"], + { timeout: 10000, maxBuffer: 65536 }, + ); + capabilities = JSON.parse(stdout); + } catch {} + let renderAvailable = capabilities.ffmpeg; + const remote = remoteRenderConfig(); + if (remote) { + try { + const response = await fetch(remote.base + "/health", { + headers: { authorization: "Bearer " + remote.token }, + signal: AbortSignal.timeout(3000), + redirect: "error", + cache: "no-store", + }); + renderAvailable = response.ok && (await response.json()).status === "ok"; + } catch { + renderAvailable = false; + } + } + const models = (await getEnabledAiModelsWithProvider(false)) + .filter((item) => isCanvasVisionModel({ ...item.model, providerType: item.provider.type })) + .map(({ model }) => ({ id: model.id, name: model.displayName, modelId: model.modelId })); + return NextResponse.json( + { capabilities: { ...capabilities, render: renderAvailable }, models }, + { headers: { "Cache-Control": "no-store" } }, + ); + } catch { + return NextResponse.json({ error: "剪辑服务配置读取失败" }, { status: 503 }); + } +} diff --git a/app/api/canvas/video/archive/route.ts b/app/api/canvas/video/archive/route.ts new file mode 100644 index 000000000..f76790464 --- /dev/null +++ b/app/api/canvas/video/archive/route.ts @@ -0,0 +1,23 @@ +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { auth } from "@/app/(auth)/auth"; +import { archiveCanvasVideo } from "@/lib/video-canvas/archive-video"; + +export const runtime = "nodejs"; +export const maxDuration = 300; +const running = new Set(); +const schema = z.object({ url: z.string().url().max(12000) }); +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const parsed = schema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) return NextResponse.json({ success: false, error: "视频地址无效" }, { status: 400 }); + if (running.has(session.user.id)) return NextResponse.json({ success: false, error: "视频正在保存,请稍候" }, { status: 429 }); + running.add(session.user.id); + try { + const url = await archiveCanvasVideo(parsed.data.url); + return NextResponse.json({ success: true, data: { url, storageStatus: "archived", permanentVideoUrl: url } }); + } catch { + return NextResponse.json({ success: false, error: "视频转存失败,原视频仍保留;请稍后重试保存,无需重新生成" }, { status: 502 }); + } finally { running.delete(session.user.id); } +} diff --git a/app/api/canvas/video/process/route.ts b/app/api/canvas/video/process/route.ts index 923d94e60..9fd412245 100644 --- a/app/api/canvas/video/process/route.ts +++ b/app/api/canvas/video/process/route.ts @@ -1,39 +1,96 @@ -import {auth} from '@/app/(auth)/auth'; -import {NextResponse} from 'next/server'; -import {z} from 'zod'; -import {mkdtemp,readFile,rm} from 'node:fs/promises'; -import {tmpdir} from 'node:os'; -import {join} from 'node:path'; -import {randomUUID} from 'node:crypto'; -import {uploadFile} from '@/lib/storage/service'; -import {downloadCanvasMedia} from '@/lib/video-canvas/download-media'; -import {processVideoFile} from '@/lib/video-canvas/media-processing'; -export const runtime='nodejs'; -export const maxDuration=300; -const running=new Set(); -const schema=z.object({operation:z.enum(['crop','audio-video']),url:z.string().url(),rect:z.object({x:z.number(),y:z.number(),width:z.number(),height:z.number()}).optional()}); -export async function POST(request:Request) { - const session=await auth(); - if(!session?.user?.id) return NextResponse.json({success:false,error:'未登录'},{status:401}); - const parsed=schema.safeParse(await request.json().catch(()=>null)); - if(!parsed.success) return NextResponse.json({success:false,error:'视频处理参数无效'},{status:400}); - if(running.has(session.user.id)||running.size>=2) return NextResponse.json({success:false,error:'视频处理中,请稍后再提交'},{status:429}); +import { auth } from "@/app/(auth)/auth"; +import { NextResponse } from "next/server"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { randomUUID } from "node:crypto"; +import { uploadFile } from "@/lib/storage/service"; +import { downloadCanvasMedia } from "@/lib/video-canvas/download-media"; +import { processVideoFile } from "@/lib/video-canvas/media-processing"; +import { videoProcessSchema } from "@/lib/video-canvas/media-contract"; +export const runtime = "nodejs"; +export const maxDuration = 300; +const running = new Set(); +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const parsed = videoProcessSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) + return NextResponse.json({ success: false, error: "视频处理参数无效" }, { status: 400 }); + if (running.has(session.user.id) || running.size >= 2) + return NextResponse.json( + { success: false, error: "视频处理中,请稍后再提交" }, + { status: 429 }, + ); running.add(session.user.id); - let dir=''; + let dir = ""; try { - dir=await mkdtemp(join(tmpdir(),'canvas-video-')); - const input=join(dir,'input'); - await downloadCanvasMedia(parsed.data.url,input); - const save=async(name:string)=>{const buffer=await readFile(join(dir,name));return (await uploadFile(`canvas-video-${randomUUID()}-${name}`,new Uint8Array(buffer).buffer)).url;}; - if(parsed.data.operation==='crop') { - const metadata=await processVideoFile(input,join(dir,'crop.mp4'),'crop',parsed.data.rect); - return NextResponse.json({success:true,data:{url:await save('crop.mp4'),...metadata}}); + dir = await mkdtemp(join(tmpdir(), "canvas-video-")); + const input = join(dir, "input"); + await downloadCanvasMedia(parsed.data.url, input); + const save = async (name: string) => { + const buffer = await readFile(join(dir, name)); + return ( + await uploadFile(`canvas-video-${randomUUID()}-${name}`, new Uint8Array(buffer).buffer) + ).url; + }; + if (parsed.data.operation === "frame" || parsed.data.operation === "slow") { + const name = parsed.data.operation === "frame" ? "frame.png" : "slow.mp4"; + const metadata = await processVideoFile( + input, + join(dir, name), + parsed.data.operation, + undefined, + undefined, + parsed.data, + ); + return NextResponse.json({ success: true, data: { url: await save(name), ...metadata } }); } - const metadata=await processVideoFile(input,join(dir,'audio.m4a'),'extract-audio'); - await processVideoFile(input,join(dir,'silent.mp4'),'silent-video'); - return NextResponse.json({success:true,data:{audioUrl:await save('audio.m4a'),url:await save('silent.mp4'),...metadata}}); - } catch(error) { - const message=error instanceof Error ? error.message : '视频处理失败'; - return NextResponse.json({success:false,error:message.startsWith('Command failed')?'视频处理失败,请确认文件完整且格式受支持':message},{status:400}); - } finally {running.delete(session.user.id);if(dir)await rm(dir,{recursive:true,force:true});} + if (parsed.data.operation === "trim") { + const metadata = await processVideoFile( + input, + join(dir, "clip.mp4"), + "trim", + undefined, + parsed.data.range, + ); + return NextResponse.json({ + success: true, + data: { url: await save("clip.mp4"), ...metadata }, + }); + } + if (parsed.data.operation === "crop") { + const metadata = await processVideoFile( + input, + join(dir, "crop.mp4"), + "crop", + parsed.data.rect, + ); + return NextResponse.json({ + success: true, + data: { url: await save("crop.mp4"), ...metadata }, + }); + } + const metadata = await processVideoFile(input, join(dir, "audio.m4a"), "extract-audio"); + await processVideoFile(input, join(dir, "silent.mp4"), "silent-video"); + return NextResponse.json({ + success: true, + data: { audioUrl: await save("audio.m4a"), url: await save("silent.mp4"), ...metadata }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : "视频处理失败"; + return NextResponse.json( + { + success: false, + error: message.startsWith("Command failed") + ? "视频处理失败,请确认文件完整且格式受支持" + : message, + }, + { status: 400 }, + ); + } finally { + running.delete(session.user.id); + if (dir) await rm(dir, { recursive: true, force: true }); + } } diff --git a/app/api/canvas/workflow/analyze/route.ts b/app/api/canvas/workflow/analyze/route.ts index 08c5633db..3d832b2be 100644 --- a/app/api/canvas/workflow/analyze/route.ts +++ b/app/api/canvas/workflow/analyze/route.ts @@ -1,993 +1,12 @@ -import { isCanvasLanguageModel, isCanvasVisionModel } from "@/lib/video-canvas/model-matching"; -import { generateText } from "ai"; -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; +import { executeWorkflowAnalysisResponse } from "@/lib/video-canvas/workflow-analysis-response"; import { NextResponse } from "next/server"; -import sharp from "sharp"; -import { z } from "zod"; import { auth } from "@/app/(auth)/auth"; -import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; -import { inlineVisionImages } from "@/lib/ai/vision-inline"; -import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags"; -import { - getAgentAiModelWithProvider, - getEnabledAiModelsWithProvider, -} from "@/lib/db/queries"; - export const runtime = "nodejs"; export const maxDuration = 180; - -const execFileAsync = promisify(execFile); -const VIDEO_ANALYSIS_FRAME_COUNT = 4; -const VIDEO_ANALYSIS_MAX_SIDE = 960; -const VIDEO_ANALYSIS_TIMEOUT_MS = 15000; - -const requestSchema = z.object({ - mode: z.enum(["video-analysis", "character-scene-extraction", "story-script", "agent-plan"]), - prompt: z.string().trim().optional(), - imageUrls: z.array(z.string().trim()).optional(), - videoUrls: z.array(z.string().trim()).optional(), - nodeTitle: z.string().trim().optional(), - preferredModelRef: z.string().trim().optional(), - model: z.string().trim().optional(), -}); - -function sanitizeUrls(value: unknown, limit = 6) { - if (!Array.isArray(value)) return []; - return value - .map((item) => String(item || "").trim()) - .filter((url) => /^https?:\/\//i.test(url)) - .slice(0, limit); -} - -function supportsVision(model: { - supportsVision?: boolean | null; - capabilities?: unknown; - defaultParams?: unknown; -}) { - return ( - Boolean(model.supportsVision) || - isAiModelMultimodal({ - capabilities: model.capabilities, - defaultParams: model.defaultParams, - }) - ); -} - -function extractJsonBlock>(text: string): T | null { - const matched = String(text || "").match(/\{[\s\S]*\}/); - if (!matched) return null; - try { - return JSON.parse(matched[0]) as T; - } catch { - return null; - } -} - -function normalizeText(value: unknown) { - return String(value || "").trim(); -} - -function shouldUseLocalWorkflowE2E(request: Request) { - const hostname = new URL(request.url).hostname; - return ( - process.env.CANVAS_E2E_LOCAL === "true" && - (hostname === "127.0.0.1" || hostname === "localhost" || hostname === "::1") - ); -} - -function normalizeStringList(value: unknown, limit = 8) { - if (!Array.isArray(value)) return []; - return Array.from( - new Set( - value - .map((item) => String(item || "").trim()) - .filter(Boolean) - .slice(0, limit) - ) - ); -} - -function normalizeShotIdeas(value: unknown) { - if (!Array.isArray(value)) return []; - return value - .map((item, index) => { - const raw = - item && typeof item === "object" && !Array.isArray(item) - ? (item as Record) - : {}; - const title = normalizeText(raw.title || `镜头 ${index + 1}`); - const description = normalizeText(raw.description || raw.prompt || raw.summary); - if (!title && !description) return null; - return { - title: title || `镜头 ${index + 1}`, - description, - }; - }) - .filter( - (item): item is { title: string; description: string } => Boolean(item) - ) - .slice(0, 6); -} - -function normalizeVideoAnalysisRows( - value: unknown, - framePreviews: string[], -) { - if (!Array.isArray(value)) return []; - return value - .map((item, index) => { - const raw = - item && typeof item === "object" && !Array.isArray(item) - ? (item as Record) - : {}; - const sceneDescription = normalizeText( - raw.sceneDescription || raw.scene || raw.title || raw.description, - ); - const narrative = normalizeText( - raw.narrative || raw.story || raw.storyContent || raw.narration || raw.summary, - ); - const shot = normalizeText(raw.shot || raw.shotNumber || raw.index || index + 1); - const startTime = normalizeText(raw.startTime || raw.start || raw.start_time); - const endTime = normalizeText(raw.endTime || raw.end || raw.end_time); - const duration = normalizeText(raw.duration || raw.durationSeconds || raw.duration_seconds); - const shotSize = normalizeText(raw.shotSize || raw.frameSize || raw.shot_type || raw.sceneScale); - const cameraAngle = normalizeText(raw.cameraAngle || raw.angle || raw.camera_angle); - const cameraMovement = normalizeText( - raw.cameraMovement || raw.cameraMove || raw.camera_motion || raw.motionType, - ); - const focusDepth = normalizeText( - raw.focusDepth || raw.depthOfField || raw.focalDepth || raw.focus_depth, - ); - const lighting = normalizeText(raw.lighting || raw.light || raw.lightDesign); - const backgroundMusic = normalizeText( - raw.backgroundMusic || raw.music || raw.bgm, - ); - const voiceAndSfx = normalizeText( - raw.voiceAndSfx || raw.sound || raw.audio || raw.sfx, - ); - const imagePrompt = normalizeText( - raw.imagePrompt || raw.image_generation_prompt || raw.image, - ); - const motionPrompt = normalizeText( - raw.motionPrompt || raw.motion || raw.videoPrompt || raw.video_motion_prompt, - ); - const keyframeIndex = Number( - raw.keyframeIndex || raw.frameIndex || raw.key_frame_index || index + 1, - ); - if ( - !sceneDescription && - !narrative && - !backgroundMusic && - !voiceAndSfx && - !imagePrompt && - !motionPrompt && - !shotSize && - !cameraAngle && - !cameraMovement && - !focusDepth && - !lighting - ) { - return null; - } - const normalizedFrameIndex = - Number.isFinite(keyframeIndex) && keyframeIndex > 0 - ? Math.max(0, Math.min(framePreviews.length - 1, Math.round(keyframeIndex) - 1)) - : index % Math.max(framePreviews.length, 1); - return { - shot: shot || String(index + 1), - startTime: startTime || (index === 0 ? "0" : ""), - endTime, - duration, - sceneDescription: sceneDescription || `镜头 ${index + 1}`, - narrative: narrative || sceneDescription || `镜头 ${index + 1}`, - shotSize: shotSize || "中景至近景", - cameraAngle: cameraAngle || "平视", - cameraMovement: cameraMovement || "缓慢推镜头 (Slow Zoom In)", - focusDepth: focusDepth || "浅景深", - lighting: lighting || "冷调低沉照明,主体光线柔和。", - backgroundMusic: backgroundMusic || "无", - voiceAndSfx: voiceAndSfx || "无", - imagePrompt: imagePrompt || sceneDescription || "延续当前视频气质生成关键帧", - motionPrompt: motionPrompt || "保持镜头运动自然、连贯、符合前后节奏。", - keyframeUrl: framePreviews[normalizedFrameIndex] || "", - }; - }) - .filter( - ( - item, - ): item is { - shot: string; - startTime: string; - endTime: string; - duration: string; - sceneDescription: string; - narrative: string; - shotSize: string; - cameraAngle: string; - cameraMovement: string; - focusDepth: string; - lighting: string; - backgroundMusic: string; - voiceAndSfx: string; - imagePrompt: string; - motionPrompt: string; - keyframeUrl: string; - } => Boolean(item), - ) - .slice(0, 8); -} - -function normalizeStoryShot(value: unknown, index: number) { - const text = normalizeText(value); - if (/^\d+$/.test(text)) return text.padStart(2, "0"); - return text || String(index + 1).padStart(2, "0"); -} - -type NormalizedStoryScriptRow = { - shot: string; - duration: string; - scene: string; - sceneDescription: string; - shotSize: string; - lighting: string; - dialogue: string; - backgroundMusic: string; - voiceAndSfx: string; - cameraMovement: string; - imagePrompt: string; - motionPrompt: string; - keyframeUrl: string; -}; - -function normalizeStoryScriptRows(value: unknown): NormalizedStoryScriptRow[] { - if (!Array.isArray(value)) return []; - return value - .map((item, index) => { - if (Array.isArray(item)) { - const cells = item.map((cell) => normalizeText(cell)); - const shot = normalizeStoryShot(cells[0], index); - const secondCellLooksLikeDuration = /^\d+(\.\d+)?\s*(s|秒)?$/i.test(cells[1] || ""); - const duration = secondCellLooksLikeDuration ? cells[1] : "5s"; - const scene = secondCellLooksLikeDuration ? cells[2] || `镜头 ${shot}` : cells[1] || `镜头 ${shot}`; - const sceneDescription = secondCellLooksLikeDuration ? cells[2] || scene : cells[2] || cells[3] || scene; - return { - shot, - duration, - scene, - sceneDescription, - shotSize: (secondCellLooksLikeDuration ? cells[3] : "") || "中景", - lighting: (secondCellLooksLikeDuration ? cells[4] : cells[4]) || "电影感光影", - dialogue: (secondCellLooksLikeDuration ? cells[5] : "") || "+", - backgroundMusic: "无", - voiceAndSfx: (secondCellLooksLikeDuration ? cells[6] : cells[3]) || "无", - cameraMovement: (secondCellLooksLikeDuration ? cells[7] : "") || "缓慢推镜头", - imagePrompt: (secondCellLooksLikeDuration ? cells[8] : cells[5]) || sceneDescription, - motionPrompt: - (secondCellLooksLikeDuration ? cells[9] || cells[7] : cells[6]) || - "保持镜头运动自然、连贯,围绕主体动作推进。", - keyframeUrl: "", - }; - } - - const raw = - item && typeof item === "object" && !Array.isArray(item) - ? (item as Record) - : {}; - const shot = normalizeStoryShot(raw.shot || raw.shotNumber || raw.index || raw.id, index); - const duration = normalizeText(raw.duration || raw.length || raw.time || raw.seconds); - const scene = normalizeText(raw.scene || raw.location || raw.title || `镜头 ${shot}`); - const sceneDescription = normalizeText( - raw.sceneDescription || - raw.description || - raw.visual || - raw.picture || - raw.screen || - raw.summary, - ); - const shotSize = normalizeText( - raw.shotSize || raw.frameSize || raw.shot_type || raw.sceneScale || raw.cameraShot, - ); - const lighting = normalizeText( - raw.lighting || raw.light || raw.atmosphere || raw.mood || raw.lightMood, - ); - const dialogue = normalizeText( - raw.dialogue || raw.dialog || raw.narration || raw.voiceover || raw.monologue || raw.line, - ); - const cameraMovement = normalizeText( - raw.cameraMovement || raw.cameraMove || raw.movement || raw.lensMovement || raw.motion, - ); - const imagePrompt = normalizeText( - raw.imagePrompt || - raw.image_generation_prompt || - raw.image || - raw.keyframePrompt || - raw.keyFramePrompt, - ); - const motionPrompt = normalizeText( - raw.motionPrompt || - raw.videoPrompt || - raw.video_motion_prompt || - raw.motion || - raw.cameraMovement, - ); - const backgroundMusic = normalizeText( - raw.backgroundMusic || raw.music || raw.bgm || raw.soundtrack, - ); - const voiceAndSfx = normalizeText( - raw.voiceAndSfx || raw.voice || raw.sound || raw.audio || raw.sfx, - ); - - if ( - !scene && - !sceneDescription && - !imagePrompt && - !motionPrompt && - !backgroundMusic && - !voiceAndSfx - ) { - return null; - } - - return { - shot, - duration: duration || "5s", - scene, - sceneDescription: sceneDescription || imagePrompt || scene || `镜头 ${shot}`, - shotSize: shotSize || "中景", - lighting: lighting || "电影感光影", - dialogue: dialogue || "+", - backgroundMusic: backgroundMusic || "无", - voiceAndSfx: voiceAndSfx || "无", - cameraMovement: cameraMovement || "缓慢推镜头", - imagePrompt: imagePrompt || sceneDescription || scene || `镜头 ${shot}关键帧`, - motionPrompt: motionPrompt || cameraMovement || "保持镜头运动自然、连贯,围绕主体动作推进。", - keyframeUrl: "", - }; - }) - .filter( - ( - item, - ): item is NormalizedStoryScriptRow => Boolean(item), - ) - .slice(0, 18); -} - -const storyExpansionPhases = [ - { - description: "建立镜头,交代环境、人物位置和当前压力", - cameraMovement: "缓慢推镜头", - imagePrompt: "广角或中景建立画面,环境信息清晰,主体位置明确", - lighting: "环境主光明确,氛围基调清晰", - motionPrompt: "镜头缓慢推进或轻微横移,建立空间关系", - shotSize: "中景", - }, - { - description: "动作推进,突出人物的关键行为或剧情转折", - cameraMovement: "跟随主体动作推进", - imagePrompt: "中景表现人物动作,服装、道具和光影保持连续", - lighting: "主体光影连续,动作区域清晰", - motionPrompt: "跟随主体动作推进,节奏自然连贯", - shotSize: "中景", - }, - { - description: "情绪反应,用近景或特写强化人物心理和冲突", - cameraMovement: "轻微推近或稳定特写", - imagePrompt: "近景或特写,突出表情、眼神、手部或关键道具细节", - lighting: "面部光线突出,情绪反差增强", - motionPrompt: "轻微推近或稳定特写,强调情绪变化", - shotSize: "特写", - }, - { - description: "关系与氛围,表现周围人物、环境反馈或声画张力", - cameraMovement: "横移扫过环境或切换视角", - imagePrompt: "加入环境反馈、群像反应或空间层次,电影感光影", - lighting: "场景氛围增强,空间层次明显", - motionPrompt: "镜头扫过环境或切换视角,保持节奏递进", - shotSize: "全景", - }, - { - description: "转场收束,为下一镜头保留动作、视线或情绪钩子", - cameraMovement: "顺势后移、上摇或转场", - imagePrompt: "构图留出转场方向,动作或视线指向下一段剧情", - lighting: "光影引导视线并保留转场方向", - motionPrompt: "镜头后移、上摇或顺势转场,衔接下一镜头", - shotSize: "大远景", - }, -]; - -function expandStoryScriptRowsToTarget( - rows: NormalizedStoryScriptRow[], - targetCount = 18, -): NormalizedStoryScriptRow[] { - if (rows.length === 0) return []; - if (rows.length >= targetCount) { - return rows.slice(0, targetCount).map((row, index) => ({ - ...row, - shot: String(index + 1).padStart(2, "0"), - })); - } - - const baseCount = Math.floor(targetCount / rows.length); - const extraCount = targetCount % rows.length; - const expanded: NormalizedStoryScriptRow[] = []; - - rows.forEach((row, rowIndex) => { - const segmentCount = baseCount + (rowIndex < extraCount ? 1 : 0); - for (let segmentIndex = 0; segmentIndex < segmentCount; segmentIndex += 1) { - const phase = - storyExpansionPhases[ - Math.min( - storyExpansionPhases.length - 1, - Math.floor((segmentIndex / Math.max(segmentCount, 1)) * storyExpansionPhases.length), - ) - ]; - expanded.push({ - ...row, - shot: String(expanded.length + 1).padStart(2, "0"), - sceneDescription: `${phase.description}:${row.sceneDescription}`, - shotSize: row.shotSize || phase.shotSize, - lighting: row.lighting || phase.lighting, - cameraMovement: row.cameraMovement || phase.cameraMovement, - imagePrompt: `${row.imagePrompt};${phase.imagePrompt}`, - motionPrompt: `${phase.motionPrompt};${row.motionPrompt}`, - }); - } - }); - - return expanded.slice(0, targetCount); -} - -function normalizeCharacters(value: unknown) { - if (!Array.isArray(value)) return []; - return value - .map((item) => { - const raw = - item && typeof item === "object" && !Array.isArray(item) - ? (item as Record) - : {}; - const name = normalizeText(raw.name || raw.title); - const description = normalizeText(raw.description || raw.summary); - const prompt = normalizeText(raw.prompt); - if (!name && !description && !prompt) return null; - return { - name: name || "未命名角色", - description, - prompt, - look: normalizeText(raw.look) || undefined, - wardrobe: normalizeText(raw.wardrobe) || undefined, - signatureAction: normalizeText(raw.signatureAction) || undefined, - traits: normalizeStringList(raw.traits, 6), - }; - }) - .filter((item): item is NonNullable => Boolean(item)) - .slice(0, 8); -} - -function normalizeScenes(value: unknown) { - if (!Array.isArray(value)) return []; - return value - .map((item) => { - const raw = - item && typeof item === "object" && !Array.isArray(item) - ? (item as Record) - : {}; - const name = normalizeText(raw.name || raw.title); - const description = normalizeText(raw.description || raw.summary); - const prompt = normalizeText(raw.prompt); - if (!name && !description && !prompt) return null; - return { - name: name || "未命名场景", - description, - prompt, - mood: normalizeText(raw.mood) || undefined, - lighting: normalizeText(raw.lighting) || undefined, - props: normalizeStringList(raw.props, 6), - }; - }) - .filter((item): item is NonNullable => Boolean(item)) - .slice(0, 8); -} - -function buildVideoAnalysisPrompt(params: { - nodeTitle?: string; - prompt: string; - frameCount: number; - imageCount: number; - videoCount: number; -}) { - return [ - "你是视频画布工作流里的视频分析节点,需要为后续分镜和生成节点提供结构化输出。", - params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", - `上游参考图数量:${params.imageCount}`, - `上游视频数量:${params.videoCount}`, - `已提供按时间顺序抽取的视频关键帧数量:${params.frameCount}`, - params.prompt ? `上游脚本/说明:\n${params.prompt}` : "上游没有额外脚本,请重点依据参考画面分析。", - "", - "请只输出 JSON,不要输出 markdown,不要输出解释。", - "JSON 结构如下:", - `{ - "summary": "一句话总结当前视频/镜头风格,不超过120字", - "highlights": ["3-5条镜头分析要点"], - "analysisRows": [ - { - "shot": "1", - "startTime": "0", - "endTime": "5", - "duration": "5", - "sceneDescription": "画面描述,说明主体、环境、动作、构图和色调", - "narrative": "叙事内容,说明这一镜头承担的剧情/情绪/信息作用", - "shotSize": "景别,例如中景至近景", - "cameraAngle": "摄影机角度,例如平视", - "cameraMovement": "摄影机运动,例如缓慢推镜头 (Slow Zoom In)", - "focusDepth": "焦距与景深,例如浅景深", - "lighting": "光线,例如冷调低沉照明,屏幕光作为侧逆光", - "backgroundMusic": "这一段适合的背景音乐氛围", - "voiceAndSfx": "人声/环境声/音效建议,没有就写无", - "imagePrompt": "后续生成关键帧时可直接使用的英文图像提示词", - "motionPrompt": "后续生成视频时可直接使用的英文运动提示词", - "keyframeIndex": 1 - } - ], - "shotIdeas": [ - { "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" } - ], - "visualMotifs": ["2-4个可复用的视觉母题"], - "recommendedNextSteps": ["2-4条建议的下游动作"] -}`, - "analysisRows 请按镜头时间顺序输出;短视频可以输出 1 行,复杂视频输出 2-6 行。", - "每行必须覆盖镜号、开始时间、结束时间、时长、画面描述、叙事内容、景别、摄影机角度、摄影机运动、焦距与景深、光线、背景音乐、人声/音效、图像生成提示词、视频运动提示词和关键帧。", - "imagePrompt 和 motionPrompt 优先使用英文;其他字段使用中文;keyframeIndex 使用 1 开始的序号,对应你看到的关键帧顺序。", - "如果没有真实视频帧,只能依据参考图和脚本推断,也要正常输出,但措辞要稳健。", - ] - .filter(Boolean) - .join("\n"); -} - -async function getVideoDurationSeconds(videoUrl: string) { - try { - const { stdout } = await execFileAsync( - "ffprobe", - [ - "-v", - "error", - "-show_entries", - "format=duration", - "-of", - "default=noprint_wrappers=1:nokey=1", - videoUrl, - ], - { - timeout: VIDEO_ANALYSIS_TIMEOUT_MS, - maxBuffer: 1024 * 1024, - }, - ); - const duration = Number.parseFloat(String(stdout || "").trim()); - return Number.isFinite(duration) && duration > 0 ? duration : null; - } catch { - return null; - } -} - -function buildVideoFrameTimestamps(durationSeconds: number | null, frameCount: number) { - const safeCount = Math.max(1, frameCount); - if (!durationSeconds || !Number.isFinite(durationSeconds) || durationSeconds <= 0.5) { - return Array.from({ length: safeCount }).map((_, index) => - Number((0.3 + index * 0.45).toFixed(2)), - ); - } - const start = Math.min(0.35, durationSeconds * 0.12); - const end = Math.max(start + 0.6, durationSeconds * 0.88); - if (safeCount === 1) return [Number(((start + end) / 2).toFixed(2))]; - const step = (end - start) / (safeCount - 1); - return Array.from({ length: safeCount }).map((_, index) => - Number((start + step * index).toFixed(2)), - ); -} - -async function downscaleVideoFrameForVision(inputBuffer: Buffer) { - return sharp(inputBuffer, { failOn: "none" }) - .rotate() - .resize({ - width: VIDEO_ANALYSIS_MAX_SIDE, - height: VIDEO_ANALYSIS_MAX_SIDE, - fit: "inside", - withoutEnlargement: true, - }) - .jpeg({ quality: 82, mozjpeg: true }) - .toBuffer(); -} - -async function extractVideoFrameDataUrls(videoUrl: string, frameCount = VIDEO_ANALYSIS_FRAME_COUNT) { - const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-")); - try { - const durationSeconds = await getVideoDurationSeconds(videoUrl); - const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount); - const frames = await Promise.all( - timestamps.map(async (timestamp, index) => { - const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`); - await execFileAsync( - "ffmpeg", - [ - "-hide_banner", - "-loglevel", - "error", - "-ss", - String(timestamp), - "-i", - videoUrl, - "-frames:v", - "1", - "-q:v", - "4", - "-y", - outputPath, - ], - { - timeout: VIDEO_ANALYSIS_TIMEOUT_MS, - maxBuffer: 2 * 1024 * 1024, - }, - ); - const buffer = await readFile(outputPath); - const processed = await downscaleVideoFrameForVision(buffer); - return `data:image/jpeg;base64,${processed.toString("base64")}`; - }), - ); - return frames.filter(Boolean); - } catch (error) { - console.warn("[Canvas Workflow Analyze] extract video frames failed:", { - message: error instanceof Error ? error.message : String(error), - videoUrl, - }); - return []; - } finally { - await rm(tempDir, { force: true, recursive: true }).catch(() => undefined); - } -} - -function buildCharacterScenePrompt(params: { - nodeTitle?: string; - prompt: string; - imageCount: number; -}) { - return [ - "你是视频画布工作流里的角色场景提取节点,需要从脚本、设定和参考图里提取角色与场景资产。", - params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", - `参考图数量:${params.imageCount}`, - params.prompt ? `输入内容:\n${params.prompt}` : "没有额外脚本,请根据参考图提取角色和场景线索。", - "", - "请只输出 JSON,不要输出 markdown,不要输出解释。", - "JSON 结构如下:", - `{ - "summary": "一句话总结当前故事世界与资产线索", - "characters": [ - { - "name": "角色名", - "description": "角色外观与身份简介", - "look": "外貌特征", - "wardrobe": "服装/道具", - "traits": ["性格关键词"], - "signatureAction": "代表动作", - "prompt": "可直接用于生成角色图/角色视频的提示词" - } - ], - "scenes": [ - { - "name": "场景名", - "description": "空间、布景和氛围简介", - "mood": "情绪与气氛", - "lighting": "布光特征", - "props": ["核心道具"], - "prompt": "可直接用于生成场景图/场景视频的提示词" - } - ], - "relationships": ["角色与场景或角色之间的关系"], - "recommendedNextSteps": ["建议连接的后续节点动作"] -}`, - "角色和场景都要尽量具体,避免空泛形容词。", - ] - .filter(Boolean) - .join("\n"); -} - -function buildAgentPlanPrompt(params: { - nodeTitle?: string; - prompt: string; - imageCount: number; - videoCount: number; -}) { - return [ - "你是视频画布里的自定义代理节点,需要把当前创作目标拆成可执行的工作流计划。", - params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", - `参考图数量:${params.imageCount}`, - `视频数量:${params.videoCount}`, - params.prompt ? `上游需求:\n${params.prompt}` : "没有额外脚本,请基于当前上下文给出稳妥计划。", - "", - "请只输出 JSON,不要输出 markdown,不要输出解释。", - "JSON 结构如下:", - `{ - "summary": "一句话说明这个代理链的目标", - "agentPlan": ["3-6个执行步骤"], - "deliverables": ["2-4个预期交付物"], - "recommendedNodes": ["建议继续连接的节点类型或能力名"] -}`, - "步骤要偏工作流视角,而不是泛泛而谈。", - ] - .filter(Boolean) - .join("\n"); -} - -function buildStoryScriptPrompt(params: { - nodeTitle?: string; - prompt: string; - imageCount: number; - videoCount: number; -}) { - return [ - "你是视频画布里的故事脚本生成节点,需要把用户输入、参考图和参考视频整理成可直接用于后续分镜/生图/生视频的结构化脚本表。", - params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", - `参考图数量:${params.imageCount}`, - `视频数量:${params.videoCount}`, - params.prompt ? `输入内容:\n${params.prompt}` : "没有额外脚本,请生成一个节奏清晰的短视频脚本。", - "", - "请只输出 JSON,不要输出 markdown,不要输出解释。", - "JSON 结构如下:", - `{ - "summary": "一句话概括故事主题、情绪和视频方向,不超过120字", - "analysisRows": [ - { - "shot": "01", - "duration": "5s", - "scene": "场景名或地点,如 盛唐·金銮殿·日", - "sceneDescription": "画面内容、人物动作和剧情推进,不要重复 scene 字段", - "shotSize": "景别,如中景、特写、全景、大远景", - "lighting": "光影氛围,如暖金主光带阴影,气氛紧张压抑", - "dialogue": "对白或旁白,没有就写 +", - "backgroundMusic": "背景音乐或情绪氛围,没有就写无", - "voiceAndSfx": "环境声、动作声或音效,没有就写无", - "cameraMovement": "运镜,如缓慢推镜头、固定、横移、上摇、环绕轨道跟拍", - "imagePrompt": "后续生成该镜头关键帧时可直接使用的中文图像提示词", - "motionPrompt": "后续生成视频时可直接使用的中文视频运动提示词" - } - ], - "assetCandidates": { - "style": "全局视觉风格,如 盛唐·国风二次元", - "characters": ["需要生成的角色资产名"], - "scenes": ["需要生成的场景资产名"], - "props": ["需要生成的道具资产名"] - }, - "shotIdeas": [ - { "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" } - ], - "recommendedNextSteps": ["2-4条建议的下游动作"] -}`, - "analysisRows 必须直接输出 18 行,按镜头时间顺序编号 01-18;每行默认 5s 或 6s,剧情高潮可用 6s。", - "每行要覆盖竞品脚本表同等信息:镜号、时长、画面描述、景别、光影氛围、对白·旁白、音效、运镜、最终图像提示词和视频运动提示词。", - "画面描述中用 @角色名 或 @场景/道具名 标出可复用资产,例如 @沈昭昭、@金銮殿、@侍卫长刀。", - "imagePrompt 要像可直接生成分镜图的中文提示词,包含主体、环境、构图、光影、风格与一致性要求;motionPrompt 要按起始状态、动作过程、结束状态、音效组织。", - "assetCandidates 根据 18 镜头抽取 3-6 个角色、2-5 个场景和 1-4 个道具,方便后续准备资产。", - "每行要像专业视频画布脚本表:场景具体、画面可生成、声音可执行、镜头节奏明确,不要只写剧情摘要。", - ] - .filter(Boolean) - .join("\n"); -} - export async function POST(request: Request) { - try { - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); - } - - const body = await request.json().catch(() => ({})); - const parsed = requestSchema.safeParse(body); - if (!parsed.success) { - return NextResponse.json( - { success: false, error: "请求参数不合法" }, - { status: 400 } - ); - } - - const prompt = normalizeText(parsed.data.prompt); - const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4); - const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4); - const extractedVideoFrames = - parsed.data.mode === "video-analysis" && videoUrls.length > 0 - ? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT) - : []; - const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model); - if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) { - return NextResponse.json( - { success: false, error: "请至少提供脚本、图片或视频上下文" }, - { status: 400 } - ); - } - - - const [allModels, agentModel] = await Promise.all([ - getEnabledAiModelsWithProvider(false), - getAgentAiModelWithProvider(), - ]); - const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0; - const compatibleModels = allModels.filter((item) => { - const descriptor = {...item.model, providerType: item.provider.type}; - return !item.model.isSubModel && item.model.isEnabled && - (hasVisionInputs ? isCanvasVisionModel(descriptor) : isCanvasLanguageModel(descriptor)); - }); - const requestedRef = preferredModelRef === "auto" ? "" : preferredModelRef; - const preferredModel = requestedRef ? compatibleModels.find( - item => item.model.id === requestedRef || item.model.modelId === requestedRef - ) : undefined; - if (requestedRef && !preferredModel) { - return NextResponse.json({ success: false, error: "所选模型不可用或不支持当前分析输入,请重新选择分析模型" }, { status: 400 }); - } - const selectedModel = preferredModel || compatibleModels.find(item => item.model.id === agentModel?.model.id) || compatibleModels[0]; - if (!selectedModel) { - return NextResponse.json({ success: false, error: hasVisionInputs ? "当前没有可用的视觉理解模型" : "当前没有可用的文字分析模型" }, { status: 400 }); - } - - const useVision = hasVisionInputs && supportsVision(selectedModel.model); - const codingModelId = getAiModelAgentCodingModelId(selectedModel.model); - const model = await getLanguageModelById(selectedModel.model.id, codingModelId); - if (!model) { - return NextResponse.json( - { success: false, error: "分析模型初始化失败" }, - { status: 500 } - ); - } - - const promptText = - parsed.data.mode === "video-analysis" - ? buildVideoAnalysisPrompt({ - frameCount: extractedVideoFrames.length, - nodeTitle: normalizeText(parsed.data.nodeTitle), - prompt, - imageCount: imageUrls.length, - videoCount: videoUrls.length, - }) - : parsed.data.mode === "character-scene-extraction" - ? buildCharacterScenePrompt({ - nodeTitle: normalizeText(parsed.data.nodeTitle), - prompt, - imageCount: imageUrls.length, - }) - : parsed.data.mode === "story-script" - ? buildStoryScriptPrompt({ - nodeTitle: normalizeText(parsed.data.nodeTitle), - prompt, - imageCount: imageUrls.length, - videoCount: videoUrls.length, - }) - : buildAgentPlanPrompt({ - nodeTitle: normalizeText(parsed.data.nodeTitle), - prompt, - imageCount: imageUrls.length, - videoCount: videoUrls.length, - }); - - const inlinedImageReferences = useVision - ? ( - await inlineVisionImages(imageUrls, { - maxImages: 4, - maxSide: 960, - timeoutMs: 12000, - retries: 1, - loggerPrefix: "[Canvas Workflow Analyze]", - }) - ).inlineDataUrls - : []; - const referenceImages = useVision - ? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6) - : []; - - const { text } = await generateText({ - model, - messages: [ - { - role: "user", - content: [ - { type: "text", text: promptText }, - ...referenceImages.map((image) => ({ type: "image" as const, image })), - ], - }, - ], - temperature: 0.2, - maxTokens: parsed.data.mode === "story-script" ? 4200 : parsed.data.mode === "video-analysis" ? 2200 : 1600, - } as any); - - const parsedJson = extractJsonBlock>(text) || {}; - const summary = - normalizeText(parsedJson.summary) || - normalizeText(text).slice(0, 160); - - if (parsed.data.mode === "video-analysis") { - const analysisRows = normalizeVideoAnalysisRows( - parsedJson.analysisRows, - extractedVideoFrames, - ); - const shotIdeas = - normalizeShotIdeas(parsedJson.shotIdeas).length > 0 - ? normalizeShotIdeas(parsedJson.shotIdeas) - : analysisRows.map((row, index) => ({ - title: `镜头 ${index + 1}`, - description: row.sceneDescription, - })); - return NextResponse.json({ - success: true, - data: { - summary, - analysisRows, - framePreviews: extractedVideoFrames, - highlights: normalizeStringList(parsedJson.highlights, 6), - shotIdeas, - visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6), - recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), - model: selectedModel.model.displayName, - raw: text, - }, - }); - } - - if (parsed.data.mode === "character-scene-extraction") { - return NextResponse.json({ - success: true, - data: { - summary, - characters: normalizeCharacters(parsedJson.characters), - scenes: normalizeScenes(parsedJson.scenes), - relationships: normalizeStringList(parsedJson.relationships, 8), - recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), - model: selectedModel.model.displayName, - raw: text, - }, - }); - } - - if (parsed.data.mode === "story-script") { - const analysisRows = expandStoryScriptRowsToTarget(normalizeStoryScriptRows( - parsedJson.analysisRows || parsedJson.storyRows || parsedJson.rows || parsedJson.shots, - )); - return NextResponse.json({ - success: true, - data: { - summary, - analysisRows, - shotIdeas: - normalizeShotIdeas(parsedJson.shotIdeas).length > 0 - ? normalizeShotIdeas(parsedJson.shotIdeas) - : analysisRows.map((row, index) => ({ - title: `镜头 ${index + 1}`, - description: row.sceneDescription, - })), - recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), - model: selectedModel.model.displayName, - raw: text, - }, - }); - } - - return NextResponse.json({ - success: true, - data: { - summary, - agentPlan: normalizeStringList(parsedJson.agentPlan, 8), - deliverables: normalizeStringList(parsedJson.deliverables, 6), - recommendedNodes: normalizeStringList(parsedJson.recommendedNodes, 6), - model: selectedModel.model.displayName, - raw: text, - }, - }); - } catch (error) { - console.error("[Canvas Workflow Analyze] Error:", error); - return NextResponse.json( - { - success: false, - error: error instanceof Error ? error.message : "工作流分析失败", - }, - { status: 500 } - ); - } + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const body = await request.json().catch(() => ({})); + return executeWorkflowAnalysisResponse(body, request.signal); } diff --git a/app/api/enhance/route.ts b/app/api/enhance/route.ts index ea888b293..bba47dd54 100644 --- a/app/api/enhance/route.ts +++ b/app/api/enhance/route.ts @@ -1,1411 +1,15 @@ -import { NextRequest, NextResponse } from "next/server"; -import { - createMediaAssetRelation, - getAiProvidersByType, - getMediaAssetsByUserAndUrls, - getSystemSetting, - upsertMediaAsset, -} from "@/lib/db/queries"; +import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; -import { uploadFile } from "@/lib/storage/service"; -import { db } from "@/lib/db"; -import { generatedImage } from "@/lib/db/schema"; -import { executeConfiguredVip98kTask } from "@/lib/ai/vip98k-runtime"; -import { createFalClient } from "@fal-ai/client"; - -// 变清晰API配置类型 -type EnhanceApiConfig = { - provider?: "vip98k" | "aoscdn"; - apiUrl?: string; - apiKey?: string; - type?: string; // "clean" | "face" - scaleFactor?: number | string; // 1 | 2 | 4 | "auto" - // 备用API配置 (Real-ESRGAN) - useBackupApi?: boolean; - backupApiUrl?: string; - backupApiKey?: string; - backupScale?: string; // "2" | "4" | "8" - backupModel?: string; -}; - -const AOS_MAX_RESOLUTION = 4096; -const ENHANCE_CREATE_TIMEOUT_MS = 120_000; -const ENHANCE_CREATE_RETRY_COUNT = 1; -const ENHANCE_SYNC_WAIT_TIMEOUT_MS = 20_000; -const ENHANCE_TASK_POLL_INTERVAL_MS = 1200; - -function getAosFriendlyErrorMessage(detail: string): string | null { - const resolutionMatch = detail.match(/resolution excess,\s*width=(\d+),\s*height=(\d+)/i); - if (resolutionMatch) { - const [, width, height] = resolutionMatch; - return `图片分辨率超限(${width}x${height}),AOS要求宽高都不超过 ${AOS_MAX_RESOLUTION}px。请缩小后重试。`; - } - - if (/invalid file/i.test(detail)) { - return "图片文件无法识别,请尝试转换为 JPG/PNG 后重试。"; - } - - return null; -} - -function isRetryableNetworkError(error: unknown): boolean { - if (!(error instanceof Error)) return false; - if (error.name === "AbortError") return true; - - const message = error.message || ""; - const cause = (error as Error & { cause?: { message?: string; code?: string } }).cause; - const causeMessage = cause?.message || ""; - const causeCode = cause?.code || ""; - const combined = `${message} ${causeMessage} ${causeCode}`; - - return /EHOSTUNREACH|ENOTFOUND|ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|No route to host/i.test(combined); -} - -async function ensureAosResolutionLimit( - buffer: Buffer, - logPrefix: string -): Promise { - try { - const sharp = await import("sharp"); - const image = sharp.default(buffer); - const metadata = await image.metadata(); - const width = metadata.width; - const height = metadata.height; - - if (!width || !height) { - return buffer; - } - - if (width <= AOS_MAX_RESOLUTION && height <= AOS_MAX_RESOLUTION) { - return buffer; - } - - const resizedBuffer = await image - .resize(AOS_MAX_RESOLUTION, AOS_MAX_RESOLUTION, { - fit: "inside", - withoutEnlargement: true, - }) - .toBuffer(); - const resizedMeta = await sharp.default(resizedBuffer).metadata(); - - console.warn( - `[${logPrefix}] 输入图片分辨率超限,已自动缩放: ${width}x${height} -> ${resizedMeta.width}x${resizedMeta.height}` - ); - return resizedBuffer; - } catch (error) { - console.error(`[${logPrefix}] 自动缩放失败,继续使用原图:`, error); - return buffer; - } -} - -/** - * 压缩图片生成缩略图(使用 sharp) - */ -async function createThumbnail(buffer: Buffer, mimeType: string): Promise { - try { - const sharp = await import("sharp"); - - // 创建缩略图(宽度 500px,高度按比例,质量 80%) - const thumbnailBuffer = await sharp.default(buffer) - .resize(500, null, { - fit: "inside", - withoutEnlargement: true, - }) - .jpeg({ quality: 80 }) // JPEG格式 - .toBuffer(); - - console.log(`[Enhance] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`); - return thumbnailBuffer; - } catch (error) { - console.error("[Enhance] Failed to create thumbnail:", error); - // 如果失败,返回原图 - return buffer; - } -} - -/** - * 将 base64 图片上传到 S3,返回缩略图和原图的 URL - */ -async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jpeg"): Promise<{ thumbnail: string; original: string }> { - try { - // 解析 base64 数据 - const buffer = Buffer.from(base64Data, "base64"); - - // 生成文件名 - const ext = mimeType.split("/")[1] || "jpg"; - const timestamp = Date.now(); - const randomStr = Math.random().toString(36).slice(2, 8); - const thumbnailFilename = `enhance-thumb-${timestamp}-${randomStr}.${ext}`; - const originalFilename = `enhance-orig-${timestamp}-${randomStr}.${ext}`; - - console.log(`[Enhance] Original size: ${buffer.length} bytes`); - - // 生成缩略图 - const thumbnailBuffer = await createThumbnail(buffer, mimeType); - - // 转换为 ArrayBuffer - const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( - thumbnailBuffer.byteOffset, - thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength - ) as ArrayBuffer; - const originalArrayBuffer = buffer.buffer.slice( - buffer.byteOffset, - buffer.byteOffset + buffer.byteLength - ) as ArrayBuffer; - - // 并行上传缩略图和原图到 S3 - console.log(`[Enhance] Uploading thumbnail and original to S3...`); - const [thumbnailResult, originalResult] = await Promise.all([ - uploadFile(thumbnailFilename, thumbnailArrayBuffer), - uploadFile(originalFilename, originalArrayBuffer), - ]); - - console.log(`[Enhance] Uploaded to S3:`); - console.log(` - Thumbnail: ${thumbnailResult.url}`); - console.log(` - Original: ${originalResult.url}`); - - return { - thumbnail: thumbnailResult.url, - original: originalResult.url, - }; - } catch (error) { - console.error("[Enhance] Failed to upload to S3:", error); - throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`); - } -} - -// 设置API路由最大执行时间为3分钟(180秒) +import { executeEnhanceGET, executeEnhancePOST } from "@/lib/canvas/operations/enhance"; +export const runtime = "nodejs"; export const maxDuration = 180; - -/** - * 使用Real-ESRGAN备用API处理图片(通过URL方式) - * @param imageUrl 图片URL(需要公开可访问) - * @param config 备用API配置 - * @returns 处理后的图片Base64和S3 URL - */ -async function processWithRealESRGAN( - imageUrl: string, - config: { - backupApiUrl: string; - backupApiKey: string; - backupScale: string; - backupModel: string; - } -): Promise<{ imageBase64: string; imageUrl: string; processingTime: number }> { - const startTime = Date.now(); - const apiUrl = config.backupApiUrl.endsWith("/") - ? config.backupApiUrl.slice(0, -1) - : config.backupApiUrl; - - const upscaleUrl = `${apiUrl}/upscale/url`; - console.log(`[Enhance-RealESRGAN] 调用URL方式API: ${upscaleUrl}`); - console.log(`[Enhance-RealESRGAN] 图片URL: ${imageUrl}`); - console.log(`[Enhance-RealESRGAN] 放大倍数: ${config.backupScale}x, 模型: ${config.backupModel}`); - - const response = await fetch(upscaleUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-API-Key": config.backupApiKey, - }, - body: JSON.stringify({ - image_url: imageUrl, - scale: Number.parseInt(config.backupScale) || 4, - model_name: config.backupModel || "RealESRGAN_x4plus", - }), - }); - - if (!response.ok) { - const errorText = await response.text(); - console.error(`[Enhance-RealESRGAN] API请求失败: ${response.status} ${response.statusText}`); - console.error(`[Enhance-RealESRGAN] 错误详情: ${errorText}`); - throw new Error(`Real-ESRGAN API请求失败: ${response.status} - ${errorText || response.statusText}`); - } - - const result = await response.json(); - console.log(`[Enhance-RealESRGAN] API响应:`, { - success: result.success, - hasImageUrl: !!result.image_url, - processingTime: result.processing_time, - totalTime: result.total_time, - }); - - if (!result.success || !result.image_url) { - throw new Error(`Real-ESRGAN处理失败: ${JSON.stringify(result)}`); - } - - // Real-ESRGAN API直接返回S3 URL,我们需要下载图片并转换为Base64 - console.log(`[Enhance-RealESRGAN] 下载处理后的图片: ${result.image_url}`); - const imageResponse = await fetch(result.image_url); - if (!imageResponse.ok) { - throw new Error(`无法下载Real-ESRGAN处理后的图片: ${imageResponse.status}`); - } - - const imageBuffer = await imageResponse.arrayBuffer(); - const imageBase64 = Buffer.from(imageBuffer).toString("base64"); - - const processingTime = (Date.now() - startTime) / 1000; - console.log(`[Enhance-RealESRGAN] ✅ 处理完成,总耗时: ${processingTime.toFixed(2)}秒`); - console.log(`[Enhance-RealESRGAN] ✅ 处理后图片大小: ${imageBuffer.byteLength} bytes`); - - return { - imageBase64, - imageUrl: result.image_url, - processingTime, - }; +export async function GET(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + return executeEnhanceGET(request, session.user.id); } - -/** - * 使用Real-ESRGAN备用API处理图片(通过Base64方式,用于没有公开URL的图片) - * @param imageBase64 图片Base64数据 - * @param config 备用API配置 - * @returns 处理后的图片Base64 - */ -async function processWithRealESRGANBase64( - imageBase64: string, - config: { - backupApiUrl: string; - backupApiKey: string; - backupScale: string; - backupModel: string; - } -): Promise<{ imageBase64: string; processingTime: number }> { - const startTime = Date.now(); - const apiUrl = config.backupApiUrl.endsWith("/") - ? config.backupApiUrl.slice(0, -1) - : config.backupApiUrl; - - const upscaleUrl = `${apiUrl}/upscale/base64`; - console.log(`[Enhance-RealESRGAN] 调用Base64方式API: ${upscaleUrl}`); - console.log(`[Enhance-RealESRGAN] 放大倍数: ${config.backupScale}x, 模型: ${config.backupModel}`); - console.log(`[Enhance-RealESRGAN] 输入图片Base64长度: ${imageBase64.length}`); - - const response = await fetch(upscaleUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-API-Key": config.backupApiKey, - }, - body: JSON.stringify({ - image_base64: imageBase64, - scale: Number.parseInt(config.backupScale) || 4, - model_name: config.backupModel || "RealESRGAN_x4plus", - }), - }); - - if (!response.ok) { - const errorText = await response.text(); - console.error(`[Enhance-RealESRGAN] API请求失败: ${response.status} ${response.statusText}`); - console.error(`[Enhance-RealESRGAN] 错误详情: ${errorText}`); - throw new Error(`Real-ESRGAN API请求失败: ${response.status} - ${errorText || response.statusText}`); - } - - const result = await response.json(); - console.log(`[Enhance-RealESRGAN] API响应:`, { - success: result.success, - hasImageBase64: !!result.image_base64, - processingTime: result.processing_time, - }); - - if (!result.success || !result.image_base64) { - throw new Error(`Real-ESRGAN处理失败: ${JSON.stringify(result)}`); - } - - const processingTime = (Date.now() - startTime) / 1000; - console.log(`[Enhance-RealESRGAN] ✅ 处理完成,总耗时: ${processingTime.toFixed(2)}秒`); - console.log(`[Enhance-RealESRGAN] ✅ 处理后图片Base64长度: ${result.image_base64.length}`); - - return { - imageBase64: result.image_base64, - processingTime, - }; -} - -async function processWithFalEsrgan( - imageBase64: string, - requestedScaleFactor: number, -): Promise<{ imageBase64: string; processingTime: number }> { - const providers = await getAiProvidersByType({ - type: "fal-image", - enabledOnly: true, - }); - const provider = providers.find((item) => item.apiKey?.trim()); - if (!provider?.apiKey) { - throw new Error("未配置可用的 FAL 专用高清服务"); - } - - const scale = Math.max(2, Math.min(8, requestedScaleFactor || 4)); - const model = scale <= 2 ? "RealESRGAN_x2plus" : "RealESRGAN_x4plus"; - const credentials = provider.apiKey - .trim() - .replace(/^(?:Key|Bearer)\s+/i, ""); - const fal = createFalClient({ credentials }); - const startedAt = Date.now(); - const result: any = await fal.subscribe("fal-ai/esrgan", { - input: { - image_url: `data:image/jpeg;base64,${imageBase64}`, - scale, - model, - output_format: "png", - }, - logs: true, - timeout: 180_000, - }); - const outputUrl = result?.data?.image?.url || result?.image?.url; - if (!outputUrl) { - throw new Error("FAL ESRGAN 专用高清服务未返回图片"); - } - - const outputResponse = await fetch(outputUrl, { - cache: "no-store", - signal: AbortSignal.timeout(120_000), - }); - if (!outputResponse.ok) { - throw new Error( - `FAL ESRGAN 高清结果下载失败(HTTP ${outputResponse.status})`, - ); - } - - return { - imageBase64: Buffer.from(await outputResponse.arrayBuffer()).toString( - "base64", - ), - processingTime: (Date.now() - startedAt) / 1000, - }; -} - -async function processWithVip98kEnhance(params: { - userId: string; - imageBase64: string; - prompt: string; - chatId: string | null; - messageId: string | null; - sourceImageUrl: string | null; - scaleFactor: number; -}) { - const startedAt = Date.now(); - const result = await executeConfiguredVip98kTask({ - modelId: "enhance", - kind: "image-tool", - prompt: `${params.prompt || "Enhance image clarity"}. Upscale ${params.scaleFactor}x while preserving composition, color and texture.`, - imageUrls: [`data:image/jpeg;base64,${params.imageBase64}`], - imageParams: { scaleFactor: params.scaleFactor }, - }); - const outputUrl = result.outputUrls[0]; - if (!outputUrl) throw new Error("VIP98K 变清晰未返回图片"); - - const response = await fetch(outputUrl, { - cache: "no-store", - signal: AbortSignal.timeout(120_000), - }); - if (!response.ok) { - throw new Error(`VIP98K 变清晰结果下载失败(HTTP ${response.status})`); - } - const outputBuffer = Buffer.from(await response.arrayBuffer()); - const outputBase64 = outputBuffer.toString("base64"); - const uploaded = await uploadBase64ToS3(outputBase64, "image/png"); - const [savedImage] = await db - .insert(generatedImage) - .values({ - userId: params.userId, - chatId: params.chatId, - messageId: params.messageId, - thumbnailUrl: uploaded.thumbnail, - originalUrl: uploaded.original, - imageSize: `${params.scaleFactor}x放大`, - prompt: params.prompt, - modelId: "enhance", - }) - .returning(); - - try { - const sourceAssets = params.sourceImageUrl - ? await getMediaAssetsByUserAndUrls({ - userId: params.userId, - urls: [params.sourceImageUrl], - }) - : []; - const media = await upsertMediaAsset({ - userId: params.userId, - chatId: params.chatId, - mediaType: "image", - sourceType: "edited", - status: "ready", - originalUrl: uploaded.original, - previewUrl: uploaded.original, - thumbnailUrl: uploaded.thumbnail, - displayName: params.prompt.slice(0, 80), - prompt: params.prompt, - modelId: "enhance", - sourceMessageId: params.messageId, - sourceTable: "GeneratedImage", - sourceRowId: savedImage?.id || null, - primaryParentAssetId: sourceAssets[0]?.id ?? null, - metadata: { provider: "vip98k", scaleFactor: params.scaleFactor }, - }); - if (media) { - for (const sourceAsset of sourceAssets) { - await createMediaAssetRelation({ - userId: params.userId, - chatId: params.chatId, - fromAssetId: sourceAsset.id, - toAssetId: media.id, - relationType: "edited_from", - sourceMessageId: params.messageId, - metadata: { prompt: params.prompt, provider: "vip98k" }, - }); - } - } - } catch (error) { - console.warn("[Enhance-VIP98K] MediaAsset registration failed", error); - } - - return { - success: true, - image_base64: outputBase64, - thumbnail_url: uploaded.thumbnail, - original_url: uploaded.original, - processing_time: (Date.now() - startedAt) / 1000, - api_type: "vip98k", - }; -} - -/** - * 轮询获取任务结果(根据官方示例) - */ -async function pollTaskResult( - apiUrl: string, - apiKey: string, - taskId: string, - delay: number = 1000, - timeout: number = 120000 // 增加到120秒(2分钟) -): Promise { - const pollUrl = `${apiUrl}/api/tasks/visual/scale/${taskId}`; - - const getTaskResult = async (): Promise => { - const response = await fetch(pollUrl, { - method: "GET", - headers: { - "X-API-KEY": apiKey, - }, - }); - - if (!response.ok) { - throw new Error(`轮询请求失败: ${response.status} ${response.statusText}`); - } - - const result = await response.json(); - - if (!result.data) { - throw result; - } - - const { progress, state } = result.data; - console.log(`[Enhance] 轮询进度: progress=${progress}%, state=${state}, task_id=${taskId}`); - - // state < 0 表示失败 - if (state < 0) { - const errMessage = String( - result.data?.err_message || result.data?.err_info || result.data?.state_detail || `state=${state}` - ); - throw new Error(`AOS任务失败: ${errMessage}`); - } - - // progress >= 100 表示完成 - if (progress >= 100) { - console.log(`[Enhance] 任务完成,task_id=${taskId}`); - return result; - } - - // 否则继续轮询 - throw null; - }; - - try { - const result = await getTaskResult(); - return result; - } catch (error) { - if (error && typeof error === "object" && "data" in error) { - throw new Error(JSON.stringify(error, null, 2)); - } - if (timeout <= 0) { - throw new Error("轮询超时"); - } - await new Promise((resolve) => { - setTimeout(() => { - resolve(undefined); - }, delay); - }); - return pollTaskResult(apiUrl, apiKey, taskId, delay, timeout - delay); - } -} - -function isEnhanceTaskStillProcessing(error: unknown): boolean { - if (!(error instanceof Error)) return false; - return /轮询超时/i.test(error.message || ""); -} - -function buildEnhanceTaskPendingResponse(taskId: string) { - return NextResponse.json( - { - success: true, - processing: true, - status: "running", - taskId, - pollAfterMs: ENHANCE_TASK_POLL_INTERVAL_MS, - }, - { status: 202 } - ); -} - -async function finalizeEnhanceTaskResult(params: { - userId: string; - taskResult: any; - prompt?: string; - chatId?: string | null; - messageId?: string | null; - startedAt?: number; - sourceImageUrls?: string[]; -}) { - if (params.taskResult?.status !== 200 || !params.taskResult?.data) { - throw new Error(params.taskResult?.message || "未知错误"); - } - - let enhancedImageBase64: string | null = null; - if (typeof params.taskResult.data.image_base64 === "string" && params.taskResult.data.image_base64) { - enhancedImageBase64 = params.taskResult.data.image_base64; - } else if (typeof params.taskResult.data.image === "string" && params.taskResult.data.image) { - const imageData = params.taskResult.data.image; - if (/^https?:\/\//i.test(imageData)) { - const imageResponse = await fetch(imageData); - if (!imageResponse.ok) { - throw new Error(`无法下载处理后的图片: ${imageResponse.status}`); - } - const downloadedBuffer = await imageResponse.arrayBuffer(); - enhancedImageBase64 = Buffer.from(downloadedBuffer).toString("base64"); - } else { - enhancedImageBase64 = imageData; - } - } - - if (!enhancedImageBase64) { - throw new Error("轮询结果未包含图片数据"); - } - - const uploadResult = await uploadBase64ToS3(enhancedImageBase64, "image/jpeg"); - const finalPrompt = params.prompt?.trim() || "图片变清晰处理"; - - let saved = false; - let saveError: unknown = null; - let savedImage: - | { - id: string; - } - | null = null; - for (let attempt = 1; attempt <= 3; attempt += 1) { - try { - ([savedImage] = await db.insert(generatedImage).values({ - userId: params.userId, - chatId: params.chatId || null, - messageId: params.messageId || null, - thumbnailUrl: uploadResult.thumbnail, - originalUrl: uploadResult.original, - imageSize: "2K", - prompt: finalPrompt, - modelId: "enhance-1.0", - }).returning()); - saved = true; - break; - } catch (dbError) { - saveError = dbError; - if (attempt < 3) { - await new Promise((resolve) => setTimeout(resolve, 120 * attempt)); - } - } - } - if (!saved) { - console.error("[Enhance] 保存到相册失败:", saveError); - throw new Error("图片变清晰成功但保存到相册失败,请稍后重试"); - } - - if (savedImage) { - try { - const sourceAssets = - Array.isArray(params.sourceImageUrls) && params.sourceImageUrls.length > 0 - ? await getMediaAssetsByUserAndUrls({ - userId: params.userId, - urls: params.sourceImageUrls, - }) - : []; - const media = await upsertMediaAsset({ - userId: params.userId, - chatId: params.chatId || null, - mediaType: "image", - sourceType: "edited", - status: "ready", - originalUrl: uploadResult.original, - previewUrl: uploadResult.original, - thumbnailUrl: uploadResult.thumbnail, - displayName: finalPrompt.slice(0, 80), - prompt: finalPrompt, - modelId: "enhance-1.0", - sourceMessageId: params.messageId || null, - sourceTable: "GeneratedImage", - sourceRowId: savedImage.id, - primaryParentAssetId: sourceAssets[0]?.id ?? null, - metadata: { - imageSize: "2K", - }, - }); - if (media) { - for (const sourceAsset of sourceAssets) { - await createMediaAssetRelation({ - userId: params.userId, - chatId: params.chatId || null, - fromAssetId: sourceAsset.id, - toAssetId: media.id, - relationType: "edited_from", - sourceMessageId: params.messageId || null, - metadata: { - prompt: finalPrompt, - }, - }); - } - } - } catch (assetError) { - console.error("[Enhance] ⚠️ MediaAsset 注册失败:", assetError); - } - } - - const processingTime = - typeof params.startedAt === "number" ? (Date.now() - params.startedAt) / 1000 : undefined; - - return { - success: true, - image_base64: enhancedImageBase64, - thumbnail_url: uploadResult.thumbnail, - original_url: uploadResult.original, - processing_time: processingTime, - }; -} - -async function executeFalEsrganFallback(params: { - userId: string; - imageBase64: string; - prompt: string; - chatId: string | null; - messageId: string | null; - sourceImageUrl: string | null; - requestedScaleFactor?: number; - reason: string; -}) { - const scaleFactor = params.requestedScaleFactor || 4; - console.warn("[Enhance-FAL-ESRGAN] 启用专用高清备用链路", { - reason: params.reason.slice(0, 500), - scaleFactor, - }); - const result = await processWithFalEsrgan(params.imageBase64, scaleFactor); - const payload = await finalizeEnhanceTaskResult({ - userId: params.userId, - taskResult: { - status: 200, - data: { image_base64: result.imageBase64 }, - }, - prompt: params.prompt || "图片变清晰处理", - chatId: params.chatId, - messageId: params.messageId, - sourceImageUrls: params.sourceImageUrl - ? [params.sourceImageUrl] - : undefined, - }); - return { - ...payload, - processing_time: result.processingTime, - api_type: "fal-esrgan", - }; -} - -export async function GET(request: NextRequest) { - try { - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json({ error: "未授权,请先登录" }, { status: 401 }); - } - - const { searchParams } = new URL(request.url); - const taskId = String(searchParams.get("taskId") || "").trim(); - if (!taskId) { - return NextResponse.json({ error: "缺少 taskId 参数" }, { status: 400 }); - } - - const prompt = searchParams.get("prompt") || "图片变清晰处理"; - const chatId = searchParams.get("chatId"); - const messageId = searchParams.get("messageId"); - const sourceImageUrl = searchParams.get("sourceImageUrl"); - - const enhanceConfig = await getSystemSetting({ key: "enhance_api_config" }); - if (!enhanceConfig?.value) { - return NextResponse.json( - { error: "变清晰API配置未设置,请在系统设置中配置" }, - { status: 400 } - ); - } - - const config = enhanceConfig.value as EnhanceApiConfig; - if (config.provider === "vip98k") { - return NextResponse.json( - { error: "VIP98K 变清晰任务在 POST 请求内完成,无需 taskId 轮询" }, - { status: 400 } - ); - } - if (config.useBackupApi) { - return NextResponse.json( - { error: "当前启用了Real-ESRGAN备用模式,不支持 taskId 轮询" }, - { status: 400 } - ); - } - if (!config.apiUrl || !config.apiKey) { - return NextResponse.json( - { error: "变清晰API地址或密钥未配置" }, - { status: 400 } - ); - } - - const apiUrl = config.apiUrl.endsWith("/") ? config.apiUrl.slice(0, -1) : config.apiUrl; - - let taskResult: any; - try { - taskResult = await pollTaskResult( - apiUrl, - config.apiKey, - taskId, - ENHANCE_TASK_POLL_INTERVAL_MS, - ENHANCE_SYNC_WAIT_TIMEOUT_MS - ); - } catch (pollError) { - if (isEnhanceTaskStillProcessing(pollError)) { - return buildEnhanceTaskPendingResponse(taskId); - } - const detail = pollError instanceof Error ? pollError.message : "轮询失败"; - return NextResponse.json( - { error: "变清晰处理失败", detail }, - { status: 500 } - ); - } - - const responsePayload = await finalizeEnhanceTaskResult({ - userId: session.user.id, - taskResult, - prompt, - chatId, - messageId, - sourceImageUrls: sourceImageUrl ? [sourceImageUrl] : undefined, - }); - - return NextResponse.json(responsePayload); - } catch (error) { - console.error("[Enhance API][TaskStatus] 处理错误:", error); - return NextResponse.json( - { error: "服务器内部错误", detail: error instanceof Error ? error.message : "未知错误" }, - { status: 500 } - ); - } -} - -/** - * 图片变清晰API路由 - * 从数据库读取变清晰API配置,调用外部API进行图片增强处理 - * 将处理后的图片上传到S3并保存到数据库 - */ -export async function POST(request: NextRequest) { - console.log("[Enhance API] ===== 收到请求 ====="); - try { - // 0. 获取用户会话 - console.log("[Enhance API] 开始获取用户会话..."); - const session = await auth(); - console.log("[Enhance API] 用户会话获取完成:", session?.user?.id ? "已登录" : "未登录"); - if (!session?.user?.id) { - return NextResponse.json( - { error: "未授权,请先登录" }, - { status: 401 } - ); - } - - const userId = session.user.id; - console.log("[Enhance API] 用户ID:", userId); - // 1. 从数据库获取变清晰API配置 - console.log("[Enhance API] 开始获取配置..."); - const enhanceConfig = await getSystemSetting({ key: "enhance_api_config" }); - console.log("[Enhance API] 配置获取完成:", enhanceConfig ? "已找到" : "未找到"); - - if (!enhanceConfig || !enhanceConfig.value) { - return NextResponse.json( - { error: "变清晰API配置未设置,请在系统设置中配置" }, - { status: 400 } - ); - } - - const config = enhanceConfig.value as EnhanceApiConfig; - const useVip98k = config.provider === "vip98k"; - - // 检查是否使用备用API - const useBackupApi = !useVip98k && config.useBackupApi === true; - console.log(`[Enhance API] 使用${useBackupApi ? "备用API (Real-ESRGAN)" : "主API"}`); - - if (useVip98k) { - // VIP98K 密钥只从 AiProvider 读取。 - } else if (useBackupApi) { - // 验证备用API配置 - if (!config.backupApiUrl || !config.backupApiKey) { - return NextResponse.json( - { error: "备用变清晰API地址或密钥未配置" }, - { status: 400 } - ); - } - } else { - // 验证主API配置 - if (!config.apiUrl || !config.apiKey) { - return NextResponse.json( - { error: "变清晰API地址或密钥未配置" }, - { status: 400 } - ); - } - } - - // 2. 获取请求体(支持 imageUrl、image_base64 或 FormData) - console.log("[Enhance API] 开始获取请求体..."); - const contentType = request.headers.get("content-type") || ""; - console.log("[Enhance API] Content-Type:", contentType); - - let imageBase64: string; - let prompt = ""; // 用户提示词(可选) - let chatId: string | null = null; // 聊天ID(可选) - let messageId: string | null = null; // 消息ID(可选) - let sourceImageUrl: string | null = null; - let requestedScaleFactor: number | undefined; - - if (contentType.includes("application/json")) { - console.log("[Enhance API] 解析JSON请求体..."); - const body = await request.json(); - prompt = body.prompt || "图片变清晰处理"; - chatId = body.chatId || null; - messageId = body.messageId || null; - const parsedScaleFactor = Number.parseInt(String(body.scaleFactor || ""), 10); - requestedScaleFactor = - Number.isFinite(parsedScaleFactor) && [1, 2, 4, 8].includes(parsedScaleFactor) - ? parsedScaleFactor - : undefined; - - // 优先使用 imageUrl,其次使用 image_base64 - if (body.imageUrl) { - sourceImageUrl = String(body.imageUrl); - // 从 URL 获取图片并转换为 Base64 - console.log(`[Enhance API] 从 URL 获取图片: ${body.imageUrl}`); - try { - const imageResponse = await fetch(body.imageUrl); - if (!imageResponse.ok) { - return NextResponse.json( - { error: `无法获取图片: ${imageResponse.status} ${imageResponse.statusText}` }, - { status: 400 } - ); - } - const arrayBuffer = await imageResponse.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - imageBase64 = buffer.toString("base64"); - console.log(`[Enhance API] 图片已转换为 Base64,大小: ${buffer.length} bytes`); - } catch (fetchError) { - console.error("[Enhance API] 获取图片失败:", fetchError); - return NextResponse.json( - { error: "无法获取图片,请检查图片链接是否有效" }, - { status: 400 } - ); - } - } else if (body.image_base64) { - console.log("[Enhance API] JSON请求体解析完成,image_base64长度:", body.image_base64?.length || 0); - imageBase64 = body.image_base64; - } else { - return NextResponse.json( - { error: "缺少 imageUrl 或 image_base64 参数" }, - { status: 400 } - ); - } - } else if (contentType.includes("multipart/form-data")) { - // FormData格式:文件上传 - const formData = await request.formData(); - const file = formData.get("file") as File; - - if (!file) { - return NextResponse.json( - { error: "缺少file参数" }, - { status: 400 } - ); - } - - // 将文件转换为Base64 - const arrayBuffer = await file.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - imageBase64 = buffer.toString("base64"); - prompt = (formData.get("prompt") as string) || "图片变清晰处理"; - chatId = (formData.get("chatId") as string) || null; - messageId = (formData.get("messageId") as string) || null; - } else { - return NextResponse.json( - { error: "不支持的Content-Type,请使用application/json或multipart/form-data" }, - { status: 400 } - ); - } - - if (useVip98k) { - try { - const scaleFactor = requestedScaleFactor || 4; - const payload = await processWithVip98kEnhance({ - userId, - imageBase64, - prompt: prompt || "图片变清晰处理", - chatId, - messageId, - sourceImageUrl, - scaleFactor, - }); - return NextResponse.json(payload); - } catch (error) { - console.error("[Enhance-VIP98K] Processing failed", error); - return NextResponse.json( - { - error: "VIP98K 变清晰处理失败", - detail: error instanceof Error ? error.message : "未知错误", - }, - { status: 500 } - ); - } - } - - // ========== 备用API处理路径 (Real-ESRGAN) ========== - if (useBackupApi) { - console.log("[Enhance API] 使用Real-ESRGAN备用API处理(URL方式)..."); - - try { - // 步骤1: 先将Base64图片上传到S3获取公开可访问的URL - console.log(`[Enhance-RealESRGAN] 📤 步骤1: 上传原图到S3获取URL...`); - const tempUploadResult = await uploadBase64ToS3(imageBase64, "image/jpeg"); - const sourceImageUrl = tempUploadResult.original; - console.log(`[Enhance-RealESRGAN] ✅ 原图已上传,URL: ${sourceImageUrl}`); - - // 步骤2: 使用URL方式调用Real-ESRGAN API - console.log(`[Enhance-RealESRGAN] 🔄 步骤2: 调用Real-ESRGAN API处理...`); - const result = await processWithRealESRGAN(sourceImageUrl, { - backupApiUrl: config.backupApiUrl!, - backupApiKey: config.backupApiKey!, - backupScale: String(requestedScaleFactor || Number.parseInt(config.backupScale || "4", 10) || 4), - backupModel: config.backupModel || "RealESRGAN_x4plus", - }); - - // 步骤3: Real-ESRGAN API返回的URL就是处理后的图片 - // 需要重新上传到我们的S3(因为Real-ESRGAN返回的URL可能在另一个S3桶) - console.log(`[Enhance-RealESRGAN] 📤 步骤3: 上传处理后的图片到S3...`); - let thumbnailUrl: string; - let originalUrl: string; - - try { - const uploadResult = await uploadBase64ToS3(result.imageBase64, "image/png"); - thumbnailUrl = uploadResult.thumbnail; - originalUrl = uploadResult.original; - console.log(`[Enhance-RealESRGAN] ✅ 处理后图片已上传: ${originalUrl}`); - } catch (uploadError) { - console.error("[Enhance-RealESRGAN] ❌ 上传到S3失败:", uploadError); - // 如果上传失败,直接使用Real-ESRGAN返回的URL - console.log(`[Enhance-RealESRGAN] ⚠️ 使用Real-ESRGAN返回的URL作为备选: ${result.imageUrl}`); - thumbnailUrl = result.imageUrl; - originalUrl = result.imageUrl; - } - - // 步骤4: 保存到数据库 - let saved = false; - let saveError: unknown = null; - let savedImage: - | { - id: string; - } - | null = null; - for (let attempt = 1; attempt <= 3; attempt += 1) { - try { - ([savedImage] = await db.insert(generatedImage).values({ - userId, - chatId: chatId || null, - messageId: messageId || null, - thumbnailUrl, - originalUrl, - imageSize: `${requestedScaleFactor || Number.parseInt(config.backupScale || "4", 10) || 4}x放大`, - prompt: prompt || "Real-ESRGAN图片变清晰处理", - modelId: `realesrgan-${config.backupModel}`, - }).returning()); - console.log(`[Enhance-RealESRGAN] ✅ 保存到数据库成功,ID: ${savedImage.id}`); - saved = true; - break; - } catch (dbError) { - saveError = dbError; - if (attempt < 3) { - await new Promise((resolve) => setTimeout(resolve, 120 * attempt)); - } - } - } - if (!saved) { - console.error("[Enhance-RealESRGAN] ❌ 保存到数据库失败:", saveError); - throw new Error("图片变清晰成功但保存到相册失败,请稍后重试"); - } - - if (savedImage) { - try { - const sourceAssets = - sourceImageUrl - ? await getMediaAssetsByUserAndUrls({ - userId, - urls: [sourceImageUrl], - }) - : []; - const media = await upsertMediaAsset({ - userId, - chatId, - mediaType: "image", - sourceType: "edited", - status: "ready", - originalUrl, - previewUrl: originalUrl, - thumbnailUrl, - displayName: (prompt || "Real-ESRGAN图片变清晰处理").slice(0, 80), - prompt: prompt || "Real-ESRGAN图片变清晰处理", - modelId: `realesrgan-${config.backupModel}`, - sourceMessageId: messageId, - sourceTable: "GeneratedImage", - sourceRowId: savedImage.id, - primaryParentAssetId: sourceAssets[0]?.id ?? null, - metadata: { - imageSize: `${requestedScaleFactor || Number.parseInt(config.backupScale || "4", 10) || 4}x放大`, - apiType: "realesrgan-url", - }, - }); - if (media) { - for (const sourceAsset of sourceAssets) { - await createMediaAssetRelation({ - userId, - chatId, - fromAssetId: sourceAsset.id, - toAssetId: media.id, - relationType: "edited_from", - sourceMessageId: messageId, - metadata: { - prompt: prompt || "Real-ESRGAN图片变清晰处理", - }, - }); - } - } - } catch (assetError) { - console.error("[Enhance-RealESRGAN] ⚠️ MediaAsset 注册失败:", assetError); - } - } - - // 返回结果 - console.log(`[Enhance-RealESRGAN] ✅ 全部完成,总耗时: ${result.processingTime.toFixed(2)}秒`); - return NextResponse.json({ - success: true, - image_base64: result.imageBase64, - thumbnail_url: thumbnailUrl, - original_url: originalUrl, - processing_time: result.processingTime, - api_type: "realesrgan-url", - }); - - } catch (error) { - console.error("[Enhance-RealESRGAN] ❌ 处理失败:", error); - try { - const fallbackPayload = await executeFalEsrganFallback({ - userId, - imageBase64, - prompt, - chatId, - messageId, - sourceImageUrl, - requestedScaleFactor, - reason: error instanceof Error ? error.message : "RealESRGAN 备用服务失败", - }); - return NextResponse.json(fallbackPayload); - } catch (falError) { - console.error("[Enhance-FAL-ESRGAN] 专用高清备用链路失败:", falError); - } - return NextResponse.json( - { - error: "Real-ESRGAN变清晰处理失败", - detail: error instanceof Error ? error.message : "未知错误", - }, - { status: 500 } - ); - } - } - - // ========== 主API处理路径 ========== - // 3. 调用变清晰API(使用异步模式,根据官方示例) - console.log("[Enhance API] 开始准备主API调用..."); - const apiUrl = config.apiUrl!.endsWith("/") - ? config.apiUrl!.slice(0, -1) - : config.apiUrl!; - - const enhanceUrl = `${apiUrl}/api/tasks/visual/scale`; - console.log("[Enhance API] API URL:", enhanceUrl); - - // 将Base64转换为Buffer - console.log("[Enhance API] 转换Base64为Buffer,长度:", imageBase64.length); - const rawImageBuffer = Buffer.from(imageBase64, "base64"); - const imageBuffer = await ensureAosResolutionLimit(rawImageBuffer, "Enhance API"); - console.log("[Enhance API] Buffer创建完成,大小:", imageBuffer.length, "bytes"); - - // 创建FormData(使用form-data包,Node.js环境) - console.log("[Enhance API] 创建FormData..."); - const FormDataLib = await import("form-data"); - const formData = new FormDataLib.default(); - console.log("[Enhance API] FormData创建完成"); - formData.append("image_file", imageBuffer, { - filename: "image.jpg", - contentType: "image/jpeg", - }); - // 使用异步模式(sync=0),根据官方示例 - formData.append("sync", "0"); - formData.append("type", config.type || "clean"); // 默认通用清晰化 - const scaleFactorValue = - requestedScaleFactor || - (typeof config.scaleFactor === "number" - ? config.scaleFactor - : config.scaleFactor === "auto" - ? undefined - : Number.parseInt(String(config.scaleFactor))); - if (scaleFactorValue) { - formData.append("scale_factor", String(scaleFactorValue)); - } - formData.append("return_type", "2"); // 返回base64 - - const headers: Record = { - "X-API-KEY": config.apiKey!, - ...formData.getHeaders(), // 包含Content-Type和boundary - }; - - const startTime = Date.now(); - let taskId: string; - - try { - console.log("[Enhance API] 开始创建任务..."); - // 将FormData转换为Buffer(form-data包是stream) - console.log("[Enhance API] 转换FormData为Buffer..."); - const formDataBuffer = await new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - formData.on("data", (chunk: any) => { - // 确保chunk是Buffer类型 - if (Buffer.isBuffer(chunk)) { - chunks.push(chunk); - } else if (typeof chunk === "string") { - chunks.push(Buffer.from(chunk, "utf-8")); - } else if (chunk instanceof Uint8Array) { - chunks.push(Buffer.from(chunk)); - } else { - console.warn("[Enhance API] 未知的chunk类型:", typeof chunk, chunk); - chunks.push(Buffer.from(String(chunk), "utf-8")); - } - }); - formData.on("end", () => { - console.log("[Enhance API] FormData stream结束,chunks数量:", chunks.length); - resolve(Buffer.concat(chunks)); - }); - formData.on("error", (error) => { - console.error("[Enhance API] FormData stream错误:", error); - reject(error); - }); - - // 重要:form-data包需要被pipe或手动触发才能开始流式传输 - // 使用PassThrough来触发stream - const { PassThrough } = require("stream"); - const passThrough = new PassThrough(); - passThrough.on("data", () => {}); // 忽略数据,我们已经在formData上监听 - formData.pipe(passThrough); - }); - - console.log("[Enhance API] FormData Buffer转换完成,大小:", formDataBuffer.length, "bytes"); - const uint8Array = new Uint8Array(formDataBuffer); - - let createResponse: Response | null = null; - let lastFetchError: unknown; - - for (let attempt = 1; attempt <= ENHANCE_CREATE_RETRY_COUNT + 1; attempt++) { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), ENHANCE_CREATE_TIMEOUT_MS); - - try { - console.log(`[Enhance API] 发送创建任务请求... attempt=${attempt}`); - createResponse = await fetch(enhanceUrl, { - method: "POST", - headers, - body: uint8Array, - signal: controller.signal, - }); - console.log("[Enhance API] 创建任务响应状态:", createResponse.status, createResponse.statusText); - clearTimeout(timeoutId); - break; - } catch (fetchError) { - clearTimeout(timeoutId); - lastFetchError = fetchError; - const canRetry = attempt <= ENHANCE_CREATE_RETRY_COUNT && isRetryableNetworkError(fetchError); - if (canRetry) { - console.warn( - `[Enhance API] 创建任务失败,准备重试: attempt=${attempt}, error=${fetchError instanceof Error ? fetchError.message : String(fetchError)}` - ); - await new Promise((resolve) => setTimeout(resolve, 1500)); - continue; - } - throw fetchError; - } - } - - if (!createResponse) { - throw lastFetchError instanceof Error ? lastFetchError : new Error("创建任务请求失败"); - } - - if (!createResponse.ok) { - const errorText = await createResponse.text(); - throw new Error(`创建任务失败: ${errorText || createResponse.statusText}`); - } - - const createResult = await createResponse.json(); - - if (!createResult.data || !createResult.data.task_id) { - throw new Error(`API未返回task_id: ${JSON.stringify(createResult)}`); - } - - taskId = createResult.data.task_id; - console.log(`[Enhance] 任务已创建,task_id: ${taskId}`); - } catch (fetchError) { - const processingTime = (Date.now() - startTime) / 1000; - console.error("[Enhance API] 网络请求失败:", { - error: fetchError, - enhanceUrl, - processingTime, - }); - - try { - const fallbackPayload = await executeFalEsrganFallback({ - userId, - imageBase64, - prompt, - chatId, - messageId, - sourceImageUrl, - requestedScaleFactor, - reason: fetchError instanceof Error ? fetchError.message : "AOS 高清任务创建失败", - }); - return NextResponse.json(fallbackPayload); - } catch (falError) { - console.error("[Enhance-FAL-ESRGAN] 专用高清备用链路失败:", falError); - } - - // 检查是否是网络连接错误 - if (fetchError instanceof Error) { - if (fetchError.name === "AbortError") { - return NextResponse.json( - { - error: "变清晰API请求超时", - detail: "创建任务请求超过120秒未响应,请检查API服务器状态或网络连接", - }, - { status: 504 } - ); - } - - // 检查错误消息和cause中的网络错误 - const errorMessage = fetchError.message || ""; - const errorCause = (fetchError as any).cause; - const causeMessage = errorCause?.message || ""; - const causeCode = errorCause?.code || ""; - - // 检查各种网络连接错误 - if (errorMessage.includes("EHOSTUNREACH") || - errorMessage.includes("ENOTFOUND") || - errorMessage.includes("ECONNREFUSED") || - errorMessage.includes("No route to host") || - causeMessage.includes("EHOSTUNREACH") || - causeMessage.includes("ENOTFOUND") || - causeMessage.includes("ECONNREFUSED") || - causeMessage.includes("No route to host") || - causeCode === "EHOSTUNREACH" || - causeCode === "ENOTFOUND" || - causeCode === "ECONNREFUSED") { - return NextResponse.json( - { - error: "无法连接到变清晰API服务器", - detail: `服务器无法访问 ${enhanceUrl},请检查:\n1. API服务器是否在线\n2. 网络连接是否正常\n3. 防火墙设置是否正确\n4. API地址和密钥是否正确`, - }, - { status: 503 } - ); - } - } - - return NextResponse.json( - { - error: "变清晰API请求失败", - detail: fetchError instanceof Error ? fetchError.message : "未知网络错误", - }, - { status: 500 } - ); - } - - // 4. 轮询获取任务结果(短等待,超时后让前端继续轮询) - console.log(`[Enhance] 开始短轮询任务结果,task_id: ${taskId}`); - let taskResult: any; - try { - taskResult = await pollTaskResult( - apiUrl, - config.apiKey!, - taskId, - ENHANCE_TASK_POLL_INTERVAL_MS, - ENHANCE_SYNC_WAIT_TIMEOUT_MS - ); - } catch (pollError) { - if (isEnhanceTaskStillProcessing(pollError)) { - return buildEnhanceTaskPendingResponse(taskId); - } - const detail = pollError instanceof Error ? pollError.message : "轮询失败"; - try { - const fallbackPayload = await executeFalEsrganFallback({ - userId, - imageBase64, - prompt, - chatId, - messageId, - sourceImageUrl, - requestedScaleFactor, - reason: detail, - }); - return NextResponse.json(fallbackPayload); - } catch (falError) { - console.error("[Enhance-FAL-ESRGAN] 专用高清备用链路失败:", falError); - } - const friendlyMessage = getAosFriendlyErrorMessage(detail); - return NextResponse.json( - { error: "变清晰处理失败", detail: friendlyMessage || detail }, - { status: friendlyMessage ? 400 : 500 } - ); - } - - const responsePayload = await finalizeEnhanceTaskResult({ - userId, - taskResult, - prompt, - chatId, - messageId, - startedAt: startTime, - sourceImageUrls: sourceImageUrl ? [sourceImageUrl] : undefined, - }); - - return NextResponse.json(responsePayload); - } catch (error) { - console.error("[Enhance API] 处理错误:", error); - return NextResponse.json( - { error: "服务器内部错误", detail: error instanceof Error ? error.message : "未知错误" }, - { status: 500 } - ); - } +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + return executeEnhancePOST(request, session.user.id); } diff --git a/app/api/extend/route.ts b/app/api/extend/route.ts index 196d74172..fb215ae44 100644 --- a/app/api/extend/route.ts +++ b/app/api/extend/route.ts @@ -1,239 +1,10 @@ -import { type NextRequest, NextResponse } from "next/server"; +import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; -import { ExtendServiceError, executeExtendJob } from "@/lib/ai/extend/service"; -import { getPointsSystemConfig } from "@/lib/db/queries"; -import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; -import { getImageToolCreditCost } from "@/lib/points/config"; -import { - getFriendlyRateLimitMessage, - getUserFacingRequestErrorMessage, - isRateLimitError, -} from "@/lib/request-error"; - +import { executeExtendPOST } from "@/lib/canvas/operations/extend"; +export const runtime = "nodejs"; export const maxDuration = 180; - -function parseIncludeBase64(value: unknown): boolean { - if (typeof value === "boolean") return value; - const normalized = String(value ?? "").trim().toLowerCase(); - return normalized === "true" || normalized === "1" || normalized === "yes"; -} - -function buildExtendSuccessPayload(result: Awaited>) { - return { - success: true, - original_url: result.original_url, - thumbnail_url: result.thumbnail_url, - ...(result.image_base64 ? { base64: result.image_base64 } : {}), - processing_time: result.processing_time, - }; -} - -export async function POST(request: NextRequest) { - try { - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json({ error: "未授权,请先登录" }, { status: 401 }); - } - - const contentType = request.headers.get("content-type") || ""; - - let imageUrl: string | null = null; - let imageBase64: string | null = null; - let prompt = "图像拓展处理"; - let chatId: string | null = null; - let messageId: string | null = null; - let includeBase64 = false; - const requestId = `extend-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; - - if (contentType.includes("application/json")) { - const body = await request.json(); - imageUrl = body.image_url ? String(body.image_url).trim() : null; - imageBase64 = body.image_base64 ? String(body.image_base64).trim() : null; - prompt = body.prompt || prompt; - chatId = body.chatId || null; - messageId = body.messageId || null; - includeBase64 = parseIncludeBase64(body.includeBase64); - - if (!imageUrl && !imageBase64) { - return NextResponse.json( - { error: "缺少image_base64或image_url参数" }, - { status: 400 }, - ); - } - - console.log("[Extend API] Request received:", { - requestId, - source: imageUrl ? "url" : "base64", - includeBase64, - chatId, - messageId, - xScale: body.xScale, - yScale: body.yScale, - }); - - const pointsConfig = await getPointsSystemConfig(); - const pointsCharge = await prepareFeaturePointsCharge({ - userId: session.user.id, - unitCost: getImageToolCreditCost(pointsConfig, "extend"), - reason: "image_extend", - modelId: "image-extend", - note: "图片扩图消耗积分", - pointsConfig, - }); - if (!pointsCharge.ok) { - return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); - } - - const result = await executeExtendJob({ - userId: session.user.id, - imageUrl, - imageBase64, - prompt, - chatId, - messageId, - xScale: body.xScale ? Number(body.xScale) : undefined, - yScale: body.yScale ? Number(body.yScale) : undefined, - includeBase64, - requestId, - }); - - if (pointsCharge.chargeable) { - try { - await pointsCharge.consume(); - } catch (pointsError) { - console.error("[Points] Failed to consume extend points:", pointsError); - } - } - - console.log("[Extend API] Request completed:", { - requestId, - includeBase64, - processingTime: result.processing_time, - originalUrl: result.original_url.slice(0, 120), - }); - - return NextResponse.json(buildExtendSuccessPayload(result)); - } - - if (contentType.includes("multipart/form-data")) { - const formData = await request.formData(); - const file = formData.get("file") as File | null; - if (!file) { - return NextResponse.json({ error: "缺少file参数" }, { status: 400 }); - } - - const arrayBuffer = await file.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - const multipartChatId = String(formData.get("chatId") || "") || null; - const multipartMessageId = String(formData.get("messageId") || "") || null; - includeBase64 = parseIncludeBase64(formData.get("includeBase64")); - console.log("[Extend API] Multipart request received:", { - requestId, - includeBase64, - chatId: multipartChatId, - messageId: multipartMessageId, - bytes: buffer.length, - }); - - const pointsConfig = await getPointsSystemConfig(); - const pointsCharge = await prepareFeaturePointsCharge({ - userId: session.user.id, - unitCost: getImageToolCreditCost(pointsConfig, "extend"), - reason: "image_extend", - modelId: "image-extend", - note: "图片扩图消耗积分", - pointsConfig, - }); - if (!pointsCharge.ok) { - return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); - } - - const result = await executeExtendJob({ - userId: session.user.id, - imageBase64: buffer.toString("base64"), - prompt: String(formData.get("prompt") || prompt), - chatId: multipartChatId, - messageId: multipartMessageId, - includeBase64, - requestId, - }); - - if (pointsCharge.chargeable) { - try { - await pointsCharge.consume(); - } catch (pointsError) { - console.error("[Points] Failed to consume extend points:", pointsError); - } - } - - console.log("[Extend API] Multipart request completed:", { - requestId, - includeBase64, - processingTime: result.processing_time, - originalUrl: result.original_url.slice(0, 120), - }); - - return NextResponse.json(buildExtendSuccessPayload(result)); - } - - return NextResponse.json( - { error: "不支持的Content-Type,请使用application/json或multipart/form-data" }, - { status: 400 }, - ); - } catch (error) { - if (error instanceof ExtendServiceError) { - console.error("[Extend API] Request failed:", { - status: error.status, - message: error.message, - detail: error.detail, - }); - if (error.status === 429 || isRateLimitError(error)) { - return NextResponse.json( - { - error: getFriendlyRateLimitMessage("当前操作过于频繁,请稍后再试"), - detail: error.detail || error.message, - retryAfter: 10, - }, - { - status: 429, - headers: { - "Retry-After": "10", - }, - }, - ); - } - - return NextResponse.json( - { - error: getUserFacingRequestErrorMessage(error, "图像拓展失败"), - detail: error.detail || error.message, - }, - { status: error.status }, - ); - } - - console.error("[Extend API] 错误:", error); - if (isRateLimitError(error)) { - return NextResponse.json( - { - error: getFriendlyRateLimitMessage("当前操作过于频繁,请稍后再试"), - detail: error instanceof Error ? error.message : String(error), - retryAfter: 10, - }, - { - status: 429, - headers: { - "Retry-After": "10", - }, - }, - ); - } - return NextResponse.json( - { - error: getUserFacingRequestErrorMessage(error, "图像拓展失败"), - detail: error instanceof Error ? error.message : String(error), - }, - { status: 500 }, - ); - } +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + return executeExtendPOST(request, session.user.id); } diff --git a/app/api/rmbg/route.ts b/app/api/rmbg/route.ts index 889f44c42..2b2e3caeb 100644 --- a/app/api/rmbg/route.ts +++ b/app/api/rmbg/route.ts @@ -1,138 +1,10 @@ -import { NextRequest, NextResponse } from "next/server"; - +import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; -import { - executeRmbgJob, - fetchImageBufferFromUrl, - RmbgServiceError, -} from "@/lib/ai/rmbg/service"; -import { getPointsSystemConfig } from "@/lib/db/queries"; -import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; -import { getImageToolCreditCost } from "@/lib/points/config"; - +import { executeRemoveBackgroundPOST } from "@/lib/canvas/operations/rmbg"; export const runtime = "nodejs"; export const maxDuration = 60; - -export async function POST(request: NextRequest) { - try { - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json({ error: "未授权,请先登录" }, { status: 401 }); - } - - const contentType = request.headers.get("content-type") || ""; - - let imageBuffer: Buffer; - let imageBase64: string | undefined; - let prompt = "抠图处理"; - let chatId: string | null = null; - let messageId: string | null = null; - let skipGallerySave = false; - let sourceImageUrl: string | null = null; - - if (contentType.includes("application/json")) { - const body = await request.json(); - prompt = body.prompt || "抠图处理"; - chatId = body.chatId || null; - messageId = body.messageId || null; - skipGallerySave = body.skipGallerySave === true || body.skip_gallery_save === 1; - - const imageUrl = body.imageUrl || body.image_url; - const explicitSourceImageUrl = body.sourceImageUrl || body.source_image_url; - - if (imageUrl) { - sourceImageUrl = String(imageUrl); - imageBuffer = await fetchImageBufferFromUrl(sourceImageUrl); - } else if (body.image_base64) { - imageBase64 = String(body.image_base64); - imageBuffer = Buffer.from(imageBase64, "base64"); - if (explicitSourceImageUrl) { - sourceImageUrl = String(explicitSourceImageUrl); - } - } else { - return NextResponse.json( - { error: "缺少 imageUrl 或 image_base64 参数" }, - { status: 400 } - ); - } - } else if (contentType.includes("multipart/form-data")) { - const formData = await request.formData(); - const file = formData.get("file") as File | null; - - if (!file) { - return NextResponse.json({ error: "缺少file参数" }, { status: 400 }); - } - - const arrayBuffer = await file.arrayBuffer(); - imageBuffer = Buffer.from(arrayBuffer); - imageBase64 = imageBuffer.toString("base64"); - prompt = (formData.get("prompt") as string) || "抠图处理"; - chatId = (formData.get("chatId") as string) || null; - messageId = (formData.get("messageId") as string) || null; - skipGallerySave = - formData.get("skipGallerySave") === "true" || formData.get("skip_gallery_save") === "1"; - sourceImageUrl = - (formData.get("sourceImageUrl") as string) || - (formData.get("source_image_url") as string) || - null; - } else { - return NextResponse.json( - { error: "不支持的Content-Type,请使用application/json或multipart/form-data" }, - { status: 400 } - ); - } - - const pointsConfig = await getPointsSystemConfig(); - const pointsCharge = await prepareFeaturePointsCharge({ - userId: session.user.id, - unitCost: getImageToolCreditCost(pointsConfig, "remove-background"), - reason: "image_remove_background", - modelId: "rmbg", - note: "图片抠图消耗积分", - pointsConfig, - }); - if (!pointsCharge.ok) { - return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); - } - - const result = await executeRmbgJob({ - userId: session.user.id, - imageBuffer, - imageBase64, - prompt, - chatId, - messageId, - skipGallerySave, - sourceImageUrl, - }); - - if (pointsCharge.chargeable) { - try { - await pointsCharge.consume(); - } catch (pointsError) { - console.error("[Points] Failed to consume RMBG points:", pointsError); - } - } - - return NextResponse.json({ - success: true, - ...result, - }); - } catch (error) { - if (error instanceof RmbgServiceError) { - return NextResponse.json( - { - error: error.message, - detail: error.detail || error.message, - }, - { status: error.status } - ); - } - - console.error("[RMBG API] 处理错误:", error); - return NextResponse.json( - { error: "服务器内部错误", detail: error instanceof Error ? error.message : "未知错误" }, - { status: 500 } - ); - } +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + return executeRemoveBackgroundPOST(request, session.user.id); } diff --git a/components/canvas/AudioGenerationComposer.tsx b/components/canvas/AudioGenerationComposer.tsx new file mode 100644 index 000000000..0c429f849 --- /dev/null +++ b/components/canvas/AudioGenerationComposer.tsx @@ -0,0 +1,306 @@ +"use client"; +import { useEffect, useState } from "react"; +import { + audioContract, + audioRequestSchema, + buildAudioRequest, + type AudioRequest, +} from "@/lib/video-canvas/audio-contract"; +type Model = { id: string; modelId: string; name: string; kind: string }; +export function AudioGenerationComposer({ + initial, + busy, + onSubmit, + onUpload, + onDraft, +}: { + initial?: Record; + busy: boolean; + onSubmit: (params: AudioRequest) => Promise; + onUpload: (file: File) => Promise; + onDraft: (params: Record) => void; +}) { + const [models, setModels] = useState([]), + [error, setError] = useState(""), + [loading, setLoading] = useState(true), + [sending, setSending] = useState(false); + const [draft, setDraft] = useState>(() => ({ + mode: "tts", + text: "", + voice: "Cherry", + format: "wav", + language: "Chinese", + instrumental: true, + gender: "female", + rate: 1, + pitch: 1, + volume: 50, + ...initial, + })); + const patch = (value: Record) => { + const next = { ...draft, ...value }; + setDraft(next); + onDraft(next); + }; + useEffect(() => { + const controller = new AbortController(); + fetch("/api/canvas/audio/models", { signal: controller.signal }) + .then(async (r) => { + const data = await r.json(); + if (!r.ok) throw Error(data.error || "音频模型读取失败"); + setModels(data.models || []); + }) + .catch((e) => { + if (!controller.signal.aborted) setError(e.message); + }) + .finally(() => { + if (!controller.signal.aborted) setLoading(false); + }); + return () => controller.abort(); + }, []); + const options = models.filter((m) => m.kind === draft.mode), + model = options.find((m) => m.id === draft.modelRef) || options[0], + contract = model ? audioContract(model.modelId) : undefined; + const voice = contract?.voices.some((v) => v.id === draft.voice) + ? draft.voice + : contract?.voices[0]?.id; + const format = contract?.formats.includes(draft.format) ? draft.format : contract?.formats[0]; + const input = "mt-1 w-full rounded-lg border border-white/15 bg-[#303030] p-2 text-sm"; + const run = async () => { + setError(""); + setSending(true); + try { + if (!model) throw Error("当前模式没有已启用的官方模型"); + const params = audioRequestSchema.parse({ + ...draft, + modelRef: model.id, + voice, + format, + ...(!contract?.controls ? { rate: 1, pitch: 1, volume: 50 } : {}), + ...(!contract?.instructions ? { instructions: "" } : {}), + }); + buildAudioRequest(params, model.modelId); + patch(params); + await onSubmit(params); + } catch (e) { + setError(e instanceof Error ? e.message : "提交失败"); + } finally { + setSending(false); + } + }; + return ( +
+

语音与音乐 · 阿里云官方

+
+
+ + +
+