diff --git a/.env.example b/.env.example index e6705c96f..948807853 100644 --- a/.env.example +++ b/.env.example @@ -153,8 +153,9 @@ PRODUCT_PLANNING_DB_SSL=false # Home Performance / Queues # ----------------------------------------------------------------------------- -CANVAS_IMAGE_TASK_CONCURRENCY=3 -CANVAS_IMAGE_TASK_BATCH_DISPATCH_INTERVAL_MS=1000 +# Legacy image queue fields: 0 = unrestricted; model/user admission caps are disabled. +CANVAS_IMAGE_TASK_CONCURRENCY=0 +CANVAS_IMAGE_TASK_BATCH_DISPATCH_INTERVAL_MS=0 EXTERNAL_IMAGE_API_MAX_CONCURRENT=3 EXTERNAL_IMAGE_API_MAX_QUEUE=20 EXTERNAL_IMAGE_API_QUEUE_TIMEOUT_MS=120000 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..e4f7b7c34 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,19 @@ +# Keep source checkouts consistent across Windows and Linux. +.gitattributes text eol=lf +.gitignore text eol=lf +*.ts text eol=lf +*.tsx text eol=lf +*.js text eol=lf +*.jsx text eol=lf +*.mjs text eol=lf +*.cjs text eol=lf +*.json text eol=lf +*.jsonc text eol=lf +*.yaml text eol=lf +*.yml text eol=lf +*.css text eol=lf +*.html text eol=lf +*.md text eol=lf +*.sh text eol=lf +*.bat text eol=crlf +*.cmd text eol=crlf diff --git a/.gitignore b/.gitignore index 2f6051ab0..9523ce24d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ coverage .next/ .next-build/ .next-previous/ +/.next-*/ out/ build @@ -50,6 +51,7 @@ yarn-error.log* /playwright-report/ /blob-report/ /playwright/* +/output/ deploy.tar.gz libvips-8.14.5-linux-x64.tar.br .temp-packages/ @@ -66,10 +68,19 @@ migration-backup-complete/ migration-*.log migration-*.txt +# Local release backups, model credentials and migration scratch data. +/.model-config-backups/ +/backups/ +/tmp/ + # 本地抓包/竞品参考产物，不能提交 artifacts/liblib-*/ analysis/qiaodou-extract/ +# 本地运行时数据库与上传文件 +.local/ +public/uploads/local/ + # 模型文件 *.model *.pth @@ -100,3 +111,26 @@ public/vendor/zoo-modeling-app/ # Drizzle migrations must be versioned (override the global *.sql ignore above) !lib/db/migrations/*.sql + +# Private smart edit runtime and media/model caches +.venv-smart-edit/ +.smart-edit-models/ +.cache/smart-edit/ + +services/video-render/dist/ + +# Downloaded and normalized subtitle fonts are runtime assets. +.cache/smart-edit-fonts/ +services/smart-edit/__pycache__/ + +# Local verification artefacts (also excluded from Tailwind's automatic source scan). +.qa/ +.next-video-*/ +.next-tvstar-*/ + +services/video-upscale/dist/ +services/video-upscale/__pycache__/ +services/video-upscale/vendor/__pycache__/ + +/services/audio-processing/dist/ +/services/audio-processing/__pycache__/ diff --git a/README.md b/README.md index d86270d6b..35a70d008 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ pnpm db:generate 推荐提交前至少运行： ```bash -git fetch github main --prune +git fetch origin --prune pnpm check:branch pnpm check:quality pnpm audit:security diff --git a/app/(admin)/settings/_components/local-ai-sections/image-pipeline-sections.tsx b/app/(admin)/settings/_components/local-ai-sections/image-pipeline-sections.tsx index 630959260..61cc133a1 100644 --- a/app/(admin)/settings/_components/local-ai-sections/image-pipeline-sections.tsx +++ b/app/(admin)/settings/_components/local-ai-sections/image-pipeline-sections.tsx @@ -665,7 +665,7 @@ export function LocalAiImagePipelineSections({

• 支持的图片格式：JPG, JPEG, PNG, BMP, WEBP, TIFF等

- • 图片大小：不超过50MB，不限制分辨率（建议按业务需求控制尺寸） + • 图片大小：不超过20MB，不限制分辨率（建议按业务需求控制尺寸）

• 返回的链接有效期为1小时，系统会自动下载并存储到S3

@@ -784,7 +784,7 @@ export function LocalAiImagePipelineSections({

• 支持2x、4x、8x放大倍数

• 处理时间：通常1-10秒，取决于图片大小

• 支持格式：JPG, PNG, JPEG等常见图片格式

-

• 文件限制：最大50MB

+

• 文件限制：最大20MB

@@ -2377,13 +2377,10 @@ export function LocalAiImagePipelineSections({ items={[ { label: - extendConfig.provider === "vip98k" || extendConfig.apiKey - ? "Key 已配置" - : "Key 待配置", - tone: - extendConfig.provider === "vip98k" || extendConfig.apiKey - ? "enabled" - : "warning", + extendConfig.provider === "vip98k" + ? "VIP98K 服务商" + : "阿里云服务商", + tone: "enabled", }, { label: `${extendConfig.xScale}x / ${extendConfig.yScale}x`, @@ -2400,22 +2397,11 @@ export function LocalAiImagePipelineSections({ VIP98K 服务商读取，仅在这里维护扩展比例。 ) : ( -
- - - setExtendConfig({ ...extendConfig, apiKey: e.target.value }) - } - placeholder="sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - type="password" - value={extendConfig.apiKey} - /> -
- 请输入阿里云Dashscope API密钥，例如: - sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxx +
+ 使用阿里云 DashScope 的 image-out-painting 模型。API Key + 从“AI 模型设置”中已启用的阿里云官方生图服务商读取， + 仅在这里维护拓展比例。
-
)}
@@ -2470,7 +2456,7 @@ export function LocalAiImagePipelineSections({ https://dashscope.aliyuncs.com/api/v1/services/aigc/image2image/out-painting

• 支持的图片格式：JPG, PNG, WebP等常见格式

-

• 最大图片大小：4MB

+

• 最大图片大小：10MB

• 处理时间：约5-15秒/张

• 放大倍数范围：1.0-2.0（推荐1.5倍等比例扩展）

diff --git a/app/(admin)/settings/_components/local-ai-sections/runtime-sections.tsx b/app/(admin)/settings/_components/local-ai-sections/runtime-sections.tsx index b9da6a47c..397c31689 100644 --- a/app/(admin)/settings/_components/local-ai-sections/runtime-sections.tsx +++ b/app/(admin)/settings/_components/local-ai-sections/runtime-sections.tsx @@ -1078,139 +1078,15 @@ export function LocalAiRuntimeSections({ - {/* 全站生图单模型并发 */} - 控制全站生图链路中，同一个模型允许同时运行的任务数，以及排队后下一批任务的发起间隔。配置保存到数据库，异步队列和直调生成都会自动读取，无需重启服务。 - - } + description="生图任务直接发起，图片先显示，后台转存到存储桶。" expanded={isLocalAiSectionExpanded("canvas-image-task-queue")} id="local-ai-section-canvas-image-task-queue" onToggle={() => toggleLocalAiSection("canvas-image-task-queue")} - summary={ - - } - title={ - - - 全站生图单模型并发与批次间隔 - - } + summary={} + title={生图任务执行} > -
-
-
生效范围
-
-

- • - 影响画布异步生图、智能体直调生图、视频分镜高清预览等全站生图入口。 -

-

- • - 多用户共享同一套并发限制，但会按模型分别计数，不同模型之间不再互相阻塞。 -

-

- • 当前配置的是单模型并发，不是全站总并发，也不是单用户并发。 -

-

- • - 批次间隔只控制排队补位的“下一批”发起节奏，不影响首批任务即时启动。 -

-

- • 数值过高可能触发上游模型 429、超时或 504，建议配合模型 Key - 池继续细分。 -

-
-
- -
-
- - - setCanvasImageTaskQueueConfig( - normalizeCanvasImageTaskQueueConfig({ - ...canvasImageTaskQueueConfig, - concurrency: e.target.value, - }) - ) - } - type="number" - value={canvasImageTaskQueueConfig.concurrency} - /> -
- 可配置范围 {MIN_CANVAS_IMAGE_TASK_CONCURRENCY}- - {MAX_CANVAS_IMAGE_TASK_CONCURRENCY}，当前为{" "} - {canvasImageTaskQueueConfig.concurrency} 并发，默认值为{" "} - {DEFAULT_CANVAS_IMAGE_TASK_CONCURRENCY}。 -
-
- -
- - - setCanvasImageTaskQueueConfig( - normalizeCanvasImageTaskQueueConfig({ - ...canvasImageTaskQueueConfig, - batchDispatchIntervalMs: Math.round( - Number.parseFloat(e.target.value || "0") * 1000 - ), - }) - ) - } - step="0.5" - type="number" - value={String( - canvasImageTaskQueueConfig.batchDispatchIntervalMs / 1000 - )} - /> -
- 可配置范围 {MIN_CANVAS_IMAGE_BATCH_DISPATCH_INTERVAL_MS / 1000}- - {MAX_CANVAS_IMAGE_BATCH_DISPATCH_INTERVAL_MS / 1000} 秒，当前为{" "} - {( - canvasImageTaskQueueConfig.batchDispatchIntervalMs / 1000 - ).toFixed(1)}{" "} - 秒，默认值为{" "} - {( - DEFAULT_CANVAS_IMAGE_BATCH_DISPATCH_INTERVAL_MS / 1000 - ).toFixed(1)}{" "} - 秒。 -
-
-
-
- -
- -
+

已取消模型及单用户并发限制，任务无需等待其他图片生成完成。任务状态、超时处理和失败重试继续保留。

 {/* 批量处理配置 */} diff --git a/app/(admin)/settings/_lib/types.ts b/app/(admin)/settings/_lib/types.ts index 1ac65384d..521541cbd 100644 --- a/app/(admin)/settings/_lib/types.ts +++ b/app/(admin)/settings/_lib/types.ts @@ -275,7 +275,6 @@ export type EnhanceConfig = { export type ExtendConfig = { provider: "vip98k" | "dashscope"; - apiKey: string; xScale: string; yScale: string; }; diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index 0e65ec869..28f553080 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -38,6 +38,7 @@ import { ImagePlus, Key, Layers, + LayoutTemplate, Layers3, Loader2, type LucideIcon, @@ -1061,6 +1062,7 @@ const CANVAS_TOOLBAR_FALLBACK_ICONS: Record = imageRemoveBackground: Scissors, imageBoxCutout: Scan, imageLocalEdit: Wand2, + imageLayoutReplace: LayoutTemplate, imageRemoveWatermark: Eraser, imageResize: Maximize2, imageTextEdit: Type, @@ -2180,7 +2182,7 @@ export function AdminSettingsClient() { usageTimeEnabled: false, usageTimeStart: "08:00", usageTimeEnd: "20:00", - usageTimeMessage: "由壹点累了也是需要休息的", + usageTimeMessage: "BuYi 累了也是需要休息的", defaultTheme: "system", bannerEnabled: true, bannerText: "限时特惠：升级会员最高立享 45% OFF!", @@ -2222,7 +2224,7 @@ export function AdminSettingsClient() { agentWorkflowDefaultModelId: null, canvasDetailDefaultModelId: null, canvasDetailAgentModelId: null, - lovartComposerPrefix: "让「由壹点AI」", + lovartComposerPrefix: "让「𝗕𝘂𝗬𝗶」", lovartComposerPreviewTexts: [ "打造引人注目的社交媒体视觉", "帮我把这张产品图做成高级电商海报", @@ -2238,8 +2240,8 @@ export function AdminSettingsClient() { lovartPromoText: "现在升级，立享 Kling 3.0 与 Nano Banana Pro，最高 5 折优惠", lovartPromoActionText: "立即升级", - lovartHeroTitle: "由壹点AI 让设计更简单", - lovartHeroSubtitle: "懂你的设计代理，帮你搞定一切", + lovartHeroTitle: "𝗕𝘂𝗬𝗶 让设计触手可及", + lovartHeroSubtitle: "下达指令，即刻呈现", surpriseCategories: ["全部", "精选"], membershipConfig: defaultMembershipConfig, membershipUserOverrides: defaultMembershipUserOverrides, @@ -2300,7 +2302,7 @@ export function AdminSettingsClient() { usageTimeStart: siteSettingsData.usageTimeStart || "08:00", usageTimeEnd: siteSettingsData.usageTimeEnd || "20:00", usageTimeMessage: - siteSettingsData.usageTimeMessage || "由壹点累了也是需要休息的", + siteSettingsData.usageTimeMessage || "BuYi 累了也是需要休息的", defaultTheme: siteSettingsData.defaultTheme || "system", bannerEnabled: siteSettingsData.bannerEnabled ?? true, bannerText: @@ -2396,7 +2398,7 @@ export function AdminSettingsClient() { (siteSettingsData as any).canvasDetailAgentModelId || null, lovartComposerPrefix: String((siteSettingsData as any).lovartComposerPrefix || "").trim() || - "让「由壹点AI」", + "让「𝗕𝘂𝗬𝗶」", lovartComposerPreviewTexts: Array.isArray( siteSettingsData.lovartComposerPreviewTexts ) @@ -2498,9 +2500,9 @@ export function AdminSettingsClient() { lovartPromoActionText: siteSettingsData.lovartPromoActionText || "立即升级", lovartHeroTitle: - siteSettingsData.lovartHeroTitle || "由壹点AI 让设计更简单", + siteSettingsData.lovartHeroTitle || "𝗕𝘂𝗬𝗶 让设计触手可及", lovartHeroSubtitle: - siteSettingsData.lovartHeroSubtitle || "懂你的设计代理，帮你搞定一切", + siteSettingsData.lovartHeroSubtitle || "下达指令，即刻呈现", surpriseCategories: Array.isArray(siteSettingsData.surpriseCategories) ? siteSettingsData.surpriseCategories .map((item: any) => String(item).trim()) @@ -2631,8 +2633,7 @@ export function AdminSettingsClient() { // 图像拓展API配置状态 const [extendConfig, setExtendConfig] = useState({ - provider: "vip98k" as "vip98k" | "dashscope", - apiKey: "", + provider: "dashscope" as "vip98k" | "dashscope", xScale: "1.5", // X轴放大倍数 yScale: "1.5", // Y轴放大倍数 }); @@ -2951,13 +2952,11 @@ export function AdminSettingsClient() { if (extendConfigData?.setting?.value) { const config = extendConfigData.setting.value as { provider?: "vip98k" | "dashscope"; - apiKey?: string; xScale?: number; yScale?: number; }; setExtendConfig({ provider: config.provider === "vip98k" ? "vip98k" : "dashscope", - apiKey: config.apiKey || "", xScale: config.xScale ? String(config.xScale) : "1.5", yScale: config.yScale ? String(config.yScale) : "1.5", }); @@ -5554,11 +5553,6 @@ export function AdminSettingsClient() { // --- 图像拓展API配置逻辑 --- const handleSaveExtendConfig = async () => { try { - if (extendConfig.provider !== "vip98k" && !extendConfig.apiKey.trim()) { - toast.error("请输入API密钥"); - return; - } - const xScale = Number.parseFloat(extendConfig.xScale); const yScale = Number.parseFloat(extendConfig.yScale); @@ -5579,7 +5573,6 @@ export function AdminSettingsClient() { key: "extend_api_config", value: { provider: extendConfig.provider, - apiKey: extendConfig.apiKey.trim(), xScale, yScale, }, @@ -8913,9 +8906,7 @@ export function AdminSettingsClient() { { key: "realtime-canvas", title: "实时与队列", - description: `单模型并发 ${canvasImageTaskQueueConfig.concurrency}，批次间隔 ${( - canvasImageTaskQueueConfig.batchDispatchIntervalMs / 1000 - ).toFixed(1)} 秒`, + description: "生图并发不限，任务即时发起", status: realtimeCanvasConfig.enabled ? "实时已启用" : "实时已关闭", tone: realtimeCanvasConfig.enabled ? "enabled" : "neutral", icon: SlidersHorizontal, @@ -8998,7 +8989,7 @@ export function AdminSettingsClient() { key: "extend", label: "图像拓展", group: "图像处理", - tone: extendConfig.apiKey ? "enabled" : "warning", + tone: "enabled", }, { key: "watermark-remove", @@ -10522,7 +10513,7 @@ export function AdminSettingsClient() { lovartComposerPrefix: e.target.value.slice(0, 40), }) } - placeholder="让「由壹点AI」" + placeholder="让「𝗕𝘂𝗬𝗶」" value={siteSettings.lovartComposerPrefix} />
@@ -10724,7 +10715,7 @@ export function AdminSettingsClient() { lovartHeroTitle: e.target.value, }) } - placeholder="由壹点AI 让设计更简单" + placeholder="𝗕𝘂𝗬𝗶 让设计触手可及" value={siteSettings.lovartHeroTitle} />
@@ -10738,7 +10729,7 @@ export function AdminSettingsClient() { lovartHeroSubtitle: e.target.value, }) } - placeholder="懂你的设计代理，帮你搞定一切" + placeholder="下达指令，即刻呈现" value={siteSettings.lovartHeroSubtitle} />
diff --git a/app/(admin)/settings/video-canvas/video-canvas-settings-client.tsx b/app/(admin)/settings/video-canvas/video-canvas-settings-client.tsx index c79fba2c7..6251dd3fc 100644 --- a/app/(admin)/settings/video-canvas/video-canvas-settings-client.tsx +++ b/app/(admin)/settings/video-canvas/video-canvas-settings-client.tsx @@ -1,4 +1,5 @@ "use client"; +import { filterCanvasModels } from "@/lib/video-canvas/model-matching"; import Link from "next/link"; import { ArrowLeft, Plus, Trash2 } from "lucide-react"; @@ -24,10 +25,6 @@ import { } from "@/components/ui/select"; import { Switch } from "@/components/ui/switch"; import { Textarea } from "@/components/ui/textarea"; -import { - isImageGenerationModelDescriptor, - isVideoModelDescriptor, -} from "@/lib/ai/model-classification"; import { fetcher } from "@/lib/utils"; import { getVideoCanvasFunctionBillingCopy, @@ -87,56 +84,23 @@ function getModelOptionLabel(model: AiModelOption) { return `${name}${id}`; } -function isImageModelOption(model: AiModelOption) { - return isImageGenerationModelDescriptor({ - providerType: model.providerType, - modelId: model.modelId, - capabilities: model.capabilities, - supportsImageGen: model.supportsImageGen, - }); -} - -function isVideoModelOption(model: AiModelOption) { - return isVideoModelDescriptor({ - providerType: model.providerType, - modelId: model.modelId, - capabilities: model.capabilities, - supportsImageGen: model.supportsImageGen, - }); -} - -function filterModelsForFunctionKind( - models: AiModelOption[], - kind: VideoCanvasFunctionDefinition["kind"], -) { - const enabled = models.filter((model) => model.isEnabled !== false && getModelOptionValue(model)); - let filtered = enabled; - if (kind === "image" || kind === "image-edit") { - filtered = enabled.filter(isImageModelOption); - } else if (kind === "video") { - filtered = enabled.filter(isVideoModelOption); - } else if (kind === "multimodal") { - filtered = enabled.filter( - (model) => model.isMultimodal || model.supportsVision || (!isImageModelOption(model) && !isVideoModelOption(model)), - ); - } - return filtered.length ? filtered : enabled; -} - function ModelSelect({ allowInherit = false, kind, + functionKey, models, onChange, value, }: { allowInherit?: boolean; kind: VideoCanvasFunctionDefinition["kind"]; + functionKey?: string; models: AiModelOption[]; onChange: (value: string) => void; value: string; }) { - const filteredModels = filterModelsForFunctionKind(models, kind); + const modelKind = functionKey === "audio-to-video" ? "audio-video" : kind === "image" || kind === "image-edit" || kind === "video" || kind === "multimodal" ? kind : "text"; + const filteredModels = filterCanvasModels(models, modelKind); const normalizedValue = value?.trim() || (allowInherit ? INHERIT_MODEL_VALUE : "auto"); const hasCurrent = normalizedValue === "auto" || @@ -851,13 +815,14 @@ export function VideoCanvasSettingsClient() {
- updateFunction(item.key, { model: value })} value={item.model || ""} - /> + /> :

{item.route === "local" ? "本地处理，无需生成模型" : "使用此功能的专用服务接口"}

}
diff --git a/app/(chat)/api/chat/_lib/detail-replacement.ts b/app/(chat)/api/chat/_lib/detail-replacement.ts index c17b68e19..defadd30d 100644 --- a/app/(chat)/api/chat/_lib/detail-replacement.ts +++ b/app/(chat)/api/chat/_lib/detail-replacement.ts @@ -2,9 +2,31 @@ import { createLogger } from "@/lib/observability/logger"; import { generateText } from "ai"; import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; +import { + DETAIL_REPLACEMENT_IMAGE_MODEL_CONFIG_KEY, + normalizeDetailReplacementImageModelConfig, +} from "@/lib/ai/detail-replacement-image-model-config"; +import { + compositeReplacementWithReferenceLocks, + parseReplacementLayoutLockRegions, + type NormalizedReplacementBox, +} from "@/lib/ai/replacement-layout-lock"; +import { + buildReplacementProductIdentityAnalysisPrompt, + buildReplacementProductStructureLockLines, + formatReplacementProductIdentityProfile, + normalizeReplacementProductImageIndexes, + parseReplacementProductIdentityProfile, + type ReplacementProductIdentityProfile, +} from "@/lib/ai/replacement-product-identity"; import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime"; +import { resolveDetailReplacementVisionRuntime } from "@/lib/ai/detail-replacement-vision-runtime"; import { inlineVisionImages } from "@/lib/ai/vision-inline"; -import { getEnabledAiModelsWithProvider, getSiteSettings } from "@/lib/db/queries"; +import { + getEnabledAiModelsWithProvider, + getSiteSettings, + getSystemSetting, +} from "@/lib/db/queries"; import { uploadFile } from "@/lib/storage/service"; import { bufferToArrayBuffer } from "./persistence"; import { isVideoProviderType } from "./video-models"; @@ -234,6 +256,8 @@ export async function normalizeReplacementResultImage(params: { referenceImageUrl: string; referenceGeometry?: ReplacementReferenceGeometry | null; scenePlan?: ReplacementScenePlan | null; + preserveReferenceOutsideSubject?: boolean; + preserveReferenceText?: boolean; uploadPrefix: string; }): Promise<{ originalUrl: string; @@ -242,6 +266,11 @@ export async function normalizeReplacementResultImage(params: { }> { const sharp = (await import("sharp")).default; const generatedBuffer = await fetchImageBufferFromUrl(params.generatedImageUrl); + let referenceBuffer: Buffer | null = null; + const getReferenceBuffer = async () => { + referenceBuffer ||= await fetchImageBufferFromUrl(params.referenceImageUrl); + return referenceBuffer; + }; let targetWidth = params.referenceGeometry?.width; let targetHeight = params.referenceGeometry?.height; const generatedMetadata = await sharp(generatedBuffer, { failOn: "none" }) @@ -250,8 +279,8 @@ export async function normalizeReplacementResultImage(params: { if (!targetWidth || !targetHeight) { try { - const referenceBuffer = await fetchImageBufferFromUrl(params.referenceImageUrl); - const referenceMetadata = await sharp(referenceBuffer, { failOn: "none" }) + const resolvedReferenceBuffer = await getReferenceBuffer(); + const referenceMetadata = await sharp(resolvedReferenceBuffer, { failOn: "none" }) .rotate() .metadata(); targetWidth = referenceMetadata.width; @@ -298,7 +327,7 @@ export async function normalizeReplacementResultImage(params: { : shouldPreserveFullGeneratedCanvas ? "contain" : "cover"; - const normalizedBuffer = + let normalizedBuffer = targetWidth && targetHeight ? await imagePipeline .resize(targetWidth, targetHeight, { @@ -312,6 +341,82 @@ export async function normalizeReplacementResultImage(params: { .png() .toBuffer() : await imagePipeline.png().toBuffer(); + + if ( + targetWidth && + targetHeight && + (params.preserveReferenceOutsideSubject || params.preserveReferenceText) + ) { + let layoutLocks = { + subjectBoundingBox: params.scenePlan?.subjectBoundingBox || null, + textBoundingBoxes: params.scenePlan?.textBoundingBoxes || [], + analyzed: params.scenePlan?.layoutLockAnalyzed === true, + }; + const needsLayoutLockFallback = + !layoutLocks.analyzed || + (params.preserveReferenceOutsideSubject && !layoutLocks.subjectBoundingBox); + + if (needsLayoutLockFallback) { + const fallbackLocks = await resolveReplacementLayoutLockRegions({ + referenceImageUrl: params.referenceImageUrl, + referenceBuffer: await getReferenceBuffer(), + }); + layoutLocks = { + subjectBoundingBox: + layoutLocks.subjectBoundingBox || fallbackLocks.subjectBoundingBox, + textBoundingBoxes: + layoutLocks.textBoundingBoxes.length > 0 + ? layoutLocks.textBoundingBoxes + : fallbackLocks.textBoundingBoxes, + analyzed: layoutLocks.analyzed || fallbackLocks.analyzed, + }; + } + + const shouldUseSafeReferenceFallback = Boolean( + (params.preserveReferenceOutsideSubject && !layoutLocks.subjectBoundingBox) || + (params.preserveReferenceText && !layoutLocks.analyzed), + ); + normalizedBuffer = shouldUseSafeReferenceFallback + ? await sharp(await getReferenceBuffer(), { failOn: "none" }) + .rotate() + .resize(targetWidth, targetHeight, { fit: "fill" }) + .png() + .toBuffer() + : await compositeReplacementWithReferenceLocks({ + generatedBuffer: normalizedBuffer, + referenceBuffer: await getReferenceBuffer(), + width: targetWidth, + height: targetHeight, + preserveReferenceOutsideSubject: Boolean( + params.preserveReferenceOutsideSubject && layoutLocks.subjectBoundingBox, + ), + subjectBoundingBox: layoutLocks.subjectBoundingBox, + textBoundingBoxes: params.preserveReferenceText + ? layoutLocks.textBoundingBoxes + : [], + }); + if (shouldUseSafeReferenceFallback) { + chatApiLogger.warn( + "[Chat API] Replacement layout lock analysis unavailable; preserved the reference image instead of returning a layout-damaged result", + { + preserveReferenceOutsideSubject: params.preserveReferenceOutsideSubject, + preserveReferenceText: params.preserveReferenceText, + hasSubjectBoundingBox: Boolean(layoutLocks.subjectBoundingBox), + layoutLockAnalyzed: layoutLocks.analyzed, + }, + ); + } + chatApiLogger.info("[Chat API] Replacement reference layout locks applied:", { + preserveReferenceOutsideSubject: Boolean( + params.preserveReferenceOutsideSubject && layoutLocks.subjectBoundingBox, + ), + preservedTextRegionCount: params.preserveReferenceText + ? layoutLocks.textBoundingBoxes.length + : 0, + usedFallbackAnalysis: needsLayoutLockFallback, + usedSafeReferenceFallback: shouldUseSafeReferenceFallback, + }); + } const thumbnailBuffer = await createAgentImageThumbnailBuffer(normalizedBuffer); const uniqueSuffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; @@ -399,6 +504,12 @@ export type ReplacementScenePlan = { generationPrompt: string; selectedProductImageIndexes: number[]; productSelectionReason: string; + productIdentitySummary?: string; + productStructureSummary?: string; + productStructuralRisks?: string; + subjectBoundingBox?: NormalizedReplacementBox | null; + textBoundingBoxes?: NormalizedReplacementBox[]; + layoutLockAnalyzed?: boolean; analysisModelName?: string | null; analysisSource: "local_moonshot" | "configured_vision" | "heuristic"; }; @@ -439,6 +550,188 @@ export type ReplacementScenePlanningRuntime = const QWEN_VL_MODEL_MATCHER = /(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i; +const replacementProductIdentityProfileCache = new Map< + string, + { + expiresAt: number; + promise: Promise; + } +>(); + +async function resolveReplacementProductIdentityProfile(params: { + productImageUrls: string[]; + planningRuntime: Exclude; +}) { + const productImageUrls = Array.from( + new Set( + params.productImageUrls + .map((url) => String(url || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)), + ), + ).slice(0, 8); + if (productImageUrls.length === 0) return null; + + const cacheKey = [ + params.planningRuntime.kind, + params.planningRuntime.modelName, + ...productImageUrls, + ].join("|"); + const now = Date.now(); + const cached = replacementProductIdentityProfileCache.get(cacheKey); + if (cached && cached.expiresAt > now) { + return cached.promise; + } + + if (replacementProductIdentityProfileCache.size >= 100) { + const oldestKey = replacementProductIdentityProfileCache.keys().next().value; + if (oldestKey) replacementProductIdentityProfileCache.delete(oldestKey); + } + + const promise = (async () => { + try { + const { inlineDataUrls } = await inlineVisionImages(productImageUrls, { + maxImages: productImageUrls.length, + maxSide: 800, + timeoutMs: 12_000, + retries: 2, + loggerPrefix: "[Chat API] Replacement product identity", + trimTransparentPadding: true, + }); + const effectiveImages = + inlineDataUrls.length === productImageUrls.length + ? inlineDataUrls + : productImageUrls; + const { text } = await generateText({ + model: params.planningRuntime.model, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: buildReplacementProductIdentityAnalysisPrompt( + productImageUrls.length, + ), + }, + ...effectiveImages.map((image) => ({ + type: "image" as const, + image, + })), + ], + }, + ], + system: + "你是高精度商品结构分析器。多张图片属于同一商品；请跨视角还原唯一结构身份，并明确每个视角最清楚的构件。", + temperature: params.planningRuntime.kind === "local_moonshot" ? 1 : 0.1, + maxTokens: 1800, + abortSignal: AbortSignal.timeout( + params.planningRuntime.kind === "local_moonshot" ? 90_000 : 60_000, + ), + } as any); + const profile = parseReplacementProductIdentityProfile( + text, + productImageUrls.length, + ); + if (!profile) { + throw new Error("商品结构画像缺少必需字段"); + } + return profile; + } catch (error) { + chatApiLogger.warn("[Chat API] Replacement product identity analysis failed:", { + productImageCount: productImageUrls.length, + modelName: params.planningRuntime.modelName, + error: error instanceof Error ? error.message : "unknown", + }); + return null; + } + })(); + + replacementProductIdentityProfileCache.set(cacheKey, { + expiresAt: now + 30 * 60 * 1000, + promise, + }); + return promise; +} + +const replacementLayoutLockCache = new Map< + string, + { + expiresAt: number; + promise: Promise>; + } +>(); + +async function resolveReplacementLayoutLockRegions(params: { + referenceImageUrl: string; + referenceBuffer: Buffer; +}) { + const now = Date.now(); + const cached = replacementLayoutLockCache.get(params.referenceImageUrl); + if (cached && cached.expiresAt > now) { + return cached.promise; + } + + if (replacementLayoutLockCache.size >= 200) { + const oldestKey = replacementLayoutLockCache.keys().next().value; + if (oldestKey) { + replacementLayoutLockCache.delete(oldestKey); + } + } + + const promise = (async () => { + try { + const sharp = (await import("sharp")).default; + const inlineBuffer = await sharp(params.referenceBuffer, { failOn: "none" }) + .rotate() + .resize(1600, 1600, { + fit: "inside", + withoutEnlargement: true, + }) + .flatten({ background: "#ffffff" }) + .jpeg({ quality: 88 }) + .toBuffer(); + const { callQwen3Vl } = await import("@/lib/ai/qwen3-vl"); + const result = await callQwen3Vl({ + imageUrl: `data:image/jpeg;base64,${inlineBuffer.toString("base64")}`, + prompt: [ + "你是电商详情图的布局区域定位器。", + "只分析当前图像，返回严格 JSON，不要解释。", + "所有坐标使用 0到1000 的归一化坐标，格式必须是 [x1,y1,x2,y2]，绝对不是 x/y/宽/高。", + "subjectBoundingBox：只框出图中当前旧商品主体的可视外轮廓区域，包含完整商品和它的自然阴影，不要包含文字；没有商品则为 null。", + "textBoundingBoxes：框出所有可见文案块，包括中英文标题、卖点、参数、标签、数字、品牌字样、按钮文字和紧邻的图标；没有则返回空数组。", + '{"subjectBoundingBox":[0,0,1000,1000],"textBoundingBoxes":[[0,0,1000,1000]]}', + ].join("\n"), + temperature: 0, + maxTokens: 1600, + timeoutMs: 60_000, + retries: 1, + }); + const parsedLocks = parseReplacementLayoutLockRegions(result.text); + if (!parsedLocks.analyzed) { + throw new Error("布局区域定位结果缺少必需字段"); + } + return parsedLocks; + } catch (error) { + replacementLayoutLockCache.delete(params.referenceImageUrl); + chatApiLogger.warn("[Chat API] Replacement layout lock fallback analysis failed:", { + referenceImageUrl: params.referenceImageUrl, + error: error instanceof Error ? error.message : "unknown", + }); + return { + subjectBoundingBox: null, + textBoundingBoxes: [], + analyzed: false, + }; + } + })(); + + replacementLayoutLockCache.set(params.referenceImageUrl, { + expiresAt: now + 30 * 60 * 1000, + promise, + }); + return promise; +} + export function normalizeDetailReplacementStrategy( value: unknown, ): AgentDetailReplacementStrategy { @@ -1441,7 +1734,27 @@ export function parseReplacementScenePlan( visualProfile: params.visualProfile, }); - const productImageSelection = buildDefaultProductImageSelection(params.productImageCount); + const defaultProductImageSelection = buildDefaultProductImageSelection( + params.productImageCount, + ); + const selectedProductImageIndexes = normalizeReplacementProductImageIndexes( + parsed.selectedProductImageIndexes ?? + parsed.selectedProductIndexes ?? + parsed.productImageIndexes, + params.productImageCount, + ); + const productImageSelection = + selectedProductImageIndexes.length > 0 + ? { + selectedProductImageIndexes, + productSelectionReason: + truncateReplacementAnalysisText( + parsed.productSelectionReason ?? parsed.selectionReason, + 240, + ) || "按当前详情底板机位选择最匹配的产品视角。", + } + : defaultProductImageSelection; + const layoutLocks = parseReplacementLayoutLockRegions(parsed); let scenePlan: ReplacementScenePlan = { pageKind, generationRoute, @@ -1486,6 +1799,9 @@ export function parseReplacementScenePlan( generationPrompt, selectedProductImageIndexes: productImageSelection.selectedProductImageIndexes, productSelectionReason: productImageSelection.productSelectionReason, + subjectBoundingBox: layoutLocks.subjectBoundingBox, + textBoundingBoxes: layoutLocks.textBoundingBoxes, + layoutLockAnalyzed: layoutLocks.analyzed, analysisModelName: null, analysisSource: "heuristic", }; @@ -1844,6 +2160,27 @@ export async function resolveDirectReplacementModelRef(params: { finalModelId?: string | null; models?: Awaited>; }) { + const models = params.models || (await getEnabledAiModelsWithProvider(false)); + const imageModels = models.filter( + ({ model, provider }) => + model.supportsImageGen && + !model.isSubModel && + !isVideoProviderType(provider?.type, model), + ); + const configuredModelSetting = await getSystemSetting({ + key: DETAIL_REPLACEMENT_IMAGE_MODEL_CONFIG_KEY, + }).catch(() => null); + const configuredModelId = normalizeDetailReplacementImageModelConfig( + configuredModelSetting?.value, + ).modelId; + const configuredModel = imageModels.find( + ({ model }) => + model.id === configuredModelId || model.modelId === configuredModelId, + ); + if (configuredModel) { + return configuredModel.model.id; + } + const explicitModel = String(params.selectedChatModel || "").trim(); if (explicitModel && explicitModel !== "auto") { return explicitModel; @@ -1853,13 +2190,6 @@ export async function resolveDirectReplacementModelRef(params: { return routedModel; } - const models = params.models || (await getEnabledAiModelsWithProvider(false)); - const imageModels = models.filter( - ({ model, provider }) => - model.supportsImageGen && - !model.isSubModel && - !isVideoProviderType(provider?.type, model), - ); const preferred = imageModels.find(({ model }) => model.isDefault) || imageModels.find(({ provider }) => String(provider?.type || "").includes("image")) || @@ -2121,6 +2451,16 @@ export async function analyzeReplacementReferenceImage(params: { export async function resolveReplacementScenePlanningRuntime(params: { models?: Awaited>; }): Promise { + try { + const dedicatedVisionRuntime = await resolveDetailReplacementVisionRuntime(); + if (dedicatedVisionRuntime) return dedicatedVisionRuntime; + } catch (error) { + chatApiLogger.warn( + "[Chat API] Failed to resolve dedicated detail replacement vision model:", + error, + ); + } + try { const siteSettings = await getSiteSettings().catch(() => null); const localDetailAgentResolved = await resolveDetailAgentLocalMoonshotModel({ @@ -2191,6 +2531,13 @@ export async function analyzeReplacementExecutionPlan(params: { const referenceVisualProfile = isDetailReplacement ? await resolveReplacementReferenceVisualProfile(params.referenceImageUrl) : null; + const productIdentityProfile = + params.planningRuntime && params.productImageUrls.length > 0 + ? await resolveReplacementProductIdentityProfile({ + productImageUrls: params.productImageUrls, + planningRuntime: params.planningRuntime, + }) + : null; if (params.planningRuntime) { try { const replacementScenePlanningTimeoutMs = @@ -2221,7 +2568,10 @@ export async function analyzeReplacementExecutionPlan(params: { ? "你是电商详情参考图背景替换流程中的“版式与背景区域规划器”。" : "你是电商详情替换流程中的“场景理解规划器”。", `当前要分析的参考图：${params.referenceLabel}`, - "你现在只分析这一张参考图本身，不分析产品图，不分析模特图。", + "当前视觉输入只包含这一张详情参考图；产品图已经单独做过跨视角结构分析，请根据下面的产品结构画像选择最匹配当前机位的产品视角，不要臆测产品结构。", + productIdentityProfile + ? `【产品结构画像】\n${formatReplacementProductIdentityProfile(productIdentityProfile)}` + : "产品结构画像暂不可用：仍需按当前参考图机位选择产品图，但不得把旧产品的腿型、支架或构件拓扑当成新产品结构。", referenceVisualProfile ? `像素初筛：nearWhite=${referenceVisualProfile.nearWhitePixelRatio.toFixed(2)}, lightLowSaturation=${referenceVisualProfile.lightLowSaturationPixelRatio.toFixed(2)}, topLight=${referenceVisualProfile.topLightLowSaturationPixelRatio.toFixed(2)}, borderLight=${referenceVisualProfile.borderLightLowSaturationPixelRatio.toFixed(2)}, dark=${referenceVisualProfile.darkPixelRatio.toFixed(2)}, likelyWhiteLayout=${referenceVisualProfile.likelyWhiteLayout ? "true" : "false"}, likelyWhiteScene=${referenceVisualProfile.likelyWhiteScene ? "true" : "false"}。${referenceVisualProfile.reason}` : "", @@ -2231,7 +2581,7 @@ export async function analyzeReplacementExecutionPlan(params: { !referenceVisualProfile?.likelyWhiteLayout && referenceVisualProfile?.likelyWhiteScene ? "像素初筛已经命中白/浅色低饱和真实场景：这不是白底说明页。若画面存在真实空间、人物或产品照片，promptPackPolicy 应为 soft 或 allow，必须使用前置工作流词包重建图片框内产品、空间、软装和灯光，使差异清晰可见；但最终仍保持白/浅色低饱和基调，不要改造成深色厨房、黑墙、复杂吊灯或强软装。" : "", - "产品图和模特图会在后续生图阶段单独提供给生图模型使用，你这里不要尝试理解它们的具体外观，也不要要求必须使用所有输入图。", + "产品图和模特图会在后续生图阶段单独提供给生图模型。你必须依据产品结构画像输出 selectedProductImageIndexes，选择最多两张最能匹配当前参考图机位并共同说明完整结构的产品图；不得固定选择前两张，也不要要求强行使用全部产品图。", params.modelImageUrl && !isProductOnlyDetailReplacement ? "你只需要根据当前参考图判断：这一屏是否存在真实的人物展示需求；如果没有人物需求，useModelImage 必须为 false。" : "本轮没有模特图，useModelImage 必须为 false。", @@ -2278,12 +2628,18 @@ export async function analyzeReplacementExecutionPlan(params: { isDetailReplacement ? "5. 判断 textPolicy：画面中存在营销标题、卖点、编号、参数、标签且与图匹配时必须 lock 或 preserve，避免后续生图改字、混色或重排。" : "", + isDetailReplacement + ? "5a. 必须输出 subjectBoundingBox 和 textBoundingBoxes。所有坐标为 0到1000 归一化坐标 [x1,y1,x2,y2]，不是 x/y/宽/高。subjectBoundingBox 紧密框住旧商品主体及自然阴影，不含文字，无商品时为 null；textBoundingBoxes 覆盖所有中英文文案、标题、卖点、参数、标签、数字、品牌字样、按钮文字及紧邻图标，无文字时为 []。" + : "", isDetailReplacement ? "6. 判断 peoplePolicy：有人物/多人/自拍/聚会/围坐/聚餐/模特互动/拿手机/用餐/试坐/承重动作时必须 preserve_relation；如果文字明确写了“多人围坐、聚餐、合照、用餐、单侧承重、横向受力测试、拿手机”等动作，generationPrompt 必须把这个动作写成硬约束；如果只提供 1 张模特图但参考图多人，仍要保留参考图人数和关系，额外人物按参考图生成相近人群。" : "", isDetailReplacement ? "7. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary；cameraSummary 必须写出产品主体的视角特征，例如正侧视/三分之四/俯视/平视、倾斜方向、可见面数量、主体朝向和透视消失方向。" : "", + isDetailReplacement + ? "7a. 输出 selectedProductImageIndexes：根据当前详情图机位，从产品结构画像的图片编号中选择最多两张最相关的产品图；第一张应最接近当前机位，第二张补足腿、支架、横梁、侧框或其他容易变形的结构。输出 productSelectionReason 说明选择依据。" + : "", isProductOnlyDetailReplacement ? "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，要求只替换产品主体，保留其余背景、文案、排版、装饰和光影完全不变。" : isDetailReplacement @@ -2309,7 +2665,7 @@ export async function analyzeReplacementExecutionPlan(params: { : "", "只返回严格 JSON，不要任何额外解释。", `JSON 格式： -{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字","pageKind":"prompt_pack_scene|white_scene|white_layout|copy_locked|people_scene|action_poster|dimension_spec|material_detail|unknown","generationRoute":"${isDetailReplacement ? "prompt_pack_scene|layout_preserve_replace|copy_locked_regenerate|people_relation_locked" : "prompt_pack_scene|layout_preserve_replace|copy_locked_regenerate|people_relation_locked|direct_passthrough"}","promptPackPolicy":"allow|soft|forbid","textPolicy":"free|preserve|lock","peoplePolicy":"none|preserve_relation|require_user_model","whiteBackgroundLikely":false,"copyImageCoupled":false,"containsPeople":false,"containsMultiplePeople":false,"routeReason":"不超过50字","useModelImage":false,"modelImageReason":"不超过40字","sceneSummary":"不超过80字","cameraSummary":"不超过60字","placementSummary":"不超过60字","preserveSummary":"不超过80字","styleSummary":"不超过80字","avoidSummary":"不超过80字","generationPrompt":"不超过300字"}`, +{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字","pageKind":"prompt_pack_scene|white_scene|white_layout|copy_locked|people_scene|action_poster|dimension_spec|material_detail|unknown","generationRoute":"${isDetailReplacement ? "prompt_pack_scene|layout_preserve_replace|copy_locked_regenerate|people_relation_locked" : "prompt_pack_scene|layout_preserve_replace|copy_locked_regenerate|people_relation_locked|direct_passthrough"}","promptPackPolicy":"allow|soft|forbid","textPolicy":"free|preserve|lock","peoplePolicy":"none|preserve_relation|require_user_model","whiteBackgroundLikely":false,"copyImageCoupled":false,"containsPeople":false,"containsMultiplePeople":false,"subjectBoundingBox":[0,0,1000,1000],"textBoundingBoxes":[[0,0,1000,1000]],"selectedProductImageIndexes":[1,2],"productSelectionReason":"不超过80字","routeReason":"不超过50字","useModelImage":false,"modelImageReason":"不超过40字","sceneSummary":"不超过80字","cameraSummary":"不超过60字","placementSummary":"不超过60字","preserveSummary":"不超过80字","styleSummary":"不超过80字","avoidSummary":"不超过80字","generationPrompt":"不超过300字"}`, ] .filter(Boolean) .join("\n"); @@ -2334,7 +2690,7 @@ export async function analyzeReplacementExecutionPlan(params: { system: "你是高精度商品详情图场景规划器。你只分析当前参考图本身，负责理解这一屏的场景、机位、主体位置和人物需求，不要臆测或依赖其他输入图片。", temperature: params.planningRuntime.kind === "local_moonshot" ? 1 : 0.1, - maxTokens: 1200, + maxTokens: 1600, abortSignal: AbortSignal.timeout(replacementScenePlanningTimeoutMs), } as any); @@ -2368,6 +2724,9 @@ export async function analyzeReplacementExecutionPlan(params: { : "configured_vision", scenePlan: { ...normalizedPlan.scenePlan, + productIdentitySummary: productIdentityProfile?.identitySummary || "", + productStructureSummary: productIdentityProfile?.structureSummary || "", + productStructuralRisks: productIdentityProfile?.structuralRisks || "", analysisModelName: params.planningRuntime.modelName, analysisSource: params.planningRuntime.kind === "local_moonshot" @@ -2413,7 +2772,12 @@ export async function analyzeReplacementExecutionPlan(params: { ); return { ...normalizedFallbackPlan.decision, - scenePlan: normalizedFallbackPlan.scenePlan, + scenePlan: { + ...normalizedFallbackPlan.scenePlan, + productIdentitySummary: productIdentityProfile?.identitySummary || "", + productStructureSummary: productIdentityProfile?.structureSummary || "", + productStructuralRisks: productIdentityProfile?.structuralRisks || "", + }, }; } @@ -2537,11 +2901,17 @@ export function buildAgentReplacementPrompt(params: { ] : []; const productAngleLockLines = [ - "【产品角度锁定规则】最终商品/主体的拍摄角度以最后一张参考图为准，而不是以前面的产品输入图为准；前面的产品图只提供款式、材质、颜色、结构和细节。", - "必须逐项对齐最后一张参考图中的产品姿态：朝向、正侧视/三分之四/俯视/平视关系、桌面或面板倾斜方向、可见侧面数量、支架/腿部/把手/转轴位置、主体外轮廓、占比、重心和透视消失方向。", + "【产品角度锁定规则】最终商品/主体的拍摄角度、朝向、占比、重心和透视消失方向以最后一张参考图为准；商品身份、外轮廓、腿型、支架拓扑、构件数量和连接方式以前面的产品输入图为准。", + "必须对齐最后一张参考图的正侧视/三分之四/俯视/平视关系、桌面或面板倾斜方向和可见侧面数量，但不得照搬旧商品的腿部、横梁、侧框、把手、转轴或内部构件设计。", "如果最后一张参考图的产品是侧视或低视角，不允许改成俯视三分之四；如果是俯视三分之四，不允许改成正侧视；不允许镜像、旋转、翻转或把产品重画成另一个角度。", "若参考图含功能标签、箭头、尺寸线、编号或引导线，它们指向的实际结构部位必须保持一致，不得因产品转角改变而让标签指向错误位置。", ]; + const productStructureLockLines = buildReplacementProductStructureLockLines({ + productImageCount: params.productImageCount, + identitySummary: params.scenePlan?.productIdentitySummary, + structureSummary: params.scenePlan?.productStructureSummary, + structuralRisks: params.scenePlan?.productStructuralRisks, + }); const normalizedBasePrompt = String(params.basePrompt || "").trim(); const promptPackInstructionLines = params.modeType === "detail_replacement" && @@ -2641,6 +3011,7 @@ export function buildAgentReplacementPrompt(params: { ...promptPackInstructionLines, ...scenePlanLines, ...marketplaceBrandRemovalLines, + ...productStructureLockLines, ...productAngleLockLines, "你现在做的是‘仅替换产品’，不是重建场景，也不是重新设计整张详情图。", "忽略用户补充提示词里的场景风格、氛围、空间改造要求；这一次只服务于产品主体替换。", @@ -2670,6 +3041,7 @@ export function buildAgentReplacementPrompt(params: { ...promptPackInstructionLines, ...scenePlanLines, ...marketplaceBrandRemovalLines, + ...productStructureLockLines, ...productAngleLockLines, "你现在做的是“详情参考图背景/场景图替换”，不是重新设计整张详情图。", "【版式锁层规则】把最后一张参考图当成 PSD 模板：文字、标题、编号、卖点标签、按钮、图标、白底/浅色底、留白、分割线、原有圆角照片区域和原有裁切范围都属于锁定版式层，不允许移动、缩放、重排或重新设计；不要把这些版式边界画成新的白色线框、描边或卡片。", @@ -2785,6 +3157,7 @@ export function buildAgentReplacementPrompt(params: { "【图片顺序规则】", ...imageOrderLines, ...scenePlanLines, + ...productStructureLockLines, ...productAngleLockLines, params.pairedProductIndex !== undefined ? `请优先使用第 ${params.pairedProductIndex + 1} 张输入商品图作为本张参考图的一对一替换主体。` diff --git a/app/(chat)/api/chat/_lib/errors.ts b/app/(chat)/api/chat/_lib/errors.ts index 9c0c42592..90d6f5dae 100644 --- a/app/(chat)/api/chat/_lib/errors.ts +++ b/app/(chat)/api/chat/_lib/errors.ts @@ -1,6 +1,6 @@ import { createLogger } from "@/lib/observability/logger"; import { NextResponse } from "next/server"; -import { getRequestErrorMessage } from "@/lib/request-error"; +import { getRequestErrorMessage, isProviderBalanceError, isRateLimitError } from "@/lib/request-error"; import { generateUUID } from "@/lib/utils"; const chatApiLogger = createLogger("chat.errors"); @@ -20,6 +20,14 @@ export function getChatStreamErrorMessage(error: unknown): string { error instanceof Error ? String(error.message || "").trim() : String(error || "").trim(); const normalized = rawMessage.toLowerCase(); + if (isProviderBalanceError(error)) { + return "当前模型服务商线路余额或额度不足，请切换模型或联系管理员；与您的积分余额无关。"; + } + + if (isRateLimitError(error)) { + return "当前模型服务请求过于频繁，请稍后重试或切换模型。"; + } + if ( normalized.includes("token quota is not enough") || normalized.includes("pre_consume_token_quota_failed") || @@ -47,6 +55,16 @@ function resolveChatApiFailure(error: unknown) { const detailMessage = getRequestErrorMessage(error); const normalized = String(detailMessage || "").toLowerCase(); + if (isProviderBalanceError(error)) { + return { + code: "server_error:chat", + message: "模型服务商线路余额或额度不足，请切换模型或联系管理员。", + status: 503, + category: "provider_balance", + detailMessage, + }; + } + if ( normalized.includes("payload too large") || normalized.includes("request entity too large") || diff --git a/app/(chat)/api/chat/_lib/image-failover.ts b/app/(chat)/api/chat/_lib/image-failover.ts index 8d1dc9e92..e4b9d7ec0 100644 --- a/app/(chat)/api/chat/_lib/image-failover.ts +++ b/app/(chat)/api/chat/_lib/image-failover.ts @@ -4,6 +4,8 @@ import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-capability"; import type { ChatMessage } from "@/lib/types"; import { convertMessagesWithImages } from "./image-media"; +import { isProviderBalanceError } from "@/lib/request-error"; +import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown"; const chatApiLogger = createLogger("chat.imageFailover"); @@ -57,7 +59,21 @@ export function resolveImageModelRuntimeId( export function normalizeFailoverErrorMessage(error: unknown): string { if (!error) return ""; if (typeof error === "string") return error; - if (error instanceof Error) return error.message || String(error); + if (error instanceof Error) { + const record = error as Error & { + status?: unknown; + statusCode?: unknown; + httpStatus?: unknown; + response?: { status?: unknown }; + }; + const status = + record.status ?? + record.statusCode ?? + record.httpStatus ?? + record.response?.status; + const message = error.message || String(error); + return status ? `[HTTP ${String(status)}] ${message}` : message; + } try { return JSON.stringify(error); } catch { @@ -68,6 +84,7 @@ export function normalizeFailoverErrorMessage(error: unknown): string { export function shouldCooldownImageFallbackModel(errorMessage: string): boolean { const msg = errorMessage.toLowerCase(); return ( + isProviderBalanceError(errorMessage) || msg.includes("model not found") || msg.includes("model_not_found") || msg.includes("no available channel for model") || @@ -76,6 +93,13 @@ export function shouldCooldownImageFallbackModel(errorMessage: string): boolean msg.includes("api key is missing") || msg.includes("invalid api key") || msg.includes("unauthorized") || + /(?:^|\D)401(?:\D|$)/.test(msg) || + msg === "forbidden" || + msg.includes("access forbidden") || + msg.includes("access denied") || + msg.includes("permission denied") || + /(?:^|\D)403(?:\D|$)/.test(msg) || + msg.includes("vip98k task failed") || msg.includes("cannot convert argument to a bytestring") || msg.includes("model maintenance") || msg.includes("maintenance") || @@ -302,9 +326,9 @@ export function buildImageFallbackCandidates( if (!sameProvider && !IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(providerType)) { return false; } - if (isImageFallbackModelCoolingDown(item.model.id)) { + if (isImageFallbackModelCoolingDown(item.model.id) || isImageFallbackModelBalanceCoolingDown(item.model)) { const cooldown = imageFailoverModelCooldown.get(item.model.id); - chatApiLogger.info(`[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "cooldown"}`); + chatApiLogger.info(`[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "provider balance cooldown"}`); return false; } return true; @@ -464,9 +488,9 @@ export function getImageCrossProviderCandidates( } if (m.model.id === options.currentModelId) return false; if (options.currentProviderId && m.provider.id === options.currentProviderId) return false; - if (isImageFallbackModelCoolingDown(m.model.id)) { + if (isImageFallbackModelCoolingDown(m.model.id) || isImageFallbackModelBalanceCoolingDown(m.model)) { const cooldown = imageFailoverModelCooldown.get(m.model.id); - chatApiLogger.info(`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}`); + chatApiLogger.info(`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "provider balance cooldown"}`); return false; } return true; diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 71286129d..82138faf1 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -1,3 +1,5 @@ +import { isExplicitDestructiveIntent } from "@/lib/ai/canvas-agent-runtime/capability-access"; +import { inferVideoTaskRequirements } from "@/lib/ai/canvas-agent-runtime/video-protocol"; import { createLogger } from "@/lib/observability/logger"; import { geolocation } from "@vercel/functions"; import { @@ -179,6 +181,7 @@ import { ChatSDKError } from "@/lib/errors"; import { getFriendlyRateLimitMessage, getRequestErrorMessage, + getUserFacingImageGenerationErrorMessage, isRateLimitError, } from "@/lib/request-error"; import { @@ -193,10 +196,12 @@ import { } from "@/lib/ai/runtime/agent-media-markdown"; import { extractImageUrlsFromText } from "@/lib/image-url"; import { saveGeneratedImages } from "@/lib/ai/runtime/generated-image-persistence"; +import { withAuthenticationFailover } from "@/lib/ai/runtime/authentication-failover"; import { recordAiModelHealthFailure, recordAiModelHealthSuccess, shouldFailoverAiModelError, + shouldFailoverVip98kRequestRejection, } from "@/lib/ai/runtime/model-health"; import type { FalImageEditTaskMetadata } from "@/lib/ai/runtime/fal-image-edit-task"; import { saveAssistantTextOnlyMessage } from "@/lib/ai/runtime/assistant-message-persistence"; @@ -363,6 +368,10 @@ import { resolveReplacementReferenceGeometry, resolveReplacementScenePlanningRuntime, } from "./_lib/detail-replacement"; +import { + selectDetailReplacementGenerationInputs, + shouldIncludeDetailReplacementModelImage, +} from "@/lib/ai/detail-replacement-inputs"; import { normalizeAgentReportForDisplay, sanitizeAgentOutputSettingsSection, @@ -417,8 +426,6 @@ const chatApiLogger = createLogger("chat.route"); const INTERNAL_IMAGE_GENERATION_FAILURE_PATTERN = /生成图片失败|图片生成失败|未找到图片数据|未解析到图片|模型未返回生成图片|图片数据不可用|未返回可用图片/i; -const DETAIL_REPLACEMENT_MODEL_TEXT_HINT_PATTERN = - /自拍|合照|聚会|聚餐|围坐|用餐|试坐|承重|受力|拿手机|手持|人物场景|人物关系|真人演示|人物展示|人像展示|固定模特|用户模特|same\s+model|fixed\s+model|person|people|human/i; function assertGeneratedImageMarkdownHasImage(markdown: string, modelLabel: string) { const normalizedMarkdown = String(markdown || "").trim(); @@ -435,41 +442,13 @@ function assertGeneratedImageMarkdownHasImage(markdown: string, modelLabel: stri function getPublicImageGenerationErrorText(error: unknown) { const message = error instanceof Error ? error.message : String(error || ""); - if (!message || INTERNAL_IMAGE_GENERATION_FAILURE_PATTERN.test(message)) { + const publicMessage = getUserFacingImageGenerationErrorMessage(error); + if (!message || (publicMessage === message && INTERNAL_IMAGE_GENERATION_FAILURE_PATTERN.test(message))) { return "图片生成失败，请稍后重试"; } - return message; + return publicMessage; } -function replacementScenePlanShouldIncludeModelImage(scenePlan: any) { - if (!scenePlan) return false; - if (scenePlan.useModelImage === true) return true; - if ( - scenePlan.peoplePolicy && - scenePlan.peoplePolicy !== "none" - ) { - return true; - } - if (scenePlan.generationRoute === "people_relation_locked") return true; - if (scenePlan.pageKind === "people_scene" || scenePlan.pageKind === "action_poster") { - return true; - } - - const text = [ - scenePlan.modelImageReason, - scenePlan.routeReason, - scenePlan.sceneSummary, - scenePlan.cameraSummary, - scenePlan.placementSummary, - scenePlan.preserveSummary, - scenePlan.styleSummary, - scenePlan.avoidSummary, - scenePlan.generationPrompt, - ] - .filter(Boolean) - .join(" "); - return DETAIL_REPLACEMENT_MODEL_TEXT_HINT_PATTERN.test(text); -} const AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT = Math.max( 1, @@ -928,6 +907,15 @@ export async function POST(request: Request) { const json = await request.json(); chatApiLogger.info("[Chat API] Received request body:", JSON.stringify(json, null, 2)); requestBody = postRequestBodySchema.parse(json); + // Agent/Auto Agent has one execution owner: the isolated native ZCode entry. + // This also upgrades existing browser tabs without routing Claw here. + if (requestBody.composerMode === "agent" && requestBody.contextPayload?.source === "canvas" && requestBody.contextPayload.projectId) { + const { POST: nativeAgentPost } = await import("@/app/api/canvas/agent-chat/route"); + return nativeAgentPost(new Request(new URL("/api/canvas/agent-chat", request.url), { + method: "POST", headers: request.headers, body: JSON.stringify(json), + })); + } + } catch (error) { chatApiLogger.error("[Chat API] Schema validation failed:", error); return new ChatSDKError("bad_request:api").toResponse(); @@ -1222,6 +1210,14 @@ export async function POST(request: Request) { } } + if (shouldRunAgentConversation && preferredAgentModelId && !shouldKeepSelectedModel) { + const preferred = await getAiModelById({ id: preferredAgentModelId }).catch(() => null); + if (!preferred?.isEnabled || preferred.healthStatus === "auto_disabled") { + preferredAgentModelId = undefined; + configuredAgentConversationModelId = undefined; + } + } + if (shouldRunAgentConversation && !preferredAgentModelId) { const configuredAgentModel = await getAgentAiModelWithProvider().catch((agentModelError) => { chatApiLogger.warn("[Chat API] Failed to load configured agent model:", agentModelError); @@ -1342,6 +1338,7 @@ export async function POST(request: Request) { } const allowAgentVideoGeneration = + (contextPayload?.source === "canvas" && inferVideoTaskRequirements(requestTextContent).expectsVideo) || Boolean(requestBody.videoParams) || isVideoProviderType(requestedModelProviderType, requestedModelConfig || undefined); let smartAutoWorkflowDecision: @@ -5723,12 +5720,15 @@ export async function POST(request: Request) { sourceImageUrls: string[]; streamLabel: string; promptForPersistence?: string | null; + isFallback?: boolean; }) => { const persistedImageSize = String(params.imageSizeLabel || "").trim() || resolvedImageSize; const assistantText = prependImageModelBadge( [ ``, + ``, + ``, `### ${params.reference.label}`, String(params.bodyText || "").trim(), buildAgentToolImagesMarkdown([ @@ -5764,26 +5764,28 @@ export async function POST(request: Request) { ], }); - await saveGeneratedImages( - assistantText, - session.user.id, - id, - assistantMessageId, - params.promptForPersistence || baseReplacementPrompt, - resolvedModelRef, - { - sourceAssetIds, - sourceImageUrls: params.sourceImageUrls, - relationType: "edited_from", - sourceType: "detail_replacement", - projectId: contextPayload?.projectId || null, - userPrompt: requestTextContent, - optimizedPrompt: params.promptForPersistence || baseReplacementPrompt, - requestedImageCount: 1, - aspectRatio: resolvedAspectRatio, - imageSize: persistedImageSize, - }, - ); + if (!params.isFallback) { + await saveGeneratedImages( + assistantText, + session.user.id, + id, + assistantMessageId, + params.promptForPersistence || baseReplacementPrompt, + resolvedModelRef, + { + sourceAssetIds, + sourceImageUrls: params.sourceImageUrls, + relationType: "edited_from", + sourceType: "detail_replacement", + projectId: contextPayload?.projectId || null, + userPrompt: requestTextContent, + optimizedPrompt: params.promptForPersistence || baseReplacementPrompt, + requestedImageCount: 1, + aspectRatio: resolvedAspectRatio, + imageSize: persistedImageSize, + }, + ); + } } catch (persistError) { chatApiLogger.warn("[Chat API] Failed to persist direct detail replacement result; streaming response anyway:", { chatId: id, @@ -5933,20 +5935,18 @@ export async function POST(request: Request) { const shouldIncludeModelImage = detailReplacementStrategy !== "product_only" && Boolean(modelInputImageUrl) && - (replacementScenePlanShouldIncludeModelImage( + (shouldIncludeDetailReplacementModelImage( replacementExecutionPlan.scenePlan, ) || selectedProductContextImageUrls.length === 0); - const sourceImageUrls = Array.from( - new Set([ - ...selectedProductContextImageUrls, - ...(shouldIncludeModelImage && modelInputImageUrl - ? [modelInputImageUrl] - : []), - ...rerunReferenceImageUrls, - reference.imageUrl, - ]), - ); + const generationInputs = selectDetailReplacementGenerationInputs({ + productImageUrls: selectedProductContextImageUrls, + modelImageUrl: modelInputImageUrl, + includeModelImage: shouldIncludeModelImage, + rerunReferenceImageUrls, + referenceImageUrl: reference.imageUrl, + }); + const sourceImageUrls = generationInputs.imageUrls; chatApiLogger.info("[Chat API] Direct detail replacement image generation started:", { referenceIndex: referenceIndex + 1, totalReferences: replacementReferences.length, @@ -5954,10 +5954,10 @@ export async function POST(request: Request) { pageKind: replacementExecutionPlan.scenePlan.pageKind, generationRoute: replacementExecutionPlan.scenePlan.generationRoute, packPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy, - selectedProductImages: selectedProductContextImageUrls.length, + selectedProductImages: generationInputs.productImageUrls.length, sourceProductImages: productInputImageUrls.length, - includeModelImage: shouldIncludeModelImage, - rerunReferenceImages: rerunReferenceImageUrls.length, + includeModelImage: Boolean(generationInputs.modelImageUrl), + rerunReferenceImages: generationInputs.rerunImageUrls.length, sourceImageCount: sourceImageUrls.length, whiteLayout: replacementExecutionPlan.scenePlan.whiteBackgroundLikely, copyCoupled: replacementExecutionPlan.scenePlan.copyImageCoupled, @@ -5972,10 +5972,10 @@ export async function POST(request: Request) { referenceLabel: reference.label, referenceIndex, totalReferences: replacementReferences.length, - hasModelImage: shouldIncludeModelImage, - productImageCount: selectedProductContextImageUrls.length, + hasModelImage: Boolean(generationInputs.modelImageUrl), + productImageCount: generationInputs.productImageUrls.length, originalProductImageCount: productInputImageUrls.length, - rerunReferenceCount: rerunReferenceImageUrls.length, + rerunReferenceCount: generationInputs.rerunImageUrls.length, templateReferenceCount: 0, scenePlan: replacementExecutionPlan.scenePlan, referenceGeometry, @@ -6007,6 +6007,10 @@ export async function POST(request: Request) { imageSize: resolvedImageSize, imageCount: 1, referenceImages: sourceImageUrls, + promptExtend: false, + providerImageParams: { + inputImageRoleByUrl: generationInputs.inputImageRoleByUrl, + }, debugLabel: "agent-detail-replacement", debugMeta: { referenceLabel: reference.label, @@ -6054,6 +6058,9 @@ export async function POST(request: Request) { referenceImageUrl: reference.imageUrl, referenceGeometry, scenePlan: replacementExecutionPlan.scenePlan, + preserveReferenceOutsideSubject: + detailReplacementStrategy === "product_only", + preserveReferenceText: true, uploadPrefix: "agent-detail-replacement", }); finalImage = { @@ -6136,8 +6143,8 @@ export async function POST(request: Request) { modelDisplayName: null, sourceImageUrls: referenceSourceImageUrls, streamLabel: "agent-mode:image:detail-replacement:item-fallback", + isFallback: true, }); - successCount += 1; } } catch (replacementPlanError) { chatApiLogger.error("[Chat API] Direct detail replacement item failed before generation:", { @@ -6160,8 +6167,8 @@ export async function POST(request: Request) { modelDisplayName: null, sourceImageUrls: fallbackSourceImageUrls, streamLabel: "agent-mode:image:detail-replacement:item-fallback", + isFallback: true, }); - successCount += 1; } }); } finally { @@ -6489,9 +6496,20 @@ export async function POST(request: Request) { ...(selectedAgentPreset.defaultParams || {}), ...(replacementSlot?.defaultParams || {}), } as Record; - const resolvedModelRef = - String(selectedChatModel || "").trim() && - String(selectedChatModel || "").trim() !== "auto" + const replacementVisionModels = await getEnabledAiModelsWithProvider().catch( + (error) => { + chatApiLogger.warn("[Chat API] Failed to load configured vision models for replacement analysis:", error); + return undefined; + }, + ); + const resolvedModelRef = isDetailReplacementPreset + ? await resolveDirectReplacementModelRef({ + selectedChatModel, + finalModelId: presetModelId, + models: replacementVisionModels, + }) + : String(selectedChatModel || "").trim() && + String(selectedChatModel || "").trim() !== "auto" ? String(selectedChatModel || "").trim() : presetModelId; const configuredReplacementAspectRatio = @@ -6513,12 +6531,6 @@ export async function POST(request: Request) { configuredAspectRatio: configuredReplacementAspectRatio, }); } - const replacementVisionModels = await getEnabledAiModelsWithProvider().catch( - (error) => { - chatApiLogger.warn("[Chat API] Failed to load configured vision models for replacement analysis:", error); - return undefined; - }, - ); const replacementScenePlanningRuntime = await resolveReplacementScenePlanningRuntime({ models: replacementVisionModels, @@ -6816,24 +6828,36 @@ export async function POST(request: Request) { !(isDetailReplacementPreset && detailReplacementStrategy === "product_only") && Boolean(modelInputImageUrl) && - (replacementScenePlanShouldIncludeModelImage( + (shouldIncludeDetailReplacementModelImage( replacementExecutionPlan.scenePlan, ) || (isDetailReplacementPreset && selectedProductContextImageUrls.length === 0)); - const sourceImageUrls = Array.from( - new Set( - [ - ...selectedProductContextImageUrls, - ...(shouldIncludeModelImage && modelInputImageUrl - ? [modelInputImageUrl] - : []), - ...replacementTemplateReferenceImageUrls, - ...rerunReferenceImageUrls, - reference.imageUrl, - ].filter(Boolean), - ), - ); + const detailGenerationInputs = isDetailReplacementPreset + ? selectDetailReplacementGenerationInputs({ + productImageUrls: selectedProductContextImageUrls, + modelImageUrl: modelInputImageUrl, + includeModelImage: shouldIncludeModelImage, + templateReferenceImageUrls: replacementTemplateReferenceImageUrls, + rerunReferenceImageUrls, + referenceImageUrl: reference.imageUrl, + }) + : null; + const sourceImageUrls = detailGenerationInputs + ? detailGenerationInputs.imageUrls + : Array.from( + new Set( + [ + ...selectedProductContextImageUrls, + ...(shouldIncludeModelImage && modelInputImageUrl + ? [modelInputImageUrl] + : []), + ...replacementTemplateReferenceImageUrls, + ...rerunReferenceImageUrls, + reference.imageUrl, + ].filter(Boolean), + ), + ); const replacementPrompt = buildAgentReplacementPrompt({ basePrompt: baseReplacementPrompt || @@ -6850,11 +6874,19 @@ export async function POST(request: Request) { referenceLabel: reference.label, referenceIndex, totalReferences: replacementReferences.length, - hasModelImage: shouldIncludeModelImage, - productImageCount: selectedProductContextImageUrls.length, + hasModelImage: detailGenerationInputs + ? Boolean(detailGenerationInputs.modelImageUrl) + : shouldIncludeModelImage, + productImageCount: detailGenerationInputs + ? detailGenerationInputs.productImageUrls.length + : selectedProductContextImageUrls.length, originalProductImageCount: productInputImageUrls.length, - rerunReferenceCount: rerunReferenceImageUrls.length, - templateReferenceCount: replacementTemplateReferenceImageUrls.length, + rerunReferenceCount: detailGenerationInputs + ? detailGenerationInputs.rerunImageUrls.length + : rerunReferenceImageUrls.length, + templateReferenceCount: detailGenerationInputs + ? detailGenerationInputs.templateImageUrls.length + : replacementTemplateReferenceImageUrls.length, pairedProductIndex, scenePlan: replacementExecutionPlan.scenePlan, referenceGeometry, @@ -6875,11 +6907,20 @@ export async function POST(request: Request) { sourceImageCount: sourceImageUrls.length, generationAspectRatio, imageOrder: { - productImages: selectedProductContextImageUrls.length, + productImages: + detailGenerationInputs !== null + ? detailGenerationInputs.productImageUrls.length + : selectedProductContextImageUrls.length, originalProductImages: productInputImageUrls.length, - hasModelImage: shouldIncludeModelImage, - templateReferenceImages: replacementTemplateReferenceImageUrls.length, - rerunReferenceImages: rerunReferenceImageUrls.length, + hasModelImage: detailGenerationInputs + ? Boolean(detailGenerationInputs.modelImageUrl) + : shouldIncludeModelImage, + templateReferenceImages: detailGenerationInputs + ? detailGenerationInputs.templateImageUrls.length + : replacementTemplateReferenceImageUrls.length, + rerunReferenceImages: detailGenerationInputs + ? detailGenerationInputs.rerunImageUrls.length + : rerunReferenceImageUrls.length, finalReferenceImageIndex: sourceImageUrls.length, }, }); @@ -6897,6 +6938,17 @@ export async function POST(request: Request) { imageSize: resolvedImageSize, imageCount: 1, referenceImages: sourceImageUrls, + promptExtend: isDetailReplacementPreset + ? false + : typeof mergedImageParams.promptExtend === "boolean" + ? mergedImageParams.promptExtend + : undefined, + providerImageParams: detailGenerationInputs + ? { + inputImageRoleByUrl: + detailGenerationInputs.inputImageRoleByUrl, + } + : undefined, debugLabel: `agent-${replacementSourceType}`, debugMeta: { referenceLabel: reference.label, @@ -6943,6 +6995,10 @@ export async function POST(request: Request) { referenceImageUrl: reference.imageUrl, referenceGeometry, scenePlan: replacementExecutionPlan.scenePlan, + preserveReferenceOutsideSubject: + isDetailReplacementPreset && + detailReplacementStrategy === "product_only", + preserveReferenceText: isDetailReplacementPreset, uploadPrefix: isDetailReplacementPreset ? "agent-detail-replacement" : "agent-sku-replacement", @@ -8830,7 +8886,7 @@ export async function POST(request: Request) { .filter(Boolean) .join("\n\n"); - const activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor( + let activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor( resolvedRuntimeModelLookupId, ).catch(() => null); if (shouldRunAgentConversation && activeRuntimeModelDescriptor) { @@ -9635,6 +9691,11 @@ export async function POST(request: Request) { alibabaModelConfig?.fallbackModelId2, ].filter(Boolean) as string[]; const allModels = await getEnabledAiModelsWithProvider(); + const currentModelEntry = resolveModelEntryByRuntimeModelId( + allModels, + actualModelId, + finalModelId, + ); const fallbackCandidates = buildImageFallbackCandidates(allModels, { currentProviderId: alibabaModelConfig?.providerId, currentModelId: alibabaModelConfig?.id, @@ -9880,6 +9941,7 @@ export async function POST(request: Request) { let imageMarkdown = ""; let usedModelId = actualModelId; + const generationStartedAt = Date.now(); // 🆕 图片生成真正开始时发出标记（用于前端显示占位图） writeAssistantTextChunk( @@ -9906,14 +9968,43 @@ export async function POST(request: Request) { } catch (primaryError) { chatApiLogger.error("[Chat API] Alibaba primary model failed:", primaryError); - if (!shouldFailoverAiModelError(primaryError)) { + await recordAiModelHealthFailure({ + model: currentModelEntry?.model || alibabaModelConfig, + provider: currentModelEntry?.provider, + error: primaryError, + context: "chat_image_primary", + elapsedMs: Date.now() - generationStartedAt, + metadata: { + runtimeModelId: actualModelId, + finalModelId, + providerType: (languageModel as any).provider, + aspectRatio: effectiveImageGenerationParams.aspectRatio, + imageSize: effectiveImageGenerationParams.imageSize, + imageCount: + effectiveImageGenerationParams.imageCount || + defaultImageQualityParams.imageCount || + 1, + }, + }).catch((healthError) => { + chatApiLogger.warn("[Chat API] Record image primary health failure failed:", healthError); + }); + + if ( + !shouldFailoverAiModelError(primaryError) && + !shouldFailoverVip98kRequestRejection( + primaryError, + (languageModel as any).provider, + ) + ) { chatApiLogger.info("[Chat API] Alibaba primary error is not eligible for image failover; returning original error."); throw primaryError; } let fallbackSuccess = false; + const triedFallbackModelIds = new Set(); for (const candidate of fallbackCandidates) { try { + triedFallbackModelIds.add(String(candidate.model.id || "")); chatApiLogger.info(`[Chat API] 🔄 Alibaba fallback model: ${candidate.model.displayName} (${candidate.provider.type})`); const fallbackResult = await generateImageWithFailoverCandidate({ candidate, @@ -9938,6 +10029,22 @@ export async function POST(request: Request) { if (shouldCooldownImageFallbackModel(fallbackErrorMessage)) { setImageFallbackModelCooldown(candidate.model.id, fallbackErrorMessage); } + await recordAiModelHealthFailure({ + model: candidate.model, + provider: candidate.provider, + error: fallbackError, + context: "chat_image_fallback", + elapsedMs: Date.now() - generationStartedAt, + metadata: { + primaryModelId: actualModelId, + fallbackModelId: candidate.model.modelId, + providerType: candidate.provider.type, + aspectRatio: effectiveImageGenerationParams.aspectRatio, + imageSize: effectiveImageGenerationParams.imageSize, + }, + }).catch((healthError) => { + chatApiLogger.warn("[Chat API] Record image fallback health failure failed:", healthError); + }); chatApiLogger.error(`[Chat API] ❌ Alibaba fallback model ${candidate.model.displayName} (${candidate.provider.type}) failed:`, fallbackError); } } @@ -9966,10 +10073,11 @@ export async function POST(request: Request) { currentModelKey: actualModelId, currentProviderId: alibabaModelConfig?.providerId, providerPriority, - }); + }).filter((candidate) => !triedFallbackModelIds.has(String(candidate.model.id || ""))); for (const candidate of crossProviderCandidates) { try { + triedFallbackModelIds.add(String(candidate.model.id || "")); chatApiLogger.info(`[Chat API] 🔁 Cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`); if ( candidate.provider.type === "fal-image" && @@ -10053,6 +10161,22 @@ export async function POST(request: Request) { if (shouldCooldownImageFallbackModel(crossErrorMessage)) { setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage); } + await recordAiModelHealthFailure({ + model: candidate.model, + provider: candidate.provider, + error: crossError, + context: "chat_image_cross_fallback", + elapsedMs: Date.now() - generationStartedAt, + metadata: { + primaryModelId: actualModelId, + fallbackModelId: candidate.model.modelId, + providerType: candidate.provider.type, + aspectRatio: effectiveImageGenerationParams.aspectRatio, + imageSize: effectiveImageGenerationParams.imageSize, + }, + }).catch((healthError) => { + chatApiLogger.warn("[Chat API] Record image cross-provider health failure failed:", healthError); + }); chatApiLogger.error(`[Chat API] ❌ Cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`, crossError); } } @@ -10066,6 +10190,19 @@ export async function POST(request: Request) { chatApiLogger.info("[Chat API] Alibaba generated, length:", imageMarkdown.length); chatApiLogger.info("[Chat API] Alibaba used model:", usedModelId); + const usedModelEntry = resolveModelEntryByRuntimeModelId( + allModels, + usedModelId, + usedModelId === actualModelId ? finalModelId : undefined, + ); + await recordAiModelHealthSuccess({ + model: + usedModelEntry?.model || + (usedModelId === actualModelId ? alibabaModelConfig : null), + }).catch((healthError) => { + chatApiLogger.warn("[Chat API] Record image health success failed:", healthError); + }); + const usedModelDisplayName = resolveModelDisplayNameByModelId(allModels, usedModelId) || alibabaModelConfig?.displayName || @@ -11106,7 +11243,7 @@ export async function POST(request: Request) { requestHeaders: request.headers, requestUrl: request.url, defaultModelId: finalModelId, - destructiveConfirmed: isCanvasOperationConfirmText(requestTextContent), + authorizeDestructive: (nodeIds) => isExplicitDestructiveIntent(requestTextContent) && nodeIds.length > 0 && nodeIds.every((nodeId) => selectedCanvasImageNodeIds.includes(nodeId)), callbacks: { onClientCommand: (command, capability) => { writeCanvasCapabilityCommandChunk( @@ -12135,8 +12272,70 @@ export async function POST(request: Request) { const shouldDisableSmoothStream = resolvedModelSupportsImageGen || isVolcanoVideoModel; + let activeConversationModel = currentModelConfig; + let activeConversationProvider = currentProviderConfig; + const recordedAuthFailures = new Set(); + const recordConversationAuthFailure = async (error: unknown) => { + const key = activeConversationModel?.id; + if (!key || recordedAuthFailures.has(key)) return; + recordedAuthFailures.add(key); + recordProviderHealthFailure({ + providerKey: activeConversationProvider?.id, + providerType: activeConversationProvider?.type, + error, + }); + await recordAiModelHealthFailure({ + model: activeConversationModel, + provider: activeConversationProvider, + error, + context: "canvas_agent_conversation_auth", + }).catch((healthError) => chatApiLogger.warn("[Chat API] Could not persist model authentication failure:", healthError)); + }; + const needsVisionFallback = convertedMessages.some((item: any) => + Array.isArray(item.content) && item.content.some((part: any) => + part.type === "image" || (part.type === "file" && String(part.mediaType || "").startsWith("image/")) + ) + ); + const streamModel = shouldRunAgentConversation && !shouldKeepSelectedModel && + currentModelConfig?.excludeFromFailover !== true && + !resolvedModelSupportsImageGen && !isVolcanoVideoModel + ? withAuthenticationFailover({ + primary: languageModel as any, + onAuthenticationFailure: recordConversationAuthFailure, + resolveFallback: async () => { + const enabled = await getEnabledAiModelsWithProvider(false); + const preferredIds = [currentModelConfig?.fallbackModelId1, currentModelConfig?.fallbackModelId2]; + const candidates = enabled.filter(({ model, provider }) => + model.id !== activeConversationModel?.id && model.isEnabled && provider.isEnabled && + !model.excludeFromFailover && !model.isSubModel && model.healthStatus !== "auto_disabled" && + !isImageGenerationModelType(provider.type, model) && !isVideoProviderType(provider.type, model) && + (!needsVisionFallback || model.supportsVision) && + (isAiModelAgentAuto(model) || preferredIds.includes(model.id)) && + !shouldAvoidProvider(getProviderHealthSnapshot(provider.id)) + ).sort((a, b) => { + const rank = (id: string, isDefault: boolean) => { + const index = preferredIds.indexOf(id); + return index >= 0 ? index : isDefault ? 2 : 3; + }; + return rank(a.model.id, a.model.isDefault) - rank(b.model.id, b.model.isDefault); + }); + for (const candidate of candidates) { + const fallback = await getLanguageModelById(candidate.model.id); + if (!fallback || typeof fallback === "string") continue; + activeConversationModel = candidate.model; + activeConversationProvider = candidate.provider; + finalModelId = candidate.model.id; + actualModelId = candidate.model.modelId; + activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor(candidate.model.id).catch(() => null); + return fallback as any; + } + return null; + }, + }) + : languageModel; const result = streamText({ - model: languageModel, + model: streamModel, + maxRetries: shouldRunAgentConversation ? 0 : undefined, system: effectiveConversationSystemPrompt, messages: convertedMessages, stopWhen: stepCountIs(shouldUseCanvasCapabilityToolLoop ? 8 : 5), diff --git a/app/(chat)/api/files/direct-upload/route.ts b/app/(chat)/api/files/direct-upload/route.ts index bfdb9fa1f..2bbfd6842 100644 --- a/app/(chat)/api/files/direct-upload/route.ts +++ b/app/(chat)/api/files/direct-upload/route.ts @@ -13,6 +13,10 @@ import { z } from "zod"; import { auth } from "@/app/(auth)/auth"; import { upsertMediaAsset } from "@/lib/db/queries"; +import { + IMAGE_UPLOAD_MAX_BYTES, + IMAGE_UPLOAD_TOO_LARGE_MESSAGE, +} from "@/lib/image-upload-limits"; import { createLogger } from "@/lib/observability/logger"; import { buildS3PublicUrl, @@ -160,7 +164,10 @@ function isAllowedDirectUpload( ); } -function getPurposeMaxBytes(purpose?: string) { +function getPurposeMaxBytes(purpose?: string, contentType?: string) { + if (String(contentType || "").toLowerCase().startsWith("image/")) { + return IMAGE_UPLOAD_MAX_BYTES; + } if (purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE) { return OPENCLAW_CHAT_VIDEO_MAX_BYTES; } @@ -281,9 +288,14 @@ export async function POST(request: Request) { { status: 400 } ); } - if (payload.size > getPurposeMaxBytes(payload.purpose)) { + const maxBytes = getPurposeMaxBytes(payload.purpose, contentType); + if (payload.size > maxBytes) { return NextResponse.json( - { error: `文件不能超过 ${getPurposeMaxBytes(payload.purpose) / 1024 / 1024}MB` }, + { + error: contentType.startsWith("image/") + ? IMAGE_UPLOAD_TOO_LARGE_MESSAGE + : `文件不能超过 ${maxBytes / 1024 / 1024}MB`, + }, { status: 413 } ); } @@ -390,8 +402,15 @@ export async function POST(request: Request) { if (!isAllowedDirectUpload(contentType, payload.purpose, payload.filename)) { return NextResponse.json({ error: "不支持续传该文件类型" }, { status: 400 }); } - if (payload.size > getPurposeMaxBytes(payload.purpose)) { - return NextResponse.json({ error: "文件超过允许的上传大小" }, { status: 413 }); + if (payload.size > getPurposeMaxBytes(payload.purpose, contentType)) { + return NextResponse.json( + { + error: contentType.startsWith("image/") + ? IMAGE_UPLOAD_TOO_LARGE_MESSAGE + : "文件超过允许的上传大小", + }, + { status: 413 } + ); } const expectedPartSize = computePartSize(payload.size); if (payload.partSize !== expectedPartSize) { @@ -505,8 +524,15 @@ export async function POST(request: Request) { { status: 400 } ); } - if (payload.size > getPurposeMaxBytes(payload.purpose)) { - return NextResponse.json({ error: "文件超过允许的上传大小" }, { status: 413 }); + if (payload.size > getPurposeMaxBytes(payload.purpose, contentType)) { + return NextResponse.json( + { + error: contentType.startsWith("image/") + ? IMAGE_UPLOAD_TOO_LARGE_MESSAGE + : "文件超过允许的上传大小", + }, + { status: 413 } + ); } if (payload.uploadId) { diff --git a/app/(chat)/api/files/upload/route.ts b/app/(chat)/api/files/upload/route.ts index 26adc7335..1da734227 100644 --- a/app/(chat)/api/files/upload/route.ts +++ b/app/(chat)/api/files/upload/route.ts @@ -1,5 +1,9 @@ import { NextResponse } from "next/server"; import { z } from "zod"; +import { + IMAGE_UPLOAD_TOO_LARGE_MESSAGE, + isImageUploadTooLarge, +} from "@/lib/image-upload-limits"; import { auth } from "@/app/(auth)/auth"; import { createAgentWorkflowPreviewImageBuffer } from "@/lib/agents/workflow-preview-images"; @@ -260,6 +264,12 @@ export async function POST(request: Request) { const clientUploadId = String(formData.get("clientUploadId") || "").trim() || null; const isSvgUpload = contentType === "image/svg+xml"; + if (contentType.startsWith("image/") && isImageUploadTooLarge(file.size)) { + return NextResponse.json( + { error: IMAGE_UPLOAD_TOO_LARGE_MESSAGE }, + { status: 413 } + ); + } if ( isSvgUpload && (uploadPurpose !== "admin-svg-icon" || !isSiteAdminType(session.user.type)) diff --git a/app/(chat)/api/models/route.ts b/app/(chat)/api/models/route.ts index c1ada40aa..131016b6e 100644 --- a/app/(chat)/api/models/route.ts +++ b/app/(chat)/api/models/route.ts @@ -102,8 +102,8 @@ export async function GET(request: Request) { }, { id: "local-e2e-video-model", - modelId: "kling-3-local", - displayName: "可灵 3.0", + modelId: "veo-3.1-fast", + displayName: "Veo 3.1 Fast（本地布局测试）", avatarUrl: null, description: "本地视频画布测试视频模型", supportsVision: true, @@ -111,18 +111,18 @@ export async function GET(request: Request) { isMultimodal: true, isDefault: false, providerName: "Local E2E", - providerType: "local", + providerType: "vip98k", isSubModel: false, parentModelId: null, - capabilities: { video: true }, + capabilities: { kind: "video", supportedModes: ["text-to-video", "first-frame", "first-last-frame", "reference-images"], supportedRatios:["16:9"], supportedResolutions:["720p","1080p"], durationOptions:[8], maxReferenceImages:3 }, videoToolCapability: { supported: true, - durations: [5, 10], + durations: [8], resolutions: ["720p", "1080p"], }, imageGenConfig: { creditCostPerVideo: 20, - videoDurationOptions: [5, 10], + videoDurationOptions: [8], }, sortOrder: 1, selectable: true, diff --git a/app/(chat)/flow-board/layout.tsx b/app/(chat)/flow-board/layout.tsx index 2e0557122..0bcc5079b 100644 --- a/app/(chat)/flow-board/layout.tsx +++ b/app/(chat)/flow-board/layout.tsx @@ -3,10 +3,5 @@ export default function FlowBoardLayout({ }: { children: React.ReactNode; }) { - return ( - <> - - {children} - - ); + return <>{children}; } diff --git a/app/(chat)/layout.tsx b/app/(chat)/layout.tsx index 646705092..1fdb87d78 100644 --- a/app/(chat)/layout.tsx +++ b/app/(chat)/layout.tsx @@ -1,5 +1,6 @@ import { Suspense } from "react"; import { ChatShell } from "@/components/chat-shell"; +import { CanvasLogoLoadingView } from "@/components/canvas/canvas-logo-loading"; import { DataStreamProvider } from "@/components/data-stream-provider"; import { auth } from "../(auth)/auth"; import { @@ -8,10 +9,14 @@ import { getTutorialVideos, } from "@/lib/db/queries"; +export const dynamic = "force-dynamic"; + export default function Layout({ children }: { children: React.ReactNode }) { return ( - }> + } + > {children} diff --git a/app/(chat)/video-canvas/page.tsx b/app/(chat)/video-canvas/page.tsx index dfffb5b4c..de9767554 100644 --- a/app/(chat)/video-canvas/page.tsx +++ b/app/(chat)/video-canvas/page.tsx @@ -1,79 +1,75 @@ -import { randomUUID } from "node:crypto"; import { redirect } from "next/navigation"; -import { auth } from "@/app/(auth)/auth"; -import { db } from "@/lib/db"; -import { canvasProject } from "@/lib/db/schema"; import { and, desc, eq } from "drizzle-orm"; +import { auth } from "@/app/(auth)/auth"; +import { VideoCanvasHome, type VideoCanvasHomeProject } from "@/components/canvas/VideoCanvasHome"; +import { db } from "@/lib/db"; +import { getSiteSettings } from "@/lib/db/queries"; +import { canvasProject } from "@/lib/db/schema"; -type VideoCanvasRedirectPageProps = { - searchParams?: Promise>; -}; +type Props = { searchParams?: Promise> }; -function getSearchParam( - params: Record, - key: string -) { - const value = params[key]; +function first(value: string | string[] | undefined) { return Array.isArray(value) ? value[0] : value; } -function buildVideoCanvasUrl( - canvasId: string, - params: Record -) { - const query = new URLSearchParams(); - query.set("mode", "video"); - query.set("workspaceType", "video"); - - for (const [key, value] of Object.entries(params)) { - if ( - key === "new" || - key === "mode" || - key === "workspaceType" || - value === undefined - ) { - continue; - } - if (Array.isArray(value)) { - value.forEach((item) => query.append(key, item)); - continue; - } - query.set(key, value); +function canvasHref(id: string, incoming: Record) { + const params = new URLSearchParams({ mode: "video", workspaceType: "video" }); + for (const [key, value] of Object.entries(incoming)) { + if (["new", "mode", "workspaceType"].includes(key) || value === undefined) continue; + for (const item of Array.isArray(value) ? value : [value]) params.append(key, item); } - - return `/canvas/${canvasId}?${query.toString()}`; + return `/canvas/${id}?${params.toString()}`; } -async function getLatestVideoCanvasId(userId: string) { - const [latestProject] = await db - .select({ id: canvasProject.id }) - .from(canvasProject) - .where( - and( - eq(canvasProject.userId, userId), - eq(canvasProject.workspaceType, "video") - ) - ) - .orderBy(desc(canvasProject.updatedAt)) - .limit(1); - - return latestProject?.id || null; +function projectPreview(thumbnail: string | null, canvasData: unknown) { + const nodes = canvasData && typeof canvasData === "object" && "nodes" in canvasData + ? (canvasData as { nodes?: unknown }).nodes : null; + const candidates = [thumbnail]; + if (Array.isArray(nodes)) for (const node of [...nodes].reverse()) { + if (!node || typeof node !== "object") continue; + const item = node as { inputType?: string; draftValues?: Record }; + const draft = item.draftValues || {}; + candidates.push( + String(draft.poster || draft.previewImageUrl || draft.thumbnailUrl || draft.resultImage || (item.inputType === "image" ? draft.mediaUrl || (Array.isArray(draft.url) ? draft.url[0] : "") : "") || ""), + ); + } + return candidates.find(value => typeof value === "string" && (value.startsWith("https://") || value.startsWith("/"))) || null; } -export default async function VideoCanvasRedirectPage({ - searchParams, -}: VideoCanvasRedirectPageProps) { - const params = (await searchParams) || {}; - const shouldCreateNew = getSearchParam(params, "new") === "1"; - - if (shouldCreateNew) { - redirect(buildVideoCanvasUrl(randomUUID(), params)); - } - +export default async function VideoCanvasHomePage({ searchParams }: Props) { const session = await auth(); - const latestCanvasId = session?.user?.id - ? await getLatestVideoCanvasId(session.user.id).catch(() => null) - : null; + if (!session?.user?.id) redirect("/login?callbackUrl=%2Fvideo-canvas"); - redirect(buildVideoCanvasUrl(latestCanvasId || randomUUID(), params)); + const params = (await searchParams) || {}; + if (first(params.new) === "1") { + const [created] = await db.insert(canvasProject).values({ + userId: session.user.id, + title: "未命名视频画布", + workspaceType: "video", + canvasData: { nodes: [], edges: [], viewport: { x: 0, y: 0, zoom: 1 } }, + }).returning({ id: canvasProject.id }); + redirect(canvasHref(created.id, params)); + } + + const [rows, siteSettings] = await Promise.all([db.select({ + id: canvasProject.id, + title: canvasProject.title, + thumbnail: canvasProject.thumbnail, + canvasData: canvasProject.canvasData, + createdAt: canvasProject.createdAt, + updatedAt: canvasProject.updatedAt, + }).from(canvasProject).where(and( + eq(canvasProject.userId, session.user.id), + eq(canvasProject.workspaceType, "video"), + )).orderBy(desc(canvasProject.updatedAt)).limit(25), getSiteSettings()]); + + const projects: VideoCanvasHomeProject[] = rows.slice(0, 24).map(row => ({ + id: row.id, + title: row.title, + thumbnail: projectPreview(row.thumbnail, row.canvasData), + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + })); + + return 24} initialProjects={projects} siteLogo={siteSettings.siteLogo} siteLogoDark={siteSettings.siteLogoDark} siteName={siteSettings.siteName} userName={session.user.name || ""} />; } diff --git a/app/api/admin/bananaimage-models/route.ts b/app/api/admin/bananaimage-models/route.ts index adcfdb079..1076af550 100644 --- a/app/api/admin/bananaimage-models/route.ts +++ b/app/api/admin/bananaimage-models/route.ts @@ -94,6 +94,8 @@ export async function POST(request: Request) { providerId, modelId: preset.modelId, ...values, + isEnabled: true, + isHidden: false, isDefault: false, isSubModel: false, parentModelId: null, diff --git a/app/api/admin/snapgen-image-models/route.ts b/app/api/admin/snapgen-image-models/route.ts index 7658324ae..b0cd744bc 100644 --- a/app/api/admin/snapgen-image-models/route.ts +++ b/app/api/admin/snapgen-image-models/route.ts @@ -76,8 +76,6 @@ export async function POST(request: Request) { description: preset.description, supportsVision: preset.supportsVision, supportsImageGen: preset.supportsImageGen, - isEnabled: true, - isHidden: false, sortOrder: preset.sortOrder, capabilities: preset.capabilities || null, defaultParams: preset.defaultParams || null, diff --git a/app/api/agents/detail-replacement/batches/route.ts b/app/api/agents/detail-replacement/batches/route.ts new file mode 100644 index 000000000..cce8997aa --- /dev/null +++ b/app/api/agents/detail-replacement/batches/route.ts @@ -0,0 +1,295 @@ +import { and, desc, eq, inArray, sql } from "drizzle-orm"; +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { db } from "@/lib/db"; +import { canvasImageTask, canvasProject, message } from "@/lib/db/schema"; +import { extractAgentProjectStateFromCanvasData, isAgentProjectRow } from "@/lib/agents/agent-project-compat"; +import { + createCanvasImageTask, + enqueueCanvasImageTask, + ensureCanvasImageTaskActive, + getCanvasImageTaskForUser, +} from "@/lib/canvas/image-task-queue"; +import { + checkUserImageQuota, + getChatById, + getEnabledAiModelsWithProvider, + getPointsSystemConfig, + getUserById, + saveChat, +} from "@/lib/db/queries"; +import { getModelGenerationCreditCost } from "@/lib/ai/image-model-pricing"; +import { resolveDirectReplacementModelRef } from "@/app/(chat)/api/chat/_lib/detail-replacement"; +import { + buildAgentSlotPrompt, + normalizeAgentSupplementPrompt, +} from "@/app/(chat)/api/chat/_lib/agent-scene-workflow"; +import { + detailReplacementSubmissionKey, + isDetailReplacementBatchRequest, + isDetailReplacementUuid, + type DetailReplacementBatchReference, + type DetailReplacementBatchRequest, + type DetailReplacementBatchResult, +} from "@/lib/agents/detail-replacement-batch"; + +export const maxDuration = 1200; + +function scopedChatId(projectId: string) { + const normalized = projectId.toLowerCase().replace(/-/g, ""); + const scoped = `${normalized.slice(0, 20)}a93e17a60001`; + return [scoped.slice(0, 8), scoped.slice(8, 12), scoped.slice(12, 16), scoped.slice(16, 20), scoped.slice(20, 32)].join("-"); +} + +function taskResponse(task: Awaited>) { + const result = task?.result as DetailReplacementBatchResult | null; + const request = task?.request as DetailReplacementBatchRequest | null; + return { + id: task?.id, + status: task?.status, + attempts: task?.attempts, + error: task?.error || null, + cards: Array.isArray(result?.cards) ? result.cards : [], + currentCardId: result?.currentCardId || null, + requestedCardIds: request?.references?.map((item) => item.id) || [], + createdAt: task?.createdAt || null, + queueStatus: task?.queueStatus || null, + }; +} + +async function findActiveBatchId(params: { + userId: string; + projectId: string; + submissionKey: string; +}) { + const [row] = await db.select({ id: canvasImageTask.id }) + .from(canvasImageTask) + .where(and( + eq(canvasImageTask.userId, params.userId), + inArray(canvasImageTask.status, ["pending", "running"]), + sql`${canvasImageTask.request}->>'taskKind' = 'agent-detail-replacement-batch'`, + sql`${canvasImageTask.request}->>'projectId' = ${params.projectId}`, + sql`${canvasImageTask.request}->>'submissionKey' = ${params.submissionKey}`, + )) + .orderBy(desc(canvasImageTask.createdAt)).limit(1); + return row?.id || null; +} + +function isUniqueViolation(error: unknown) { + for (let cursor: unknown = error; cursor && typeof cursor === "object"; ) { + if ((cursor as { code?: unknown }).code === "23505") return true; + cursor = (cursor as { cause?: unknown }).cause; + } + return false; +} + +export async function GET(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + const params = new URL(request.url).searchParams; + const projectId = String(params.get("projectId") || "").trim(); + const taskId = String(params.get("taskId") || "").trim(); + if (!isDetailReplacementUuid(projectId)) { + return NextResponse.json({ error: "项目编号无效" }, { status: 400 }); + } + const [project] = await db.select({ id: canvasProject.id }).from(canvasProject).where(and( + eq(canvasProject.id, projectId), eq(canvasProject.userId, session.user.id), + )).limit(1); + if (!project) return NextResponse.json({ error: "项目不存在" }, { status: 404 }); + + const [candidate] = await db.select({ id: canvasImageTask.id, request: canvasImageTask.request }) + .from(canvasImageTask) + .where(and( + eq(canvasImageTask.userId, session.user.id), + ...(isDetailReplacementUuid(taskId) ? [eq(canvasImageTask.id, taskId)] : []), + sql`${canvasImageTask.request}->>'taskKind' = 'agent-detail-replacement-batch'`, + sql`${canvasImageTask.request}->>'projectId' = ${projectId}`, + )) + // A batch that is still working matters more than a newer finished one, so the + // workspace keeps showing live progress after a rerun of a single card. + .orderBy( + sql`case when ${canvasImageTask.status} in ('pending', 'running') then 0 else 1 end`, + desc(canvasImageTask.createdAt), + ).limit(1); + if (!candidate || !isDetailReplacementBatchRequest(candidate.request)) { + return NextResponse.json({ task: null }); + } + let task = await getCanvasImageTaskForUser({ taskId: candidate.id, userId: session.user.id }); + if (task && (task.status === "failed" || + (task.status === "running" && Date.now() - task.updatedAt.getTime() > 90_000))) { + await ensureCanvasImageTaskActive({ taskId: task.id, userId: session.user.id }); + task = await getCanvasImageTaskForUser({ taskId: candidate.id, userId: session.user.id }); + } + return NextResponse.json({ task: taskResponse(task) }); +} + +export async function POST(request: Request) { + try { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + const body = await request.json(); + const projectId = String(body.projectId || "").trim(); + const taskId = String(body.requestId || "").trim(); + if (!isDetailReplacementUuid(projectId) || !isDetailReplacementUuid(taskId)) { + return NextResponse.json({ error: "任务编号或项目编号无效" }, { status: 400 }); + } + const [project] = await db.select().from(canvasProject).where(and( + eq(canvasProject.id, projectId), eq(canvasProject.userId, session.user.id), + )).limit(1); + if (!project || !isAgentProjectRow(project)) { + return NextResponse.json({ error: "风格化项目不存在" }, { status: 404 }); + } + + const state = extractAgentProjectStateFromCanvasData(project.canvasData); + const knownReferences = new Map( + (state.workflowState.agentModeState?.detailReplacement?.references || []) + .map((item) => [String(item.id), String(item.imageUrl)]), + ); + const requested = Array.isArray(body.references) ? body.references : []; + const references: DetailReplacementBatchReference[] = requested.map((raw: unknown) => { + const item = raw && typeof raw === "object" ? raw as Record : {}; + return { + id: String(item.id || "").slice(0, 128), + label: String(item.label || "详情参考图").slice(0, 255), + imageUrl: String(item.imageUrl || "").trim(), + candidateId: String(item.candidateId || "").slice(0, 128) || null, + passthrough: item.passthrough === true, + rerunReferenceImages: Array.isArray(item.rerunReferenceImages) + ? item.rerunReferenceImages.slice(0, 6).map((entry: { url?: string }) => ({ url: String(entry?.url || "") })) + : [], + }; + }); + if ( + references.length === 0 || references.length > 40 || + new Set(references.map((item) => item.id)).size !== references.length || + references.some((item) => !item.id || knownReferences.get(item.id) !== item.imageUrl) + ) { + return NextResponse.json({ error: "详情参考图尚未保存或已发生变化，请刷新后重试" }, { status: 400 }); + } + const productImageUrls = Array.isArray(body.productImageUrls) + ? Array.from(new Set(body.productImageUrls.map((url: unknown) => String(url || "").trim()) + .filter((url: string) => /^https?:\/\//i.test(url)))).slice(0, 16) as string[] + : []; + if (productImageUrls.length === 0 && !references.every((item) => item.passthrough)) { + return NextResponse.json({ error: "请先在输入图区选择至少一张产品图" }, { status: 400 }); + } + const strategy = body.replacementStrategy === "product_only" ? "product_only" : "scene_replace"; + const prompt = String(body.prompt || "").trim().slice(0, 24_000); + if (strategy !== "product_only" && !prompt) { + return NextResponse.json({ error: "请先配置风格化提示词" }, { status: 400 }); + } + const models = await getEnabledAiModelsWithProvider(); + const model = await resolveDirectReplacementModelRef({ + selectedChatModel: String(body.model || "auto"), + finalModelId: String(body.model || "auto"), + models, + }); + if (!model || model === "auto" || !models.some((entry) => entry.model.id === model)) { + return NextResponse.json({ error: "详情替换模型不可用，请检查模型配置" }, { status: 400 }); + } + const quota = await checkUserImageQuota(session.user.id); + if (!quota.allowed) { + return NextResponse.json({ error: "本月图片生成配额已用完" }, { status: 429 }); + } + const requestedImageCount = references.filter((item) => !item.passthrough).length; + if (quota.limit != null && quota.used + requestedImageCount > quota.limit) { + return NextResponse.json({ error: `本月剩余生图配额不足 ${requestedImageCount} 张` }, { status: 429 }); + } + const pointsConfig = await getPointsSystemConfig(); + if (pointsConfig.enabled && pointsConfig.blockOnInsufficient !== false) { + const selectedModel = models.find((entry) => entry.model.id === model)?.model; + const unitCost = getModelGenerationCreditCost( + selectedModel as Parameters[0], "image", { + imageSize: ["1K", "2K", "4K"].includes(body.imageSize) ? body.imageSize : "2K", + }); + const user = await getUserById({ id: session.user.id }); + const expectedCost = unitCost * requestedImageCount; + if (!user?.isHighVip && expectedCost > Number(user?.pointsBalance || 0)) { + return NextResponse.json({ error: `积分不足，本批次预计消耗 ${expectedCost} 积分` }, { status: 402 }); + } + } + const chatId = scopedChatId(projectId); + const existingChat = await getChatById({ id: chatId }); + if (existingChat && existingChat.userId !== session.user.id) { + return NextResponse.json({ error: "无权访问项目对话" }, { status: 403 }); + } + if (!existingChat) await saveChat({ + id: chatId, userId: session.user.id, title: project.title || "详情替换", visibility: "private", + }); + const taskRequest: DetailReplacementBatchRequest = { + taskKind: "agent-detail-replacement-batch", + projectId, + chatId, + model, + prompt: strategy === "product_only" + ? "仅替换参考详情图中的产品主体" + : buildAgentSlotPrompt({ + workflowPrompt: prompt, + slotPrompt: "", + userSupplementPrompt: normalizeAgentSupplementPrompt( + String(body.userPrompt || ""), "详情参考图替换", + ), + includeSlotTitle: false, + modeType: "detail_replacement", + }), + userPrompt: String(body.userPrompt || "").slice(0, 12_000), + imageSize: ["1K", "2K", "4K"].includes(body.imageSize) ? body.imageSize : "2K", + replacementStrategy: strategy, + productImageUrls, + modelImageUrl: /^https?:\/\//i.test(String(body.modelImageUrl || "")) + ? String(body.modelImageUrl) : null, + references, + }; + const submissionKey = detailReplacementSubmissionKey(taskRequest); + taskRequest.submissionKey = submissionKey; + + const duplicateId = await findActiveBatchId({ + userId: session.user.id, projectId, submissionKey, + }); + if (duplicateId) { + await ensureCanvasImageTaskActive({ taskId: duplicateId, userId: session.user.id }); + const duplicate = await getCanvasImageTaskForUser({ taskId: duplicateId, userId: session.user.id }); + return NextResponse.json({ task: taskResponse(duplicate) }); + } + + let task: Awaited>; + try { + task = await createCanvasImageTask({ + taskId, userId: session.user.id, request: taskRequest as unknown as Record, + }); + } catch (error) { + if (!isUniqueViolation(error)) throw error; + // Two identical submissions raced past the lookup above; the database + // unique index kept one of them, so answer with that batch. + const racedId = await findActiveBatchId({ + userId: session.user.id, projectId, submissionKey, + }); + if (!racedId) throw error; + const raced = await getCanvasImageTaskForUser({ taskId: racedId, userId: session.user.id }); + return NextResponse.json({ task: taskResponse(raced) }); + } + const existingTask = await getCanvasImageTaskForUser({ taskId: task.id, userId: session.user.id }); + const storedRequest = existingTask?.request as DetailReplacementBatchRequest | null; + if ( + !isDetailReplacementBatchRequest(storedRequest) || + storedRequest.projectId !== projectId || + (storedRequest.submissionKey && storedRequest.submissionKey !== submissionKey) + ) { + return NextResponse.json({ error: "任务编号已被其他请求使用" }, { status: 409 }); + } + await db.insert(message).values({ + id: taskId, + chatId, + role: "user", + parts: [{ type: "text", text: taskRequest.userPrompt || `批量替换 ${references.length} 张详情图` }], + attachments: [], + createdAt: new Date(), + }).onConflictDoNothing({ target: message.id }); + await enqueueCanvasImageTask(task.id); + const current = await getCanvasImageTaskForUser({ taskId, userId: session.user.id }); + return NextResponse.json({ task: taskResponse(current) }); + } catch (error) { + console.error("[Detail Replacement Batch] Failed to start", error); + return NextResponse.json({ error: error instanceof Error ? error.message : "提交详情替换任务失败" }, { status: 500 }); + } +} diff --git a/app/api/angle-edit/route.ts b/app/api/angle-edit/route.ts index 403c2a204..6a4ab34b0 100644 --- a/app/api/angle-edit/route.ts +++ b/app/api/angle-edit/route.ts @@ -1,385 +1,10 @@ -import { NextRequest, NextResponse } from "next/server"; -import { - createMediaAssetRelation, - getMediaAssetsByUserAndUrls, - getPointsSystemConfig, - getSystemSetting, - upsertMediaAsset, -} from "@/lib/db/queries"; +import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; -import { uploadFile } from "@/lib/storage/service"; -import { db } from "@/lib/db"; -import { generatedImage } from "@/lib/db/schema"; -import { createLogger } from "@/lib/observability/logger"; -import { getImageToolCreditCost } from "@/lib/points/config"; -import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; - -const logger = createLogger("angle-edit-api"); - -// 角度编辑API配置类型 -type AngleEditConfig = { - enabled?: boolean; - falApiKey?: string; - loraScale?: number; - inferenceSteps?: number; - guidanceScale?: number; - maxImageSize?: number; - timeout?: number; -}; - -// 设置API路由最大执行时间为3分钟（180秒） +import { executeAngleEditPOST } from "@/lib/canvas/operations/angle-edit"; +export const runtime = "nodejs"; export const maxDuration = 180; - -/** - * 压缩图片生成缩略图 - */ -async function createThumbnail(buffer: Buffer): Promise { - try { - const sharp = await import("sharp"); - - const thumbnailBuffer = await sharp.default(buffer) - .resize(500, null, { - fit: "inside", - withoutEnlargement: true, - }) - .jpeg({ quality: 80 }) - .toBuffer(); - - logger.debug("thumbnail_created", { - originalBytes: buffer.length, - thumbnailBytes: thumbnailBuffer.length, - }); - return thumbnailBuffer; - } catch (error) { - logger.warn("thumbnail_creation_failed", error); - return buffer; - } -} - -/** - * 将 base64 图片上传到 S3 - */ -async function uploadBase64ToS3(base64Data: string, prefix: string = "angle-edit"): Promise<{ thumbnail: string; original: string }> { - try { - const buffer = Buffer.from(base64Data, "base64"); - const timestamp = Date.now(); - const randomStr = Math.random().toString(36).slice(2, 8); - const thumbnailFilename = `${prefix}-thumb-${timestamp}-${randomStr}.png`; - const originalFilename = `${prefix}-orig-${timestamp}-${randomStr}.png`; - - logger.debug("source_image_prepared", { originalBytes: buffer.length }); - - const thumbnailBuffer = await createThumbnail(buffer); - - const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( - thumbnailBuffer.byteOffset, - thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength - ) as ArrayBuffer; - const originalArrayBuffer = buffer.buffer.slice( - buffer.byteOffset, - buffer.byteOffset + buffer.byteLength - ) as ArrayBuffer; - - logger.debug("storage_upload_started", { - hasOriginal: originalArrayBuffer.byteLength > 0, - hasThumbnail: thumbnailArrayBuffer.byteLength > 0, - prefix, - }); - const [thumbnailResult, originalResult] = await Promise.all([ - uploadFile(thumbnailFilename, thumbnailArrayBuffer), - uploadFile(originalFilename, originalArrayBuffer), - ]); - - logger.info("storage_upload_completed", { - originalUrl: originalResult.url, - thumbnailUrl: thumbnailResult.url, - }); - - return { - thumbnail: thumbnailResult.url, - original: originalResult.url, - }; - } catch (error) { - logger.error("storage_upload_failed", error); - throw new Error(`图片上传失败: ${error instanceof Error ? error.message : "未知错误"}`); - } -} - -export async function POST(request: NextRequest) { - const startTime = Date.now(); - - try { - // 1. 验证用户登录 - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json({ error: "未登录" }, { status: 401 }); - } - - logger.info("request_received", { authenticated: true }); - - // 2. 从数据库获取配置 - const angleEditConfig = await getSystemSetting({ key: "angle_edit_config" }); - const config = (angleEditConfig?.value as AngleEditConfig) || {}; - - // 验证功能是否启用 - if (!config.enabled) { - return NextResponse.json( - { error: "角度编辑功能未启用，请联系管理员在系统设置中开启" }, - { status: 400 } - ); - } - - // 验证 API Key - if (!config.falApiKey) { - return NextResponse.json( - { error: "fal.ai API Key 未配置，请联系管理员配置" }, - { status: 500 } - ); - } - - logger.debug("config_loaded", { - enabled: Boolean(config.enabled), - hasApiKey: Boolean(config.falApiKey), - loraScale: config.loraScale || 1, - }); - - // 3. 解析请求 - const body = await request.json(); - const { - image_url, - image_base64, - // 数值参数 (官方 API 格式) - horizontal_angle, // 0-360 度 - vertical_angle, // -30 到 90 度 - zoom, // 0-10 - lora_scale, - } = body; - - if (!image_url && !image_base64) { - return NextResponse.json({ error: "缺少图片数据" }, { status: 400 }); - } - - // 使用请求中的参数或配置中的默认值 - const finalLoraScale = lora_scale ?? config.loraScale ?? 1; - const finalHorizontalAngle = horizontal_angle ?? 0; - const finalVerticalAngle = vertical_angle ?? 0; - const finalZoom = zoom ?? 5; - - logger.info("request_parameters_normalized", { - hasBase64Input: Boolean(image_base64), - hasUrlInput: Boolean(image_url), - horizontalAngle: finalHorizontalAngle, - loraScale: finalLoraScale, - verticalAngle: finalVerticalAngle, - zoom: finalZoom, - }); - - const pointsConfig = await getPointsSystemConfig(); - const pointsCharge = await prepareFeaturePointsCharge({ - userId: session.user.id, - unitCost: getImageToolCreditCost(pointsConfig, "angle-edit"), - reason: "image_angle_edit", - modelId: "qwen-image-edit-2511-multiple-angles", - note: "多角度编辑消耗积分", - pointsConfig, - }); - if (!pointsCharge.ok) { - return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); - } - - // 4. 准备图片输入 - let imageInput: string; - if (image_url) { - imageInput = image_url; - } else { - const mimeType = image_base64.startsWith("/9j/") ? "image/jpeg" : "image/png"; - imageInput = `data:${mimeType};base64,${image_base64}`; - } - - // 5. 动态导入 fal.ai SDK 并调用官方多角度 API - logger.info("provider_call_started", { - modelId: "fal-ai/qwen-image-edit-2511-multiple-angles", - }); - - const fal = await import("@fal-ai/serverless-client"); - - fal.config({ - credentials: config.falApiKey, - }); - - // 使用官方多角度 API 端点 - // 文档: https://fal.ai/models/fal-ai/qwen-image-edit-2511-multiple-angles - const result = await fal.subscribe("fal-ai/qwen-image-edit-2511-multiple-angles", { - input: { - image_urls: [imageInput], // 官方使用 image_urls (数组) - horizontal_angle: finalHorizontalAngle, // 水平旋转角度 0-360 - vertical_angle: finalVerticalAngle, // 垂直角度 -30 到 90 - zoom: finalZoom, // 缩放 0-10 - lora_scale: finalLoraScale, // LoRA 强度 - num_inference_steps: config.inferenceSteps || 28, - guidance_scale: config.guidanceScale || 4.5, // 官方默认 4.5 - enable_safety_checker: true, - output_format: "png", - }, - logs: true, - onQueueUpdate: (update) => { - if (update.status === "IN_PROGRESS") { - logger.debug("provider_progress", { status: update.status }); - } - }, - }); - - // 6. 处理结果 - const apiResult = result as { images?: Array<{ url?: string }>; prompt?: string }; - const outputImage = apiResult.images?.[0]; - if (!outputImage?.url) { - throw new Error("API 未返回有效图片"); - } - - const outputImageUrl = outputImage.url; - logger.info("provider_call_completed", { - hasGeneratedPrompt: Boolean(apiResult.prompt), - imageCount: apiResult.images?.length ?? 0, - outputImageUrl, - }); - - // 7. 下载图片并上传到我们的 S3 - const imageResponse = await fetch(outputImageUrl); - if (!imageResponse.ok) { - throw new Error(`下载输出图片失败: ${imageResponse.status}`); - } - const imageBuffer = Buffer.from(await imageResponse.arrayBuffer()); - const imageBase64Result = imageBuffer.toString("base64"); - - const s3Result = await uploadBase64ToS3(imageBase64Result, "angle-edit"); - logger.info("result_image_persisted", { - originalUrl: s3Result.original, - thumbnailUrl: s3Result.thumbnail, - }); - - // 8. 保存到数据库 - const processingTime = (Date.now() - startTime) / 1000; - - const [savedImage] = await db - .insert(generatedImage) - .values({ - userId: session.user.id, - thumbnailUrl: s3Result.thumbnail, - originalUrl: s3Result.original, - prompt: `角度编辑: H${finalHorizontalAngle}° V${finalVerticalAngle}° Z${finalZoom}`, - modelId: "qwen-image-edit-2511-multiple-angles", - createdAt: new Date(), - }) - .returning(); - - if (pointsCharge.chargeable) { - try { - await pointsCharge.consume({ - note: `多角度编辑消耗积分（${savedImage.id}）`, - }); - } catch (pointsError) { - logger.error("points_consume_failed", pointsError, { - imageId: savedImage.id, - userId: session.user.id, - }); - } - } - - try { - const sourceAssets = - typeof image_url === "string" && image_url.trim() - ? await getMediaAssetsByUserAndUrls({ - userId: session.user.id, - urls: [image_url], - }) - : []; - const media = await upsertMediaAsset({ - userId: session.user.id, - mediaType: "image", - sourceType: "edited", - status: "ready", - originalUrl: s3Result.original, - previewUrl: s3Result.original, - thumbnailUrl: s3Result.thumbnail, - displayName: `角度编辑 ${finalHorizontalAngle}/${finalVerticalAngle}`.slice(0, 80), - prompt: `角度编辑: H${finalHorizontalAngle}° V${finalVerticalAngle}° Z${finalZoom}`, - modelId: "qwen-image-edit-2511-multiple-angles", - sourceTable: "GeneratedImage", - sourceRowId: savedImage.id, - primaryParentAssetId: sourceAssets[0]?.id ?? null, - metadata: { - horizontalAngle: finalHorizontalAngle, - verticalAngle: finalVerticalAngle, - zoom: finalZoom, - }, - }); - if (media) { - for (const sourceAsset of sourceAssets) { - await createMediaAssetRelation({ - userId: session.user.id, - fromAssetId: sourceAsset.id, - toAssetId: media.id, - relationType: "edited_from", - metadata: { - horizontalAngle: finalHorizontalAngle, - verticalAngle: finalVerticalAngle, - zoom: finalZoom, - }, - }); - } - } - } catch (assetError) { - logger.warn("media_asset_registration_failed", assetError); - } - - logger.info("request_completed", { - processingSeconds: Number(processingTime.toFixed(2)), - }); - - // 9. 返回结果 - return NextResponse.json({ - success: true, - image_url: s3Result.original, - thumbnail_url: s3Result.thumbnail, - image_base64: imageBase64Result, - processing_time: processingTime, - angles: { - horizontal_angle: finalHorizontalAngle, - vertical_angle: finalVerticalAngle, - zoom: finalZoom, - }, - prompt: apiResult.prompt, - }); - - } catch (error) { - logger.error("request_failed", error); - - // 输出详细的错误信息 - if (error && typeof error === "object" && "status" in error) { - const apiError = error as { status?: number; body?: unknown; message?: string }; - logger.warn("provider_error_summary", { - bodyType: typeof apiError.body, - hasBody: apiError.body != null, - message: apiError.message, - status: apiError.status, - }); - - if (apiError.status === 403) { - return NextResponse.json( - { - error: "API 访问被拒绝 (403 Forbidden)。请检查 API Key 是否正确，以及是否有权限访问该模型。", - details: apiError.body, - }, - { status: 403 } - ); - } - } - - return NextResponse.json( - { - error: error instanceof Error ? error.message : "角度编辑处理失败", - }, - { status: 500 } - ); - } +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + return executeAngleEditPOST(request, session.user.id); } diff --git a/app/api/canvas/3d/model-proxy/route.ts b/app/api/canvas/3d/model-proxy/route.ts index f7be306e6..c9a0db29a 100644 --- a/app/api/canvas/3d/model-proxy/route.ts +++ b/app/api/canvas/3d/model-proxy/route.ts @@ -14,6 +14,7 @@ const ALLOWED_MODEL_HOST_DOMAINS = [ "worldlabs.ai", "s3.1080p.hk", "img.yydd.ai", + "scqny.jiansun.vip", "blob.vercel-storage.com", "aliyuncs.com", "amazonaws.com", diff --git a/app/api/canvas/[id]/image-archives/route.ts b/app/api/canvas/[id]/image-archives/route.ts new file mode 100644 index 000000000..68df25d96 --- /dev/null +++ b/app/api/canvas/[id]/image-archives/route.ts @@ -0,0 +1,65 @@ +import { NextRequest, NextResponse } from "next/server"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { auth } from "@/app/(auth)/auth"; +import { db } from "@/lib/db"; +import { canvasProject, imageArchiveTask, generatedImage, canvasImageTask } from "@/lib/db/schema"; +import { imageArchiveSourceKey } from "@/lib/storage/image-archive-db"; +import { isGeneratedImageSource } from "@/lib/storage/image-archive-urls"; +import { scheduleImageArchiveWorker } from "@/lib/storage/image-archive-worker"; + +export async function POST(request: NextRequest, { params }: { params: Promise<{ id: string }> }) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + const { id } = await params; + const [project] = await db + .select({ id: canvasProject.id }) + .from(canvasProject) + .where(and(eq(canvasProject.id, id), eq(canvasProject.userId, session.user.id))) + .limit(1); + if (!project) return NextResponse.json({ error: "画布不存在" }, { status: 404 }); + const body = await request.json(); + if ( + !Array.isArray(body.urls) || + body.urls.length > 100 || + body.urls.some((url: unknown) => typeof url !== "string" || !isGeneratedImageSource(url)) + ) { + return NextResponse.json({ error: "图片地址格式错误" }, { status: 400 }); + } + const urls: string[] = [...new Set(body.urls)]; + if (!urls.length) return NextResponse.json({ replacements: {}, pending: [] }); + const source = imageArchiveTask.sourceUrl; + const rows = await db + .select({ + sourceUrl: imageArchiveTask.sourceUrl, + targetUrl: imageArchiveTask.targetUrl, + status: imageArchiveTask.status, + }) + .from(imageArchiveTask) + .where( + and( + inArray(imageArchiveTask.sourceUrl, urls.map(imageArchiveSourceKey)), + sql`( + exists (select 1 from ${generatedImage} where ${generatedImage.userId} = ${session.user.id} + and (${generatedImage.originalUrl} = ${source} or ${generatedImage.thumbnailUrl} = ${source} + or ${generatedImage.originalUrl} = ${imageArchiveTask.targetUrl} or ${generatedImage.thumbnailUrl} = ${imageArchiveTask.targetUrl})) + or exists (select 1 from ${canvasImageTask} where ${canvasImageTask.userId} = ${session.user.id} + and (position(${source} in ${canvasImageTask.result}::text) > 0 or position(${imageArchiveTask.targetUrl} in ${canvasImageTask.result}::text) > 0)) + or exists (select 1 from ${canvasProject} where ${canvasProject.id} = ${id} and ${canvasProject.userId} = ${session.user.id} + and (position(${source} in ${canvasProject.canvasData}::text) > 0 or position(${imageArchiveTask.targetUrl} in ${canvasProject.canvasData}::text) > 0)) + )`, + ), + ); + const byKey = new Map(rows.map((row) => [row.sourceUrl, row])); + const replacements: Record = {}; + const pending: string[] = []; + for (const url of urls) { + const row = byKey.get(imageArchiveSourceKey(url)); + if (row?.targetUrl) replacements[url] = row.targetUrl; + else if (row && row.status !== "abandoned") pending.push(url); + } + if (rows.some((row) => row.status === "pending")) scheduleImageArchiveWorker(true); + return NextResponse.json( + { replacements, pending }, + { headers: { "Cache-Control": "private, no-store" } }, + ); +} diff --git a/app/api/canvas/[id]/route.ts b/app/api/canvas/[id]/route.ts index bb8f5a7dc..f2e7338b8 100644 --- a/app/api/canvas/[id]/route.ts +++ b/app/api/canvas/[id]/route.ts @@ -3,6 +3,7 @@ import { auth } from '@/app/(auth)/auth'; import { db } from '@/lib/db'; import { canvasProject } from '@/lib/db/schema'; import { syncCanvasNodesForProject } from '@/lib/db/queries'; +import { resolveArchivedImageUrls } from '@/lib/storage/image-archive-db'; import { eq, and } from 'drizzle-orm'; import { buildStaleCanvasSaveConflictPayload, @@ -102,7 +103,7 @@ export async function GET( // 更新画布（支持 upsert：如果不存在则创建） export async function PUT( request: NextRequest, - { params }: { params: Promise<{ id: string }> } + { params }: { params: Promise<{ id: string }> }, ) { try { const { id } = await params; @@ -116,29 +117,34 @@ export async function PUT( const session = await auth(); if (!session?.user?.id) { - return NextResponse.json({ success: false, error: '未登录' }, { status: 401 }); + return NextResponse.json( + { success: false, error: "未登录" }, + { status: 401 }, + ); } - const { title, description, thumbnail, canvasData } = body; - const baseCanvasRevision = body?.baseCanvasUpdatedAt || body?.baseCanvasRevision; + const { title, description, canvasData } = body; + const thumbnail = await resolveArchivedImageUrls(body.thumbnail); + const baseCanvasRevision = + body?.baseCanvasUpdatedAt || body?.baseCanvasRevision; const workspaceType = body && Object.prototype.hasOwnProperty.call(body, "workspaceType") ? resolveWorkspaceType(body.workspaceType) : undefined; - // 先检查画布是否存在 - const [existingProject] = await db - .select() - .from(canvasProject) - .where( - and( - eq(canvasProject.id, id), - eq(canvasProject.userId, session.user.id) + return await db.transaction(async (tx) => { + const [existingProject] = await tx + .select() + .from(canvasProject) + .where( + and( + eq(canvasProject.id, id), + eq(canvasProject.userId, session.user.id), + ), ) - ); - - if (existingProject) { + .for("update"); if ( + existingProject && canvasData !== undefined && baseCanvasRevision && isStaleCanvasSaveBaseRevision({ @@ -149,101 +155,66 @@ export async function PUT( ) { return NextResponse.json( buildStaleCanvasSaveConflictPayload(existingProject.updatedAt), - { status: 409 } + { status: 409 }, ); } - + if (!existingProject) { + // The parent must exist before node/asset foreign keys are synchronized. + const [created] = await tx + .insert(canvasProject) + .values({ + id, + userId: session.user.id, + createdAt: new Date(), + updatedAt: new Date(), + title: title || getDefaultWorkspaceTitle(workspaceType || "canvas"), + workspaceType: workspaceType || "canvas", + canvasData: { + nodes: [], + edges: [], + viewport: { x: 0, y: 0, zoom: 1 }, + }, + }) + .onConflictDoNothing({ target: canvasProject.id }) + .returning(); + if (!created) + return NextResponse.json( + { success: false, error: "画布已被创建，请重新加载后保存" }, + { status: 409 }, + ); + } const syncedCanvasData = canvasData !== undefined - ? await syncCanvasNodesForProject({ - projectId: id, - userId: session.user.id, - canvasData, - }) + ? await syncCanvasNodesForProject( + { projectId: id, userId: session.user.id, canvasData }, + tx, + ) : undefined; - - // 更新现有画布 - const updateData: any = { updatedAt: new Date() }; + const updateData: any = { updatedAt: new Date(Math.max(Date.now(), (existingProject?.updatedAt.getTime() || 0) + 1)) }; if (title !== undefined) updateData.title = title; if (description !== undefined) updateData.description = description; if (thumbnail !== undefined) updateData.thumbnail = thumbnail; if (workspaceType !== undefined) updateData.workspaceType = workspaceType; - if (syncedCanvasData !== undefined) updateData.canvasData = syncedCanvasData; - - const [updatedProject] = await db + if (syncedCanvasData !== undefined) + updateData.canvasData = syncedCanvasData; + const [saved] = await tx .update(canvasProject) .set(updateData) .where( and( eq(canvasProject.id, id), - eq(canvasProject.userId, session.user.id) - ) + eq(canvasProject.userId, session.user.id), + ), ) .returning(); - - return NextResponse.json({ success: true, data: updatedProject }); - } else { - const syncedCanvasData = - canvasData !== undefined - ? await syncCanvasNodesForProject({ - projectId: id, - userId: session.user.id, - canvasData, - }) - : undefined; - - const nextWorkspaceType = workspaceType || "canvas"; - const nextCanvasData = syncedCanvasData || { - nodes: [], - edges: [], - viewport: { x: 0, y: 0, zoom: 1 }, - }; - const nextTitle = title || getDefaultWorkspaceTitle(nextWorkspaceType); - const insertData: any = { - id, // 使用指定的 ID - userId: session.user.id, - title: nextTitle, - workspaceType: nextWorkspaceType, - canvasData: nextCanvasData, - }; - const conflictUpdateData: any = { - title: nextTitle, - workspaceType: nextWorkspaceType, - canvasData: nextCanvasData, - updatedAt: new Date(), - }; - if (description !== undefined) { - insertData.description = description; - conflictUpdateData.description = description; - } - if (thumbnail !== undefined) { - insertData.thumbnail = thumbnail; - conflictUpdateData.thumbnail = thumbnail; - } - - // 创建新画布（使用指定 ID），并兼容同一用户并发创建同一画布的场景。 - const [newProject] = await db - .insert(canvasProject) - .values(insertData) - .onConflictDoUpdate({ - target: canvasProject.id, - set: conflictUpdateData, - where: eq(canvasProject.userId, session.user.id), - }) - .returning(); - - if (!newProject) { - return NextResponse.json( - { success: false, error: '画布ID已被占用' }, - { status: 409 } - ); - } - - return NextResponse.json({ success: true, data: newProject }); - } + return NextResponse.json({ success: true, data: saved }); + }); } catch (error) { - console.error('更新/创建画布失败:', error); - return NextResponse.json({ success: false, error: '更新画布失败' }, { status: 500 }); + console.error("更新/创建画布失败:", error); + return NextResponse.json( + { success: false, error: "更新画布失败" }, + { status: 500 }, + ); } } diff --git a/app/api/canvas/[id]/tvstar-sessions/route.ts b/app/api/canvas/[id]/tvstar-sessions/route.ts new file mode 100644 index 000000000..751fb9741 --- /dev/null +++ b/app/api/canvas/[id]/tvstar-sessions/route.ts @@ -0,0 +1,57 @@ +import { and, desc, eq, lte } from "drizzle-orm"; +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { auth } from "@/app/(auth)/auth"; +import { db } from "@/lib/db"; +import { tvstarSession } from "@/lib/db/schema"; +import { ownsTaskProject } from "@/lib/video-canvas/task-store"; +import { tvstarSessionSchema } from "@/lib/video-canvas/tvstar-session"; + +export const dynamic = "force-dynamic"; + +async function authorize(projectId: string) { + const session = await auth(); + if (!session?.user?.id) return { error: NextResponse.json({ error: "未登录" }, { status: 401 }) }; + if (!z.string().uuid().safeParse(projectId).success || !await ownsTaskProject(session.user.id, projectId)) { + return { error: NextResponse.json({ error: "画布不存在" }, { status: 404 }) }; + } + return { userId: session.user.id }; +} + +export async function GET(_request: Request, { params }: { params: Promise<{ id: string }> }) { + const { id } = await params; + const access = await authorize(id); + if (access.error) return access.error; + const rows = await db.select({ snapshot: tvstarSession.snapshot, shareToken: tvstarSession.shareToken }) + .from(tvstarSession) + .where(and(eq(tvstarSession.userId, access.userId!), eq(tvstarSession.projectId, id))) + .orderBy(desc(tvstarSession.clientUpdatedAt)).limit(20); + return NextResponse.json({ sessions: rows.map(({ snapshot, shareToken }) => ({ + ...(snapshot as Record), shareToken: shareToken || undefined, + })) }, { headers: { "Cache-Control": "no-store" } }); +} + +export async function PUT(request: Request, { params }: { params: Promise<{ id: string }> }) { + const { id } = await params; + const access = await authorize(id); + if (access.error) return access.error; + const raw = await request.text(); + if (raw.length > 900_000) return NextResponse.json({ error: "对话记录过大" }, { status: 413 }); + let body: unknown; + try { body = JSON.parse(raw); } catch { return NextResponse.json({ error: "请求格式无效" }, { status: 400 }); } + const parsed = z.object({ sessions: z.array(tvstarSessionSchema).max(20) }).safeParse(body); + if (!parsed.success) return NextResponse.json({ error: "对话记录格式无效" }, { status: 400 }); + for (const session of parsed.data.sessions) { + const clientUpdatedAt = new Date(Math.min(session.updatedAt, Date.now() + 60_000)); + await db.insert(tvstarSession).values({ + id: session.id, projectId: id, userId: access.userId!, title: session.title, + snapshot: session, clientUpdatedAt, + }).onConflictDoNothing({ target: tvstarSession.id }); + await db.update(tvstarSession).set({ title: session.title, snapshot: session, clientUpdatedAt, updatedAt: new Date() }) + .where(and( + eq(tvstarSession.id, session.id), eq(tvstarSession.projectId, id), eq(tvstarSession.userId, access.userId!), + lte(tvstarSession.clientUpdatedAt, clientUpdatedAt), + )); + } + return NextResponse.json({ ok: true }); +} diff --git a/app/api/canvas/[id]/tvstar-sessions/share/route.ts b/app/api/canvas/[id]/tvstar-sessions/share/route.ts new file mode 100644 index 000000000..6737b109c --- /dev/null +++ b/app/api/canvas/[id]/tvstar-sessions/share/route.ts @@ -0,0 +1,58 @@ +import { randomBytes } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { auth } from "@/app/(auth)/auth"; +import { db } from "@/lib/db"; +import { tvstarSession } from "@/lib/db/schema"; +import { ownsTaskProject } from "@/lib/video-canvas/task-store"; +import { publicTVstarSnapshot, tvstarSessionSchema } from "@/lib/video-canvas/tvstar-session"; + +export const dynamic = "force-dynamic"; + +async function owner(projectId: string) { + const session = await auth(); + if (!session?.user?.id) return { error: NextResponse.json({ error: "未登录" }, { status: 401 }) }; + if (!z.string().uuid().safeParse(projectId).success || !await ownsTaskProject(session.user.id, projectId)) { + return { error: NextResponse.json({ error: "画布不存在" }, { status: 404 }) }; + } + return { userId: session.user.id }; +} + +export async function POST(request: Request, { params }: { params: Promise<{ id: string }> }) { + const { id } = await params; + const access = await owner(id); + if (access.error) return access.error; + const raw = await request.text(); + if (raw.length > 300_000) return NextResponse.json({ error: "对话记录过大" }, { status: 413 }); + let body: unknown; + try { body = JSON.parse(raw); } catch { return NextResponse.json({ error: "请求格式无效" }, { status: 400 }); } + const parsed = z.object({ session: tvstarSessionSchema }).safeParse(body); + if (!parsed.success) return NextResponse.json({ error: "对话记录格式无效" }, { status: 400 }); + const current = parsed.data.session; + const condition = and(eq(tvstarSession.id, current.id), eq(tvstarSession.projectId, id), eq(tvstarSession.userId, access.userId!)); + await db.insert(tvstarSession).values({ + id: current.id, projectId: id, userId: access.userId!, title: current.title, + snapshot: current, clientUpdatedAt: new Date(Math.min(current.updatedAt, Date.now() + 60_000)), + }).onConflictDoNothing({ target: tvstarSession.id }); + const existing = (await db.select({ shareToken: tvstarSession.shareToken }).from(tvstarSession).where(condition).limit(1))[0]; + if (!existing) return NextResponse.json({ error: "对话不存在" }, { status: 404 }); + const token = existing.shareToken || randomBytes(24).toString("base64url"); + if (!existing.shareToken) { + await db.update(tvstarSession).set({ + shareToken: token, sharedSnapshot: publicTVstarSnapshot(current), sharedAt: new Date(), updatedAt: new Date(), + }).where(condition); + } + return NextResponse.json({ url: `/tvstar/share/${token}`, token }); +} + +export async function DELETE(request: Request, { params }: { params: Promise<{ id: string }> }) { + const { id } = await params; + const access = await owner(id); + if (access.error) return access.error; + const sessionId = new URL(request.url).searchParams.get("sessionId") || ""; + if (!z.string().uuid().safeParse(sessionId).success) return NextResponse.json({ error: "对话 ID 无效" }, { status: 400 }); + await db.update(tvstarSession).set({ shareToken: null, sharedSnapshot: null, sharedAt: null, updatedAt: new Date() }) + .where(and(eq(tvstarSession.id, sessionId), eq(tvstarSession.projectId, id), eq(tvstarSession.userId, access.userId!))); + return NextResponse.json({ ok: true }); +} diff --git a/app/api/canvas/agent-capabilities/route.ts b/app/api/canvas/agent-capabilities/route.ts new file mode 100644 index 000000000..fdbfeaccc --- /dev/null +++ b/app/api/canvas/agent-capabilities/route.ts @@ -0,0 +1,40 @@ +import { PROJECT_AGENT_OPERATIONS, projectOperationAuthorized } from "@/lib/ai/canvas-agent-runtime/project-operation-catalog"; +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { and, eq } from "drizzle-orm"; +import { auth } from "@/app/(auth)/auth"; +import { db } from "@/lib/db"; +import { canvasProject } from "@/lib/db/schema"; +import { getSystemSetting } from "@/lib/db/queries"; +import { + normalizeVideoCanvasFunctionConfig, + VIDEO_CANVAS_FUNCTION_CONFIG_KEY, +} from "@/lib/video-canvas/config"; +import { getCanvasAgentAccessManifest } from "@/lib/ai/canvas-agent-runtime/capability-access"; +export const dynamic = "force-dynamic"; +export async function GET(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + const parsed = z.string().uuid().safeParse(new URL(request.url).searchParams.get("projectId")); + if (!parsed.success) return NextResponse.json({ error: "画布编号无效" }, { status: 400 }); + const [project] = await db + .select({ id: canvasProject.id, workspace: canvasProject.workspaceType }) + .from(canvasProject) + .where(and(eq(canvasProject.id, parsed.data), eq(canvasProject.userId, session.user.id))) + .limit(1); + if (!project) return NextResponse.json({ error: "画布不存在" }, { status: 404 }); + const config = normalizeVideoCanvasFunctionConfig( + (await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }))?.value, + ); + return NextResponse.json( + { + workspace: project.workspace, + projectServices: PROJECT_AGENT_OPERATIONS.map(item => ({ ...item, authorized: projectOperationAuthorized(item) })), + utilityTools: ["getWeather", "createDocument", "updateDocument", "requestSuggestions", "loadProjectSkill"], + capabilities: getCanvasAgentAccessManifest(config).filter( + (item) => item.workspace === (project.workspace === "video" ? "video" : "canvas"), + ), + }, + { headers: { "Cache-Control": "private, no-store" } }, + ); +} diff --git a/app/api/canvas/agent-chat/route.ts b/app/api/canvas/agent-chat/route.ts new file mode 100644 index 000000000..69c930789 --- /dev/null +++ b/app/api/canvas/agent-chat/route.ts @@ -0,0 +1,491 @@ +import { agentModelCandidates } from "@/lib/ai/runtime/agent-model-failover"; +import { agentModelCircuitStore } from "@/lib/ai/runtime/agent-model-circuit-store"; +import { createAgentVisionExecutor } from "@/lib/ai/runtime/agent-vision-failover"; +import { canvasAgentProgress } from "@/lib/ai/canvas-agent-runtime/progress"; +import { getStyleProImageModelEntries } from "@/lib/style-pro/model-options"; +import { registerProjectAgentTools } from "@/lib/ai/canvas-agent-runtime/project-tools"; +import { isZcodeWorkspaceAuthorized } from "@/lib/ai/canvas-agent-runtime/authorization"; +import { + createUIMessageStream, + createUIMessageStreamResponse, + generateText, +} from "ai"; +import { after, NextResponse } from "next/server"; +import { z } from "zod"; +import { db } from "@/lib/db"; +import { siteSettings, canvasProject } from "@/lib/db/schema"; +import { eq } from "drizzle-orm"; +import { auth } from "@/app/(auth)/auth"; +import { postRequestBodySchema } from "@/app/(chat)/api/chat/schema"; +import { entitlementsByUserType } from "@/lib/ai/entitlements"; +import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; +import { + getAgentAiModelWithProvider, + getAccessibleAgentPresetById, + getEnabledAiModelsWithProvider, + getChatById, + getMessagesByChatId, + getMessageCountByUserId, + getSiteSettings, + saveChat, + saveMessages, +} from "@/lib/db/queries"; +import { runZcodeCanvasTask } from "@/lib/ai/zcode-agent/task"; +import { canvasAgentReplyText } from "@/lib/ai/zcode-agent/assistant-output"; +import { createZcodeModelGateway } from "@/lib/ai/zcode-agent/model-gateway"; +import { newAgentRunSnapshot, publicAgentError } from "@/lib/ai/canvas-agent-runtime/protocol"; +import { shouldUseCanvasAgentRuntime } from "@/lib/ai/canvas-agent-runtime/routing"; +import { + acquireAgentRun, + agentRunView, + AgentRunError, + checkpointAgentRun, + consumeAgentRunInputs, + heartbeatAgentRun, +} from "@/lib/ai/canvas-agent-runtime/store"; +import { readOwnedCanvas } from "@/lib/ai/canvas-agent-runtime/canvas"; +import { buildCanvasRuntimeTools } from "@/lib/ai/canvas-agent-runtime/tools"; +import { canvasRuntimeSystemPrompt } from "@/lib/ai/canvas-agent-runtime/prompt"; +import { verifyCanvasAgentResults } from "@/lib/ai/canvas-agent-runtime/review"; +import { inferCanvasTaskRequirements } from "@/lib/ai/canvas-agent-runtime/requirements"; +import { + resolveRuntimeConversationSources, + persistRuntimeConversationResults, +} from "@/lib/ai/canvas-agent-runtime/conversation"; +import { getCanvasAgentFeatureFlagsFromSystemSettings } from "@/lib/canvas/agent-feature-settings"; +import { canApplyCanvasAgentMutations } from "@/lib/canvas/agent-feature-flags"; + +import { + videoAgentSettingsSchema, + inferVideoTaskRequirements, + videoNodeUrl, + videoRuntimeSystemPrompt, +} from "@/lib/ai/canvas-agent-runtime/video-protocol"; +import { readOwnedVideoCanvas } from "@/lib/ai/canvas-agent-runtime/video-store"; +import { + buildVideoRuntimeTools, + createVideoRuntimePorts, +} from "@/lib/ai/canvas-agent-runtime/video-tools"; +import { probeVideoMetadata } from "@/lib/ai/canvas-agent-runtime/video-probe"; +import { verifyVideoAgentResults } from "@/lib/ai/canvas-agent-runtime/video-review"; + +export const runtime = "nodejs"; +export const maxDuration = 1200; +const requestSchema = postRequestBodySchema.extend({ + resumeRunId: z.string().uuid().optional(), + selectedImageModel: z.string().min(1).max(180).optional(), + workspaceVariant: z.enum(["canvas", "video"]).default("canvas"), + videoAgentSettings: videoAgentSettingsSchema.optional(), +}); +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const body = requestSchema.parse(await request.json()); + if (!shouldUseCanvasAgentRuntime(body)) + throw new AgentRunError("此请求不属于画布 Agent 任务", 400); + const projectId = body.contextPayload!.projectId!; + const [ownedProject] = await db.select({ userId: canvasProject.userId, workspaceType: canvasProject.workspaceType }) + .from(canvasProject).where(eq(canvasProject.id, projectId)).limit(1); + if (!ownedProject || ownedProject.userId !== session.user.id) throw new AgentRunError("画布不存在", 404); + if (!isZcodeWorkspaceAuthorized(ownedProject.workspaceType)) throw new AgentRunError("视频画布暂未授权给 ZCode Agent，请使用手工功能。", 403); + const isVideo = ownedProject.workspaceType === "video"; + const preset = body.agentPresetId ? await getAccessibleAgentPresetById({ id: body.agentPresetId, userId: session.user.id }) : null; + if (preset && String(preset.type) === "video") throw new AgentRunError("视频技能暂未授权给 ZCode Agent", 403); + if (body.agentPresetId && (!preset || preset.status !== "active")) throw new AgentRunError("此 Agent 预设不可用", 403); + const objective = body.message.parts + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("\n") + .trim(); + if (!objective) throw new AgentRunError("请描述需要完成的创作任务", 400); + const settings = await getSiteSettings(); + if ( + !settings.agentEnabled || + !canApplyCanvasAgentMutations(await getCanvasAgentFeatureFlagsFromSystemSettings()) + ) { + throw new AgentRunError("管理员尚未开启画布 Agent 操作", 403); + } + const [usageSettings] = await db.select().from(siteSettings).limit(1); + if (usageSettings?.usageTimeEnabled) { + const time = new Intl.DateTimeFormat("en-GB", { + timeZone: "Asia/Shanghai", + hour: "2-digit", + minute: "2-digit", + hourCycle: "h23", + }).format(new Date()); + if ( + time < (usageSettings.usageTimeStart || "08:00") || + time > (usageSettings.usageTimeEnd || "20:00") + ) + throw new AgentRunError(usageSettings.usageTimeMessage || "当前不在系统使用时间段内", 403); + } + const count = await getMessageCountByUserId({ id: session.user.id, differenceInHours: 24 }); + if (count > entitlementsByUserType[session.user.type].maxMessagesPerDay) + throw new AgentRunError("今日对话次数已达到限额", 429); + const imageCanvas = isVideo ? null : await readOwnedCanvas(session.user.id, projectId); + if (imageCanvas?.project.workspaceType === "video") + throw new AgentRunError("请使用视频画布 Agent 入口", 400); + const videoCanvas = isVideo ? await readOwnedVideoCanvas(session.user.id, projectId) : null; + const nodes = imageCanvas?.nodes || []; + const availableIds = isVideo + ? videoCanvas!.nodes.map((node) => node.id) + : nodes.map((node) => node.id); + const selectedNodeIds = body.contextPayload?.selectedNodeIds || []; + if (selectedNodeIds.some((id) => !availableIds.includes(id))) { + throw new AgentRunError("选中图片尚未保存到画布，请等待保存完成后重试。"); + } + const existingChat = await getChatById({ id: body.id }); + if (existingChat && existingChat.userId !== session.user.id) + throw new AgentRunError("对话不存在", 404); + if (!existingChat) + await saveChat({ + id: body.id, + userId: session.user.id, + title: objective.slice(0, 80), + visibility: "private", + }); + const conversation = isVideo + ? { + sourceNodeIds: selectedNodeIds, + targetReason: "video-selected-nodes", + previousContext: null, + } + : await resolveRuntimeConversationSources({ + userId: session.user.id, + chatId: body.id, + projectId, + text: objective, + parts: body.message.parts, + nodes, + selectedNodeIds, + }); + const candidates = agentModelCandidates(await getAgentAiModelWithProvider(), await getEnabledAiModelsWithProvider()); + const configured = candidates[0]; + if (!configured) throw new AgentRunError("请先在后台配置可用的多模态 Agent 对话模型", 503); + let model = await getLanguageModelById(configured.model.id); + for (const candidate of candidates.slice(1)) { + if (model) break; + model = await getLanguageModelById(candidate.model.id); + } + if (!model) throw new AgentRunError("Agent 模型暂不可用", 503); + const visionModel = model; + const executeVisionModel = createAgentVisionExecutor(candidates, agentModelCircuitStore, getLanguageModelById); + // Validate the user's image choice independently of the Agent planning model. + if (body.selectedImageModel) { + const images = await getStyleProImageModelEntries(); + if (!images.some(({ model }) => model.id === body.selectedImageModel && model.healthStatus !== "auto_disabled")) + throw new AgentRunError("你选择的生图模型当前不可用，请重新选择；不会自动换模型生成。", 400); + } + const assistantMessageId = crypto.randomUUID(); + const previousVideoMessages = + existingChat + ? (await getMessagesByChatId({ id: body.id })).slice(-6).map((message) => ({ + role: message.role as "user" | "assistant", + content: (message.parts as Array<{ type: string; text?: string }>) + .filter((part) => part.type === "text") + .map((part) => part.text || "") + .join("\n") + .slice(0, 4000), + })) + : []; + const initialSnapshot = newAgentRunSnapshot(objective, [ + ...previousVideoMessages, + { role: "user", content: objective }, + ]); + initialSnapshot.workspace = isVideo ? "video" : "canvas"; + if (isVideo) { + initialSnapshot.videoRequirements = inferVideoTaskRequirements(objective); + initialSnapshot.videoSettings = videoAgentSettingsSchema.parse({ + ...body.videoAgentSettings, + ...(initialSnapshot.videoRequirements.duration + ? { duration: initialSnapshot.videoRequirements.duration } + : {}), + ...(initialSnapshot.videoRequirements.ratio + ? { ratio: initialSnapshot.videoRequirements.ratio } + : {}), + }); + } + initialSnapshot.sourceNodeIds = conversation.sourceNodeIds; + initialSnapshot.requirements = { + ...inferCanvasTaskRequirements(objective, initialSnapshot.sourceNodeIds.length), + sourceCount: initialSnapshot.sourceNodeIds.length, + }; + const run = await acquireAgentRun({ + id: body.message.id, + userId: session.user.id, + projectId, + chatId: body.id, + assistantMessageId, + resumeRunId: body.resumeRunId, + snapshot: initialSnapshot, + }); + const snapshot = run.snapshot; + snapshot.imagePreferences = { + ...snapshot.imagePreferences, + // Each submitted turn captures the current controls, including switching back to Auto. + modelId: body.selectedImageModel, + parameters: body.imageParams || snapshot.imagePreferences?.parameters, + }; + const taskSourceNodeIds = snapshot.sourceNodeIds ?? selectedNodeIds; + const context = { + recentConversation: previousVideoMessages, + preset: preset ? { name: preset.name, type: preset.type, promptTemplate: preset.promptTemplate, defaultModelId: preset.defaultModelId, defaultParams: preset.defaultParams, slots: preset.slots.map(slot => ({ title: slot.title, promptTemplate: slot.promptTemplate, defaultParams: slot.defaultParams })) } : null, + videoSettings: snapshot.videoSettings, + selectedNodeIds: taskSourceNodeIds, + nodes: nodes + .filter((node) => taskSourceNodeIds.includes(node.id)) + .map((node) => ({ + id: node.id, + type: node.type, + position: node.position, + size: node.size, + title: node.data.title, + })), + skillInstructions: body.hiddenPrompt || undefined, + imagePreferences: snapshot.imagePreferences, + targetReason: conversation.targetReason, + previousConversationContext: conversation.previousContext, + }; + await saveMessages({ + messages: [ + { + id: body.message.id, + chatId: body.id, + role: "user", + parts: body.message.parts, + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + let archiveVideoResults: (() => Promise) | undefined; + const stream = createUIMessageStream({ + execute: async ({ writer }) => { + const emit = (type: string, data: unknown) => writer.write({ type, data, ...(type === "data-canvasAgentRun" ? { transient: true } : {}) } as never); + let cancelled = false; + let leaseLost = false; + const activeModelController = new AbortController(); + const withModel: typeof executeVisionModel = (execute, signal) => executeVisionModel(execute, + signal ? AbortSignal.any([signal, activeModelController.signal]) : activeModelController.signal); + const heartbeat = setInterval(() => { + void heartbeatAgentRun(run) + .then((current) => { + if (!current) leaseLost = true; + if (current?.cancelRequested) cancelled = true; + if (cancelled || leaseLost) activeModelController?.abort(); + }) + .catch(() => { + leaseLost = true; + activeModelController?.abort(); + }); + emit("data-keepalive", { label: "canvas-agent-runtime", ts: Date.now() }); + }, 10_000); + const textId = crypto.randomUUID(); + writer.write({ type: "start", messageId: assistantMessageId }); + writer.write({ type: "text-start", id: textId }); + let streamedText = ""; + const emitAssistantText = (delta: string) => { + streamedText += delta; + writer.write({ type: "text-delta", id: textId, delta }); + }; + try { + const videoPorts = isVideo + ? await createVideoRuntimePorts({ + userId: session.user.id, + chatId: body.id, + projectId, + runId: run.id, + assistantMessageId, + request, + snapshot, + emit, + }) + : null; + archiveVideoResults = videoPorts?.archiveResults; + const tools = isVideo + ? buildVideoRuntimeTools({ + runId: run.id, + projectId, + selectedNodeIds, + snapshot, + ports: videoPorts!, + }) + : await buildCanvasRuntimeTools({ + userId: session.user.id, + chatId: body.id, + projectId, + runId: run.id, + assistantMessageId, + selectedNodeIds: taskSourceNodeIds, + request, + model: visionModel || model, + supportsVision: Boolean(visionModel), + withModel, + imageSize: snapshot.imagePreferences?.parameters?.imageSize, + imagePreferences: snapshot.imagePreferences, + preferredModelId: snapshot.imagePreferences?.modelId || (preset?.type === "image" ? preset.defaultModelId || undefined : undefined), + operationSelectedNodeIds: selectedNodeIds, + objective: [snapshot.objective, ...snapshot.inputs.map((input) => input.text)].join( + "\n", + ), + emit, + }); + await registerProjectAgentTools({ session, projectId, request, objective: snapshot.objective, + definitions: tools.definitions, executable: tools.executable, emit, + imagePreferences: snapshot.imagePreferences }); + const gateway = await createZcodeModelGateway(configured, undefined, candidates); + const status = await runZcodeCanvasTask({ + runId: run.id, snapshot, definitions: tools.definitions, + instructions: (isVideo ? videoRuntimeSystemPrompt : canvasRuntimeSystemPrompt)(snapshot.objective, snapshot.summary, snapshot.inputs.map(input => input.text), context), + context, gateway, signal: activeModelController.signal, + onAssistantText: emitAssistantText, + tools: tools.executable, + checkpoint: async (state, stateStatus) => { + if (leaseLost) throw new AgentRunError("任务执行已中断"); + const saved = await checkpointAgentRun(run, state, stateStatus); + cancelled ||= saved.cancelRequested; + emit("data-canvasAgentRun", agentRunView(saved)); + }, + consumeInputs: async (state) => { + if (leaseLost) throw new AgentRunError("任务执行已中断"); + const control = await consumeAgentRunInputs(run, state); + cancelled ||= control.cancelled; + return { cancelled }; + }, + isCancelled: async () => { + const current = await heartbeatAgentRun(run); + if (!current) throw new AgentRunError("任务执行已中断"); + cancelled ||= current.cancelRequested; + return cancelled; + }, + verify: async (state) => + isVideo + ? verifyVideoAgentResults({ + snapshot: state, + nodes: (await readOwnedVideoCanvas(session.user.id, projectId)).nodes, + inspect: async (node, requirements) => { + const metadata = await probeVideoMetadata(videoNodeUrl(node)); + const report = await videoPorts!.analyze({ + mode: "video-analysis", + prompt: `请基于真实抽帧画面检查主体、场景、镜头连贯性，说明不确定内容。用户要求：${requirements}`, + imageUrls: [], + videoUrls: [videoNodeUrl(node)], + }); + const verdict = await generateText({ + model, + system: + '你是视频验收员。依据实际抽帧分析判断是否满足用户要求；不能从文件名或提示词判断。若报告无法读取真实帧，必须不通过。抽帧只能检查画面，不能证明音频、精确时长或完整运动，涉及这些要求无法确认时说明需要人工检查。仅返回 JSON {"passed":boolean,"issues":string[]}。', + prompt: JSON.stringify({ requirements, report }), + maxOutputTokens: 1200, + maxRetries: 0, + abortSignal: AbortSignal.timeout(60_000), + }); + state.inputTokens += verdict.usage.inputTokens || 0; + state.outputTokens += verdict.usage.outputTokens || 0; + const match = verdict.text.match(/\{[\s\S]*\}/); + if (!match) throw new Error("视频检查未返回有效结果"); + return { + ...z + .object({ passed: z.boolean(), issues: z.array(z.string()).max(12) }) + .parse(JSON.parse(match[0])), + ...metadata, + }; + }, + }) + : verifyCanvasAgentResults({ + snapshot: state, + userId: session.user.id, + projectId, + model: visionModel || model, + supportsVision: Boolean(visionModel), + withModel, + signal: activeModelController.signal, + }), + }); + const finalText = canvasAgentReplyText(snapshot); + const progress = canvasAgentProgress(agentRunView({ ...run, snapshot, status })); + writer.write({ type: "data-canvasAgentProgress", id: run.id, data: progress }); + if (!isVideo) + await persistRuntimeConversationResults({ + userId: session.user.id, + chatId: body.id, + projectId, + assistantMessageId, + snapshot, + }).catch((error) => { + console.warn("[Canvas Agent] Conversation memory update failed", { + runId: run.id, + error: publicAgentError(error), + }); + }); + await saveMessages({ + messages: [ + { + id: assistantMessageId, + chatId: body.id, + role: "assistant", + parts: [{ type: "text", text: finalText }, { type: "data-canvasAgentProgress", data: progress }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + emitAssistantText(finalText.startsWith(streamedText) ? finalText.slice(streamedText.length) : `\n\n${snapshot.message}`); + } catch (error) { + const statusText = publicAgentError(error); + const text = canvasAgentReplyText({ ...snapshot, message: statusText }); + await checkpointAgentRun(run, { ...snapshot, message: statusText }, "needs_review").catch( + () => undefined, + ); + const progress = canvasAgentProgress(agentRunView({ ...run, snapshot: { ...snapshot, message: statusText }, status: "needs_review" })); + writer.write({ type: "data-canvasAgentProgress", id: run.id, data: progress }); + await saveMessages({ messages: [{ id: assistantMessageId, chatId: body.id, role: "assistant", + parts: [{ type: "text", text }, { type: "data-canvasAgentProgress", data: progress }], + attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, + permanentVideoUrl: null, archivedAt: null, videoMetadata: null }] }).catch(() => undefined); + emitAssistantText(text.startsWith(streamedText) ? text.slice(streamedText.length) : `\n\n${statusText}`); + } finally { + clearInterval(heartbeat); + writer.write({ type: "text-end", id: textId }); + writer.write({ type: "finish" }); + } + }, + }); + // Browser disconnection does not cancel paid image work. Keep consuming on the server. + const [client, background] = stream.tee(); + after(async () => { + const reader = background.getReader(); + try { + while (!(await reader.read()).done) { + /* Drain the server branch. */ + } + } finally { + reader.releaseLock(); + await archiveVideoResults?.(); + } + }); + return createUIMessageStreamResponse({ + stream: client, + headers: { "Cache-Control": "no-store", "X-Accel-Buffering": "no" }, + }); + } catch (error) { + return NextResponse.json( + { error: publicAgentError(error), message: publicAgentError(error) }, + { + status: + error instanceof AgentRunError ? error.status : error instanceof z.ZodError ? 400 : 503, + }, + ); + } +} diff --git a/app/api/canvas/agent-runs/route.ts b/app/api/canvas/agent-runs/route.ts new file mode 100644 index 000000000..5bd4b98ed --- /dev/null +++ b/app/api/canvas/agent-runs/route.ts @@ -0,0 +1,65 @@ +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { auth } from "@/app/(auth)/auth"; +import { + AgentRunError, + agentRunView, + assertAgentRunOwnership, + controlAgentRun, + getAgentRun, +} from "@/lib/ai/canvas-agent-runtime/store"; +import { publicAgentError } from "@/lib/ai/canvas-agent-runtime/protocol"; + +export const dynamic = "force-dynamic"; +const scope = z.object({ + chatId: z.string().uuid(), + projectId: z.string().uuid(), + runId: z.string().uuid().optional(), +}); +const control = scope + .extend({ + runId: z.string().uuid(), + action: z.enum(["steer", "cancel"]), + inputId: z.string().uuid().optional(), + text: z.string().trim().min(1).max(2000).optional(), + }) + .refine( + (input) => input.action !== "steer" || Boolean(input.inputId && input.text), + "补充要求缺少内容或请求号", + ); + +export async function GET(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const input = scope.parse(Object.fromEntries(new URL(request.url).searchParams)); + await assertAgentRunOwnership(session.user.id, input.projectId, input.chatId); + const run = await getAgentRun(session.user.id, input.chatId, input.projectId, input.runId); + return NextResponse.json( + { run: run ? agentRunView(run) : null }, + { headers: { "Cache-Control": "private, no-store" } }, + ); + } catch (error) { + return failure(error); + } +} +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const input = control.parse(await request.json()); + await assertAgentRunOwnership(session.user.id, input.projectId, input.chatId); + return NextResponse.json({ run: await controlAgentRun({ ...input, userId: session.user.id }) }); + } catch (error) { + return failure(error); + } +} +function failure(error: unknown) { + return NextResponse.json( + { error: publicAgentError(error) }, + { + status: + error instanceof AgentRunError ? error.status : error instanceof z.ZodError ? 400 : 503, + }, + ); +} diff --git a/app/api/canvas/audio-separate/route.ts b/app/api/canvas/audio-separate/route.ts new file mode 100644 index 000000000..c5050c390 --- /dev/null +++ b/app/api/canvas/audio-separate/route.ts @@ -0,0 +1,112 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { uploadFile } from "@/lib/storage/service"; +import { ffmpeg, mediaSafety, probeMedia } from "@/lib/video-canvas/smart-edit/local"; +import { + remoteAudio, + remoteAudioConfig, +} from "@/lib/video-canvas/smart-edit/remote-audio"; + +export const runtime = "nodejs"; +export const maxDuration = 600; + +const running = new Set(); +const allowedStems = ["vocals", "background"] as const; + +export async function POST(request: Request) { + const session = await auth(); + const userId = session?.user?.id; + if (!userId) + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const body = (await request.json().catch(() => null)) as { + url?: unknown; + stem?: unknown; + } | null; + const url = typeof body?.url === "string" ? body.url.trim() : ""; + const stem = typeof body?.stem === "string" ? body.stem : ""; + if (!/^https?:\/\//i.test(url) || !allowedStems.includes(stem as (typeof allowedStems)[number])) + return NextResponse.json({ success: false, error: "参数不完整" }, { status: 400 }); + if (!remoteAudioConfig()) + return NextResponse.json( + { success: false, error: "音频分离服务未配置，请联系管理员" }, + { status: 503 } + ); + const lockKey = `${userId}:${url}:${stem}`; + if (running.has(lockKey) || running.size >= 3) + return NextResponse.json( + { success: false, error: "已有分离任务在进行，请稍候" }, + { status: 429 } + ); + running.add(lockKey); + let dir = ""; + try { + dir = await mkdtemp(path.join(tmpdir(), "canvas-separate-")); + const input = path.join(dir, "input"); + const response = await fetch(url, { + signal: AbortSignal.timeout(180000), + cache: "no-store", + }); + if (!response.ok) throw new Error("素材下载失败，请确认素材仍可访问"); + const bytes = Buffer.from(await response.arrayBuffer()); + if (!bytes.length || bytes.length > 150 * 1024 * 1024) + throw new Error("素材必须非空且不超过 125 MiB"); + await writeFile(input, bytes); + const info = await probeMedia(input, 600); + if (!info.hasAudio) throw new Error("当前视频没有音轨"); + const inputAudio = path.join(dir, "separation-input.wav"); + const durationSeconds = Math.round(info.duration / 1e6); + await ffmpeg( + [ + ...mediaSafety, + "-i", + input, + "-vn", + "-t", + String(durationSeconds), + "-ac", + "2", + "-ar", + "44100", + "-c:a", + "pcm_f32le", + inputAudio, + ], + undefined, + 240000 + ); + const requestKey = createHash("sha256") + .update(`${url}:${stem}:${(await stat(inputAudio)).size}`) + .digest("hex"); + await remoteAudio("separate", inputAudio, userId, requestKey, dir); + const stemFile = path.join(dir, `${stem}.wav`); + const wav = await readFile(stemFile); + if (wav.length < 1024) throw new Error("分离结果异常，请稍后重试"); + const uploaded = await uploadFile( + `canvas-separate-${randomUUID()}.wav`, + await new Blob([wav]).arrayBuffer() + ); + return NextResponse.json({ + success: true, + data: { + url: uploaded.url, + duration: Math.round(info.duration / 1e6), + }, + }); + } catch (error) { + return NextResponse.json( + { + success: false, + error: + error instanceof Error ? error.message : "人声分离失败，原音轨已保留", + }, + { status: 500 } + ); + } finally { + running.delete(lockKey); + if (dir) await rm(dir, { recursive: true, force: true }); + } +} diff --git a/app/api/canvas/audio/models/route.ts b/app/api/canvas/audio/models/route.ts new file mode 100644 index 000000000..92418a3dc --- /dev/null +++ b/app/api/canvas/audio/models/route.ts @@ -0,0 +1,24 @@ +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { listCanvasAudioModels } from "@/lib/video-canvas/audio-generation"; +import { audioContract } from "@/lib/video-canvas/audio-contract"; +export async function GET() { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const models = await listCanvasAudioModels(); + return NextResponse.json( + { + models: models.map(({ model }) => ({ + id: model.id, + modelId: model.modelId, + name: model.displayName, + ...audioContract(model.modelId), + })), + }, + { headers: { "Cache-Control": "no-store" } }, + ); + } catch { + return NextResponse.json({ error: "音频模型读取失败，请稍后重试" }, { status: 503 }); + } +} diff --git a/app/api/canvas/clawd-chat/artifact/route.ts b/app/api/canvas/clawd-chat/artifact/route.ts new file mode 100644 index 000000000..2f4e76be8 --- /dev/null +++ b/app/api/canvas/clawd-chat/artifact/route.ts @@ -0,0 +1,266 @@ +import { lookup } from "node:dns/promises"; +import { isIP } from "node:net"; +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { auth } from "@/app/(auth)/auth"; +import { getChatById, getMessageById, getSystemSetting } from "@/lib/db/queries"; +import { + normalizeOpenClawChatConfig, + OPENCLAW_CHAT_CONFIG_KEY, +} from "@/lib/openclaw-chat/config"; +import { normalizeClawdTaskMetadata } from "@/lib/openclaw-chat/task-state"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; +export const maxDuration = 120; + +const querySchema = z.object({ + chatId: z.string().uuid(), + messageId: z.string().uuid(), + index: z.coerce.number().int().min(0).max(127), + download: z.enum(["0", "1"]).optional().default("0"), +}); + +const MAX_REDIRECTS = 4; +const RESPONSE_HEADERS = [ + "accept-ranges", + "content-length", + "content-range", + "etag", + "last-modified", +] as const; + +function isPrivateIpv4(address: string) { + const octets = address.split(".").map(Number); + if (octets.length !== 4 || octets.some((value) => !Number.isInteger(value))) { + return true; + } + const [a, b] = octets; + return ( + a === 0 || + a === 10 || + a === 127 || + (a === 100 && b >= 64 && b <= 127) || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168) || + a >= 224 + ); +} + +function isPrivateIpAddress(address: string) { + if (isIP(address) === 4) return isPrivateIpv4(address); + if (isIP(address) !== 6) return true; + const normalized = address.toLowerCase(); + if (normalized.startsWith("::ffff:")) { + return isPrivateIpv4(normalized.slice("::ffff:".length)); + } + return ( + normalized === "::" || + normalized === "::1" || + normalized.startsWith("fc") || + normalized.startsWith("fd") || + normalized.startsWith("fe8") || + normalized.startsWith("fe9") || + normalized.startsWith("fea") || + normalized.startsWith("feb") || + normalized.startsWith("ff") + ); +} + +async function validateTargetUrl(target: URL, trustedOrigin: string) { + if (target.username || target.password) return false; + if (target.origin === trustedOrigin) { + return target.protocol === "http:" || target.protocol === "https:"; + } + if (target.protocol !== "https:") return false; + const hostname = target.hostname.toLowerCase().replace(/^\[|\]$/g, ""); + if ( + hostname === "localhost" || + hostname.endsWith(".localhost") || + hostname.endsWith(".local") + ) { + return false; + } + if (isIP(hostname)) return !isPrivateIpAddress(hostname); + + const addresses = await lookup(hostname, { all: true, verbatim: true }).catch( + () => [] + ); + return ( + addresses.length > 0 && + addresses.every((entry) => !isPrivateIpAddress(entry.address)) + ); +} + +function normalizeRangeHeader(value: string | null) { + const range = String(value || "").trim(); + if (!range) return null; + const match = range.match(/^bytes=(\d*)-(\d*)$/); + if (!match || (!match[1] && !match[2])) return ""; + if (match[1] && match[2] && Number(match[1]) > Number(match[2])) return ""; + return range; +} + +function inferContentType(filename: string, declaredType: string) { + if (declaredType) return declaredType; + const lower = filename.toLowerCase(); + if (lower.endsWith(".glb")) return "model/gltf-binary"; + if (lower.endsWith(".gltf")) return "model/gltf+json"; + if (lower.endsWith(".png")) return "image/png"; + if (lower.endsWith(".webp")) return "image/webp"; + if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg"; + return "application/octet-stream"; +} + +function safeDownloadName(value: string) { + const normalized = value.replace(/[\r\n"\\/]/g, "_").trim() || "artifact"; + return normalized.slice(0, 180); +} + +async function fetchArtifact(params: { + request: Request; + initialUrl: URL; + method: "GET" | "HEAD"; + range: string | null; + token: string; + trustedOrigin: string; +}) { + let target = params.initialUrl; + for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects += 1) { + if (!(await validateTargetUrl(target, params.trustedOrigin))) { + throw new Error("不允许访问该文件地址"); + } + const headers = new Headers({ Accept: "*/*" }); + if (params.range) headers.set("Range", params.range); + if (target.origin === params.trustedOrigin && params.token) { + headers.set("Authorization", `Bearer ${params.token}`); + } + const response = await fetch(target, { + method: params.method, + headers, + cache: "no-store", + redirect: "manual", + signal: params.request.signal, + }); + if (![301, 302, 303, 307, 308].includes(response.status)) { + return { response, target }; + } + if (redirects === MAX_REDIRECTS) throw new Error("文件重定向次数过多"); + const location = response.headers.get("location"); + if (!location) throw new Error("文件重定向地址无效"); + target = new URL(location, target); + } + throw new Error("文件地址无效"); +} + +async function handleRequest(request: Request, method: "GET" | "HEAD") { + const session = await auth().catch(() => null); + if (!session?.user?.id) { + return NextResponse.json({ error: "未登录" }, { status: 401 }); + } + + const url = new URL(request.url); + const parsed = querySchema.safeParse({ + chatId: url.searchParams.get("chatId"), + messageId: url.searchParams.get("messageId"), + index: url.searchParams.get("index"), + download: url.searchParams.get("download") || undefined, + }); + if (!parsed.success) { + return NextResponse.json({ error: "请求参数不正确" }, { status: 400 }); + } + + const chat = await getChatById({ id: parsed.data.chatId }).catch(() => null); + if (!chat || chat.userId !== session.user.id) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + const rows = await getMessageById({ id: parsed.data.messageId }).catch( + () => [] + ); + const message = rows.find((row) => row.chatId === parsed.data.chatId); + const metadata = normalizeClawdTaskMetadata(message?.videoMetadata); + const artifact = metadata?.artifacts?.[parsed.data.index]; + if (!message || !artifact) { + return NextResponse.json({ error: "文件不存在" }, { status: 404 }); + } + + const setting = await getSystemSetting({ key: OPENCLAW_CHAT_CONFIG_KEY }).catch( + () => null + ); + const config = normalizeOpenClawChatConfig(setting?.value); + let target: URL; + let trustedOrigin: string; + try { + target = new URL(artifact.downloadUrl); + trustedOrigin = new URL(config.baseUrl).origin; + } catch { + return NextResponse.json({ error: "文件地址无效" }, { status: 400 }); + } + + const range = normalizeRangeHeader(request.headers.get("range")); + if (range === "") { + return NextResponse.json({ error: "Range 请求格式无效" }, { status: 400 }); + } + + try { + const { response, target: finalTarget } = await fetchArtifact({ + request, + initialUrl: target, + method, + range, + token: config.token, + trustedOrigin, + }); + if (!response.ok && response.status !== 206) { + return NextResponse.json( + { error: `文件读取失败（HTTP ${response.status}）` }, + { status: response.status >= 400 && response.status < 600 ? response.status : 502 } + ); + } + + const headers = new Headers(); + for (const name of RESPONSE_HEADERS) { + const value = response.headers.get(name); + if (value) headers.set(name, value); + } + headers.set( + "content-type", + response.headers.get("content-type") || + inferContentType(artifact.filename || finalTarget.pathname, artifact.mimeType) + ); + headers.set("cache-control", "private, max-age=3600, must-revalidate"); + headers.set("x-content-type-options", "nosniff"); + if (parsed.data.download === "1") { + const filename = safeDownloadName(artifact.filename || artifact.displayName); + headers.set( + "content-disposition", + `attachment; filename*=UTF-8''${encodeURIComponent(filename)}` + ); + } + + return new Response(method === "HEAD" ? null : response.body, { + status: response.status, + headers, + }); + } catch (error) { + console.warn("[Clawd Artifact] Proxy failed:", { + chatId: parsed.data.chatId, + messageId: parsed.data.messageId, + artifactIndex: parsed.data.index, + error: error instanceof Error ? error.message : String(error), + }); + return NextResponse.json( + { error: error instanceof Error ? error.message : "文件读取失败" }, + { status: 502 } + ); + } +} + +export async function GET(request: Request) { + return handleRequest(request, "GET"); +} + +export async function HEAD(request: Request) { + return handleRequest(request, "HEAD"); +} diff --git a/app/api/canvas/clawd-chat/history/route.ts b/app/api/canvas/clawd-chat/history/route.ts index 97bbf669d..28d829e49 100644 --- a/app/api/canvas/clawd-chat/history/route.ts +++ b/app/api/canvas/clawd-chat/history/route.ts @@ -25,7 +25,7 @@ export async function GET(request: Request) { const url = new URL(request.url); const limit = Math.max( 1, - Math.min(100, Math.round(Number(url.searchParams.get("limit") || 50))) + Math.min(500, Math.round(Number(url.searchParams.get("limit") || 100))) ); const chats = await getClawdChatsByUserId({ diff --git a/app/api/canvas/clawd-chat/route.ts b/app/api/canvas/clawd-chat/route.ts index 513fc3bc6..d8908b61c 100644 --- a/app/api/canvas/clawd-chat/route.ts +++ b/app/api/canvas/clawd-chat/route.ts @@ -29,6 +29,7 @@ import { OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS, type OpenClawChatMediaAttachment, } from "@/lib/openclaw-chat/media-attachments"; +import { extractOpenClawArtifacts } from "@/lib/openclaw-chat/artifacts"; import { buildOpenClawDisplayMessage, buildOpenClawMessageWithSpreadsheetMarkdown, @@ -298,7 +299,7 @@ async function executeClawdTask(params: ExecuteClawdTaskParams) { params.spreadsheetMarkdownTruncated || undefined, ...params.metadata, }, - stream_progress: false, + stream_progress: !params.sensitiveInput, sensitive_input: params.sensitiveInput, }; @@ -337,6 +338,9 @@ async function executeClawdTask(params: ExecuteClawdTaskParams) { assistantMessageId: params.assistantMessageId, answer, requestId: getRequestId(payload), + artifacts: extractOpenClawArtifacts(payload, { + baseUrl: params.config.baseUrl, + }), }); clearClawdRequestCancellation(params.cancelKey); return; @@ -370,6 +374,7 @@ async function executeClawdTask(params: ExecuteClawdTaskParams) { ...target, answer: outcome.answer, requestId: outcome.requestId, + artifacts: outcome.artifacts, }); } else if (outcome.status === "canceled") { await markClawdTaskCanceled({ @@ -576,7 +581,7 @@ export async function POST(request: Request) { : "image/jpeg"), })); - await persistPendingClawdTask({ + const persistedTask = await persistPendingClawdTask({ chatId: parsed.id, userMessageId, assistantMessageId, @@ -587,6 +592,22 @@ export async function POST(request: Request) { skillTagName, processingHint, }); + if ( + persistedTask && + persistedTask.assistantMessageId !== assistantMessageId + ) { + return NextResponse.json( + buildTaskResponse({ + chatId: parsed.id, + userMessageId, + assistantMessageId: persistedTask.assistantMessageId, + status: persistedTask.status, + processingHint: + persistedTask.processingHint || DEFAULT_PROCESSING_HINT, + }), + { status: 202, headers: { "Retry-After": "2" } } + ); + } const userProfile = await getUserById({ id: session.user.id }).catch( () => null diff --git a/app/api/canvas/clawd-chat/status/route.ts b/app/api/canvas/clawd-chat/status/route.ts index 70c369393..423940dd4 100644 --- a/app/api/canvas/clawd-chat/status/route.ts +++ b/app/api/canvas/clawd-chat/status/route.ts @@ -75,6 +75,7 @@ async function reconcileDetachedOpenClawTask(params: { ...target, answer: outcome.answer, requestId: outcome.requestId, + artifacts: outcome.artifacts, }); } else if (outcome.status === "failed") { await markClawdTaskFailed({ ...target, error: outcome.error }); diff --git a/app/api/canvas/director/upload-model/route.ts b/app/api/canvas/director/upload-model/route.ts new file mode 100644 index 000000000..03600c800 --- /dev/null +++ b/app/api/canvas/director/upload-model/route.ts @@ -0,0 +1,45 @@ +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { uploadFile } from "@/lib/storage/service"; +import { + DIRECTOR_MAX_GLB_BYTES, + validateDirectorGlb, +} from "@/lib/video-canvas/director-glb"; +export const runtime = "nodejs"; +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const form = await request.formData(); + const file = form.get("file"); + if ( + !(file instanceof File) || + !/\.glb$/i.test(file.name) || + file.size > DIRECTOR_MAX_GLB_BYTES + ) + return NextResponse.json( + { error: "请上传 25 MiB 以内的 GLB 模型" }, + { status: 400 }, + ); + const buffer = await file.arrayBuffer(); + try { + validateDirectorGlb(buffer); + } catch (error) { + return NextResponse.json( + { error: error instanceof Error ? error.message : "无效 GLB" }, + { status: 400 }, + ); + } + const result = await uploadFile( + `director-${crypto.randomUUID()}.glb`, + buffer, + ); + return NextResponse.json({ success: true, data: result }); + } catch { + return NextResponse.json( + { error: "模型上传失败，请重试" }, + { status: 500 }, + ); + } +} diff --git a/app/api/canvas/download-image/route.ts b/app/api/canvas/download-image/route.ts index e1e50363b..9a299e7d6 100644 --- a/app/api/canvas/download-image/route.ts +++ b/app/api/canvas/download-image/route.ts @@ -1,5 +1,9 @@ -import { NextRequest, NextResponse } from "next/server"; +import { resolveCanvasCdnDownload } from "@/lib/storage/canvas-cdn-download"; +import { prepareImageArchiveDownload } from "@/lib/storage/image-download-preparation"; +import { after, NextRequest, NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; +import { getDownloadPreparationQueue, type DownloadPreparationInput } from "@/lib/canvas/download-preparation"; +import { extractImageProxySourceUrl, inferOriginalImageUrl } from "@/lib/image-url"; import { buildAttachmentContentDisposition, fetchCanvasDownloadAsset, @@ -9,6 +13,31 @@ import { export const runtime = "nodejs"; export const maxDuration = 120; +async function prepareDownload(input: DownloadPreparationInput, signal: AbortSignal) { + // A proxy candidate for the same original must not repeat the same slow + // upstream request through our own server (and create another HTTP timeout). + const sources = [...new Set(input.urls.map(url => { + const source = extractImageProxySourceUrl(url) || url; + return inferOriginalImageUrl(source) || source; + }))]; + for (let attempt = 0; attempt < 3; attempt++) { + signal.throwIfAborted(); + for (const url of sources) { + signal.throwIfAborted(); + const result = await fetchCanvasDownloadAsset({ + file: { url, name: "original" }, index: 0, + requestOrigin: input.requestOrigin, cookie: input.cookie, + includeSequencePrefix: false, timeoutMs: 12_000, retryCount: 1, signal, + }); + if (result.ok) return { buffer: result.bytes, contentType: result.contentType }; + } + // Each pass resolves the latest archive URL inside fetchCanvasDownloadAsset. + // No AI/model request is involved and a completed source fetch is shared. + if (attempt < 2) await new Promise(resolve => setTimeout(resolve, 1_000)); + } + throw new Error("Original image sources unavailable"); +} + function getRequestOrigin(request: NextRequest): string { const forwardedHost = (request.headers.get("x-forwarded-host") || "").split(",")[0]?.trim(); const forwardedProto = (request.headers.get("x-forwarded-proto") || "").split(",")[0]?.trim(); @@ -55,12 +84,41 @@ export async function POST(request: NextRequest) { const body = (await request.json().catch(() => ({}))) as Record; const requestOrigin = getRequestOrigin(request); const urls = resolveDownloadUrls(body, requestOrigin); + const requestedUrls = [...urls]; if (urls.length === 0) { return NextResponse.json({ success: false, error: "图片地址无效" }, { status: 400 }); } const name = sanitizeCanvasDownloadFileName(String(body.name || ""), "canvas-image"); + if (body.preferDirect === true) { + const { task, direct } = await resolveCanvasCdnDownload(urls, `${name}.png`).catch(() => ({ task: null, direct: null })); + if (direct) return NextResponse.json(direct, { headers: { "Cache-Control": "no-store" } }); + if (task?.targetUrl) { + if (!urls.includes(task.targetUrl)) urls.unshift(task.targetUrl); + after(() => prepareImageArchiveDownload(task.id)); + } + } const cookie = request.headers.get("cookie") || ""; + if (body.preparation === "async") { + const result = await getDownloadPreparationQueue(prepareDownload, after).request({ + userId: session.user.id, urls: requestedUrls, requestOrigin, cookie, + }); + if (result.status === "preparing") { + return NextResponse.json({ delivery: "preparing", retryAfter: 3, + message: "正在准备原图，完成后将自动下载" }, { + status: 202, headers: { "Retry-After": "3", "Cache-Control": "no-store" }, + }); + } + if (result.status === "failed") { + return NextResponse.json({ success: false, error: result.error }, { status: 502 }); + } + return new NextResponse(new Uint8Array(result.asset.buffer), { headers: { + "Content-Type": result.asset.contentType, + "Content-Length": String(result.asset.buffer.length), + "Content-Disposition": buildAttachmentContentDisposition(`${name}.png`), + "Cache-Control": "no-store", + } }); + } let result: Awaited> | null = null; const failures: string[] = []; diff --git a/app/api/canvas/download-zip-task/route.ts b/app/api/canvas/download-zip-task/route.ts index 44b130417..2f5de4a34 100644 --- a/app/api/canvas/download-zip-task/route.ts +++ b/app/api/canvas/download-zip-task/route.ts @@ -1,54 +1,22 @@ +import { writeDownloadZip, readDownloadZip } from "@/lib/canvas/download-zip-stream"; +import { claimZipTask, loadZipTask, saveZipTask, zipTaskPath, assertZipDiskSpace, cleanupZipTasks, assertZipTaskCapacity, type ZipTask } from "@/lib/canvas/download-zip-task-store"; +import { parseDownloadByteRange } from "@/lib/canvas/download-byte-range"; +import { createReadStream } from "node:fs"; +import { mkdir, stat, writeFile, rm } from "node:fs/promises"; +import { join } from "node:path"; import { randomUUID } from "node:crypto"; -import { NextRequest, NextResponse } from "next/server"; +import { after, NextRequest, NextResponse } from "next/server"; import JSZip from "jszip"; import { auth } from "@/app/(auth)/auth"; import { fetchCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download"; - +import { publishCanvasZip } from "@/lib/storage/canvas-zip-delivery"; export const runtime = "nodejs"; export const maxDuration = 300; - -type DownloadFileItem = { - url: string; - name?: string; -}; - -type ZipTaskStatus = "pending" | "running" | "completed" | "failed"; - -type ZipTask = { - id: string; - userId: string; - folderName: string; - files: DownloadFileItem[]; - totalCount: number; - processedCount: number; - successCount: number; - failedCount: number; - compressPercent: number; - status: ZipTaskStatus; - stage: string; - error?: string; - zipBuffer?: Buffer; - contentDisposition?: string; - createdAt: number; - updatedAt: number; - expiresAt: number; -}; - -const MAX_CONCURRENCY = 8; +type DownloadFileItem = { url: string; name?: string }; const MAX_DOWNLOAD_FILES = 200; -const TASK_TTL_MS = 30 * 60 * 1000; - -const globalTaskStore = globalThis as typeof globalThis & { - __canvasDownloadZipTasks?: Map; -}; - -function getTaskStore() { - if (!globalTaskStore.__canvasDownloadZipTasks) { - globalTaskStore.__canvasDownloadZipTasks = new Map(); - } - return globalTaskStore.__canvasDownloadZipTasks; -} - +const TASK_TTL_MS = 60 * 60_000; +const running = (globalThis as typeof globalThis & { __canvasZipRunning?: Set }); +const active = running.__canvasZipRunning ??= new Set(); function sanitizeFolderName(value: string): string { const cleaned = String(value || "") .replace(/[\\/:*?"<>|]+/g, "-") @@ -56,7 +24,7 @@ function sanitizeFolderName(value: string): string { .replace(/-+/g, "-") .replace(/^-+|-+$/g, "") .trim(); - return cleaned || `canvas-images-${Date.now()}`; + return cleaned && !/^\.+$/.test(cleaned) ? cleaned.slice(0, 100) : `canvas-images-${Date.now()}`; } function buildContentDisposition(folderName: string): string { @@ -110,258 +78,161 @@ function toDownloadItems(input: unknown, origin: string): DownloadFileItem[] { .slice(0, MAX_DOWNLOAD_FILES); } -function cleanupExpiredTasks() { - const now = Date.now(); - const store = getTaskStore(); - for (const [taskId, task] of store.entries()) { - if (task.expiresAt <= now) { - store.delete(taskId); - } - } -} - -function getTaskProgress(task: ZipTask) { - if (task.status === "completed") { - return 100; - } - if (task.status === "failed") { - return Math.max( - 0, - Math.min(100, Math.round((task.processedCount / Math.max(1, task.totalCount)) * 85)) - ); - } - if (task.processedCount >= task.totalCount && task.totalCount > 0) { - return Math.min(99, 85 + Math.round((task.compressPercent / 100) * 14)); - } - return Math.max( - 4, - Math.min(85, Math.round((task.processedCount / Math.max(1, task.totalCount)) * 85)) - ); -} function serializeTask(task: ZipTask) { - const downloadUrl = - task.status === "completed" - ? `/api/canvas/download-zip-task?taskId=${encodeURIComponent(task.id)}&download=1` - : ""; - return { - id: task.id, - folderName: task.folderName, - status: task.status, - stage: task.stage, - totalCount: task.totalCount, - processedCount: task.processedCount, - successCount: task.successCount, - failedCount: task.failedCount, - progress: getTaskProgress(task), - error: task.error || "", - downloadUrl, - createdAt: task.createdAt, - updatedAt: task.updatedAt, - expiresAt: task.expiresAt, - }; + return { id: task.id, folderName: task.folderName, status: task.status, stage: task.stage, + totalCount: task.totalCount, processedCount: task.processedCount, successCount: task.successCount, + failedCount: task.failedCount, progress: task.status === "completed" ? 100 : Math.min(99, + Math.round(task.processedCount / Math.max(1, task.totalCount) * 85 + task.compressPercent * .14)), + error: task.error || "", expiresAt: task.expiresAt, + downloadUrl: task.status === "completed" ? `/api/canvas/download-zip-task?taskId=${task.id}&download=1` : "" }; } - -async function processZipTask(params: { - task: ZipTask; - requestOrigin: string; - cookie: string; -}) { - const { task, requestOrigin, cookie } = params; - const store = getTaskStore(); +async function processTask(id: string, userId: string, requestOrigin: string, cookie: string) { + if (active.has(id) || active.size >= 2) return; + active.add(id); + let release: (() => Promise) | null = null; + let task: ZipTask | null = null; try { - task.status = "running"; - task.stage = "正在读取源图"; - task.updatedAt = Date.now(); - - const zip = new JSZip(); - const folder = zip.folder(task.folderName) || zip; - const results: Array< - | Awaited> - | null - > = new Array(task.files.length).fill(null); + release = await claimZipTask(id); + if (!release) return; + task = await loadZipTask(id, userId); + if (!task || task.status === "completed" || task.status === "failed") return; + await assertZipDiskSpace(); + task.status = "running"; task.stage = "正在准备原图"; task.updatedAt = Date.now(); + await saveZipTask(task); + // Stage each image on disk, not all PNG buffers in the web process heap. + // A restart resumes from these files using the current owner's session. let cursor = 0; - const workerCount = Math.min(MAX_CONCURRENCY, task.files.length); - - await Promise.all( - Array.from({ length: workerCount }, async () => { - while (true) { - const index = cursor; - cursor += 1; - if (index >= task.files.length) break; - - const file = task.files[index]!; - results[index] = await fetchCanvasDownloadAsset({ - file, - index, - requestOrigin, - cookie, - }); - - const result = results[index]; - task.processedCount += 1; - if (result?.ok) { - task.successCount += 1; - } else { - task.failedCount += 1; - } - task.stage = `正在处理 ${task.processedCount}/${task.totalCount} 张`; - task.updatedAt = Date.now(); - } - }) - ); - - const failures: Array<{ index: number; url: string; name?: string; error: string }> = []; - for (const [index, item] of results.entries()) { - if (!item || !item.ok) { - failures.push({ - index: index + 1, - url: item?.ok === false ? item.url : task.files[index]?.url || "", - name: item?.ok === false ? item.name : task.files[index]?.name, - error: item?.ok === false ? item.error : "未知错误", - }); - continue; + let writes = Promise.resolve(); + const persist = () => { const current = task!; writes = writes.then(() => saveZipTask(current)); return writes; }; + const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => { + while (cursor < task!.files.length) { + const index = cursor++; const file = task!.files[index]; + const path = join(zipTaskPath(id), `${index}.image`); + if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue; + delete file.prepared; delete file.filename; + const result = await fetchCanvasDownloadAsset({ file, index, requestOrigin, cookie, + signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" })); + if (result.ok) { + await assertZipDiskSpace(result.bytes.length); + await writeFile(path, result.bytes, { mode: 0o600 }); + file.prepared = `${index}.image`; file.filename = result.filename; delete file.error; + } else file.error = result.error; + task!.processedCount = task!.files.filter(f => f.prepared || f.error).length; + task!.successCount = task!.files.filter(f => f.prepared).length; + task!.failedCount = task!.files.filter(f => f.error).length; + task!.stage = `正在准备原图 ${task!.processedCount}/${task!.totalCount}`; + task!.updatedAt = Date.now(); await persist(); } - folder.file(item.filename, item.bytes); - } - - if (failures.length > 0) { - folder.file( - "_download-failures.txt", - [ - "以下图片暂时无法下载。可稍后重试，或复制链接检查源图是否仍可访问。", - "", - ...failures.map((item) => { - const label = item.name ? `${item.index}. ${item.name}` : `${item.index}. image`; - return `${label}\nURL: ${item.url}\n原因: ${item.error}`; - }), - ].join("\n\n") - ); - console.warn("[CanvasDownloadZipTask] Partial download failures:", failures); - } - - task.stage = "正在生成 ZIP 文件"; - task.updatedAt = Date.now(); - task.zipBuffer = await zip.generateAsync( - { - type: "nodebuffer", - compression: "DEFLATE", - compressionOptions: { level: 6 }, - }, - (metadata) => { - task.compressPercent = Math.max(0, Math.min(100, metadata.percent || 0)); - task.updatedAt = Date.now(); - } - ); + })); + const rejected = results.find(result => result.status === "rejected"); + if (rejected?.status === "rejected") throw rejected.reason; + if (!task.successCount) throw new Error("所选原图暂时都无法读取，请稍后重试；画布图片不会被删除"); + const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip; + for (const file of task.files) if (file.prepared && file.filename) + folder.file(file.filename, createReadStream(join(zipTaskPath(id), file.prepared)) + .once("close", () => { void rm(join(zipTaskPath(id), file.prepared!), { force: true }).catch(() => undefined); })); + if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error) + .map(f => `${f.name || "图片"}：${f.error}`).join("\n")); + task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task); + task.zipFile = await writeDownloadZip(zip, task.id, percent => { task!.compressPercent = percent; }); task.contentDisposition = buildContentDisposition(task.folderName); - task.status = "completed"; - task.stage = "打包完成"; - task.compressPercent = 100; - task.updatedAt = Date.now(); - task.expiresAt = Date.now() + TASK_TTL_MS; - } catch (error) { - console.error("[CanvasDownloadZipTask] Failed:", error); - task.status = "failed"; - task.stage = "打包失败"; - task.error = error instanceof Error ? error.message : "打包下载失败"; - task.updatedAt = Date.now(); - task.expiresAt = Date.now() + TASK_TTL_MS; - } finally { - store.set(task.id, task); - } -} - -export async function POST(request: NextRequest) { - try { - cleanupExpiredTasks(); - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); - } - - const body = (await request.json().catch(() => ({}))) as Record; - const requestOrigin = getRequestOrigin(request); - const files = toDownloadItems(body.files, requestOrigin); - const folderName = sanitizeFolderName(String(body.folderName || "")); - if (files.length === 0) { - return NextResponse.json({ success: false, error: "没有可下载的图片" }, { status: 400 }); - } - - const task: ZipTask = { - id: randomUUID(), - userId: session.user.id, - folderName, - files, - totalCount: files.length, - processedCount: 0, - successCount: 0, - failedCount: 0, - compressPercent: 0, - status: "pending", - stage: "等待服务端开始打包", - createdAt: Date.now(), - updatedAt: Date.now(), - expiresAt: Date.now() + TASK_TTL_MS, - }; - getTaskStore().set(task.id, task); - - void processZipTask({ - task, - requestOrigin, - cookie: request.headers.get("cookie") || "", - }); - - return NextResponse.json({ - success: true, - task: serializeTask(task), - }); - } catch (error) { - console.error("[CanvasDownloadZipTask] Create failed:", error); - return NextResponse.json({ success: false, error: "创建打包任务失败" }, { status: 500 }); - } -} - -export async function GET(request: NextRequest) { - try { - cleanupExpiredTasks(); - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); - } - - const taskId = String(request.nextUrl.searchParams.get("taskId") || "").trim(); - if (!taskId) { - return NextResponse.json({ success: false, error: "缺少任务 ID" }, { status: 400 }); - } - - const task = getTaskStore().get(taskId); - if (!task || task.userId !== session.user.id) { - return NextResponse.json({ success: false, error: "打包任务不存在或已过期" }, { status: 404 }); - } - - if (request.nextUrl.searchParams.get("download") === "1") { - if (task.status !== "completed" || !task.zipBuffer) { - return NextResponse.json({ success: false, error: "ZIP 文件尚未准备好" }, { status: 409 }); + task.stage = "正在准备下载链接"; await saveZipTask(task); + try { + const cdn = await publishCanvasZip({ id: task.id, ...task.zipFile, disposition: task.contentDisposition }); + if (cdn) { + task.cdnObject = { storageConfigId: cdn.storageConfigId, key: cdn.key }; + if (cdn.url) task.cdnUrl = cdn.url; } - return new NextResponse(task.zipBuffer, { - status: 200, - headers: { - "Content-Type": "application/zip", - "Content-Disposition": task.contentDisposition || buildContentDisposition(task.folderName), - "Content-Length": String(task.zipBuffer.length), - "Cache-Control": "no-store", - "X-Download-Success-Count": String(task.successCount), - "X-Download-Failed-Count": String(task.failedCount), - }, - }); - } - - return NextResponse.json({ - success: true, - task: serializeTask(task), - }); + } catch { console.warn("[CanvasDownloadZipTask] CDN delivery unavailable; complete local ZIP retained", { taskId: id }); } + task.status = "completed"; task.stage = task.failedCount ? `已打包 ${task.successCount} 张，${task.failedCount} 张未能读取` : "打包完成"; + task.compressPercent = 100; task.updatedAt = Date.now(); task.expiresAt = Date.now() + TASK_TTL_MS; + await saveZipTask(task); + // The complete ZIP is enough for retries; release staged duplicate bytes. + await Promise.all(task.files.filter(f => f.prepared).map(f => rm(join(zipTaskPath(id), f.prepared!), { force: true }).catch(() => undefined))); + console.info("[CanvasDownloadZipTask] Complete", { taskId: id, count: task.successCount, failed: task.failedCount, + bytes: task.zipFile.bytes, delivery: task.cdnUrl ? "cdn" : "local" }); } catch (error) { - console.error("[CanvasDownloadZipTask] Query failed:", error); - return NextResponse.json({ success: false, error: "查询打包任务失败" }, { status: 500 }); + if (task) { + task.status = "failed"; task.stage = "打包未完成"; + task.error = error instanceof Error ? error.message : "打包失败，请稍后重试"; + task.updatedAt = Date.now(); task.expiresAt = Date.now() + TASK_TTL_MS; await saveZipTask(task).catch(() => undefined); + } + console.error("[CanvasDownloadZipTask] Preparation failed", { taskId: id, message: task?.error }); + } finally { await release?.().catch(() => undefined); active.delete(id); } +} +function schedule(task: ZipTask, request: NextRequest) { + const run = () => processTask(task.id, task.userId, getRequestOrigin(request), request.headers.get("cookie") || ""); + after(run); +} +export async function POST(request: NextRequest) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const body = await request.json(); + let id = typeof body.taskId === "string" ? body.taskId : randomUUID(); + let task = await loadZipTask(id, session.user.id); + if (!task) { + await assertZipTaskCapacity(session.user.id); + if (await stat(zipTaskPath(id)).catch(() => null)) id = randomUUID(); + const files = toDownloadItems(body.files, getRequestOrigin(request)); + if (!files.length || !Array.isArray(body.files) || body.files.length > MAX_DOWNLOAD_FILES || files.length !== body.files.length) + return NextResponse.json({ error: "请选择 1 到 200 张可下载图片" }, { status: 400 }); + // Exclusive creation prevents an arbitrary taskId from overwriting another owner's manifest. + await mkdir(zipTaskPath(id), { recursive: false, mode: 0o700 }).catch(async error => { + if (error.code !== "ENOENT") throw error; + await mkdir((await import("@/lib/canvas/download-zip-stream")).downloadZipDirectory, { recursive: true, mode: 0o700 }); + await mkdir(zipTaskPath(id), { mode: 0o700 }); + }); + const now = Date.now(); + task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files, + totalCount: files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0, + status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + 6 * TASK_TTL_MS }; + await saveZipTask(task); + } else if (task.status === "failed" && body.retry === true) { + task.status = "pending"; task.error = undefined; + await assertZipTaskCapacity(session.user.id); + for (const file of task.files) delete file.error; + task.processedCount = task.successCount = task.files.filter(file => file.prepared).length; + task.failedCount = 0; task.updatedAt = Date.now(); task.expiresAt = Date.now() + 6 * TASK_TTL_MS; + await saveZipTask(task); + } + if (task.status === "pending" || task.status === "running") schedule(task, request); + after(() => cleanupZipTasks().catch(() => { console.warn("[CanvasDownloadZipTask] Expired file cleanup deferred"); })); + return NextResponse.json({ success: true, task: serializeTask(task) }, { status: 202, headers: { "Cache-Control": "no-store" } }); + } catch (error) { + console.warn("[CanvasDownloadZipTask] Create rejected", { message: (error as Error).message }); + return NextResponse.json({ error: "无法创建打包任务，请稍后重试" }, { status: 400 }); + } +} +export async function GET(request: NextRequest) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const id = request.nextUrl.searchParams.get("taskId") || ""; + const task = await loadZipTask(id, session.user.id); + if (!task) return NextResponse.json({ error: "打包任务不存在或已过期，请重新打包" }, { status: 404 }); + if (request.nextUrl.searchParams.get("download") === "1") { + if (task.status !== "completed" || !task.zipFile) return NextResponse.json({ error: "ZIP 尚未准备好" }, { status: 409 }); + if (task.cdnUrl && request.nextUrl.searchParams.get("local") !== "1") + return NextResponse.redirect(task.cdnUrl, 307); + const info = await stat(task.zipFile.path).catch(() => null); + if (!info || info.size !== task.zipFile.bytes) return NextResponse.json({ error: "下载文件已过期，请重新打包" }, { status: 410 }); + const etag = `"${task.id}-${info.size}"`; + const headers: Record = { "Content-Type": "application/zip", "Content-Disposition": task.contentDisposition!, + "Accept-Ranges": "bytes", "ETag": etag, "Cache-Control": "private, no-store", + "X-Download-Success-Count": String(task.successCount), "X-Download-Failed-Count": String(task.failedCount) }; + const ifRange = request.headers.get("if-range"); + let range; + try { range = parseDownloadByteRange(!ifRange || ifRange === etag ? request.headers.get("range") : null, info.size); } + catch { return new NextResponse(null, { status: 416, headers: { ...headers, "Content-Range": `bytes */${info.size}` } }); } + headers["Content-Length"] = String(range ? range.end - range.start + 1 : info.size); + if (range) headers["Content-Range"] = `bytes ${range.start}-${range.end}/${info.size}`; + return new NextResponse(readDownloadZip(task.zipFile.path, range || undefined), { status: range ? 206 : 200, headers }); + } + if (task.status === "pending" || task.status === "running") schedule(task, request); + return NextResponse.json({ success: true, task: serializeTask(task) }, { headers: { "Cache-Control": "no-store" } }); + } catch { + return NextResponse.json({ error: "查询打包任务失败，请稍后重试" }, { status: 500 }); } } diff --git a/app/api/canvas/download-zip/route.ts b/app/api/canvas/download-zip/route.ts index 5a9b15675..fca2f03e7 100644 --- a/app/api/canvas/download-zip/route.ts +++ b/app/api/canvas/download-zip/route.ts @@ -1,3 +1,4 @@ +import { streamDownloadZip } from "@/lib/canvas/download-zip-stream"; import { NextRequest, NextResponse } from "next/server"; import JSZip from "jszip"; import { auth } from "@/app/(auth)/auth"; @@ -159,13 +160,7 @@ export async function POST(request: NextRequest) { console.warn("[CanvasDownloadZip] Partial download failures:", failures); } - const zipBuffer = await zip.generateAsync({ - type: "nodebuffer", - compression: "DEFLATE", - compressionOptions: { level: 6 }, - }); - - return new NextResponse(zipBuffer, { + return new NextResponse(streamDownloadZip(zip), { status: 200, headers: { "Content-Type": "application/zip", diff --git a/app/api/canvas/generate/image/route.ts b/app/api/canvas/generate/image/route.ts index 4d605a07a..5060327d3 100644 --- a/app/api/canvas/generate/image/route.ts +++ b/app/api/canvas/generate/image/route.ts @@ -1,62 +1,10 @@ +import { executeCanvasImageResponse } from "@/lib/canvas/generate-image-response"; import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; -import { - createCanvasImageTask, - enqueueCanvasImageTask, - getCanvasImageTaskForUser, -} from "@/lib/canvas/image-task-queue"; -import { - generateCanvasImage, - type CanvasGenerateImageRequest, -} from "@/lib/canvas/generate-image"; -import { getPointsSystemConfig } from "@/lib/db/queries"; -import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; -import { - getImageToolCreditCost, - type ImageToolCreditFeature, -} from "@/lib/points/config"; +import { getCanvasImageTaskForUser } from "@/lib/canvas/image-task-queue"; export const maxDuration = 1200; -interface GenerateImageRequest extends CanvasGenerateImageRequest { - async?: boolean; -} - -function normalizeBillingFeature( - value: unknown -): Extract | null { - if (value === "multi-view" || value === "repaint") return value; - return null; -} - -function toGenerateImagePayload(body: GenerateImageRequest): CanvasGenerateImageRequest { - return { - prompt: String(body.prompt || "").trim(), - negativePrompt: - typeof body.negativePrompt === "string" ? body.negativePrompt : undefined, - model: typeof body.model === "string" ? body.model : undefined, - aspectRatio: - typeof body.aspectRatio === "string" && body.aspectRatio.trim() - ? body.aspectRatio.trim() - : "auto", - imageSize: - typeof body.imageSize === "string" && body.imageSize.trim() - ? body.imageSize.trim() - : undefined, - imageCount: - typeof body.imageCount === "number" && Number.isFinite(body.imageCount) - ? body.imageCount - : 1, - promptExtend: - typeof body.promptExtend === "boolean" ? body.promptExtend : undefined, - referenceImage: - typeof body.referenceImage === "string" ? body.referenceImage : undefined, - referenceImages: Array.isArray(body.referenceImages) - ? body.referenceImages.map((item) => String(item || "").trim()).filter(Boolean) - : [], - }; -} - export async function GET(request: Request) { try { const session = await auth(); @@ -67,10 +15,7 @@ export async function GET(request: Request) { const { searchParams } = new URL(request.url); const taskId = String(searchParams.get("taskId") || "").trim(); if (!taskId) { - return NextResponse.json( - { success: false, error: "缺少 taskId 参数" }, - { status: 400 } - ); + return NextResponse.json({ success: false, error: "缺少 taskId 参数" }, { status: 400 }); } const task = await getCanvasImageTaskForUser({ taskId, userId: session.user.id }); @@ -99,105 +44,15 @@ export async function GET(request: Request) { success: false, error: error instanceof Error ? error.message : "查询任务失败", }, - { status: 500 } + { status: 500 }, ); } } export async function POST(request: Request) { - try { - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json( - { success: false, error: "未登录" }, - { status: 401 } - ); - } - - const body: GenerateImageRequest = await request.json(); - const payload: CanvasGenerateImageRequest = { - ...toGenerateImagePayload(body), - ownerKey: session.user.id, - }; - if (!payload.prompt) { - return NextResponse.json( - { success: false, error: "请输入提示词" }, - { status: 400 } - ); - } - - const billingFeature = normalizeBillingFeature(body.billingFeature); - const pointsCharge = billingFeature - ? await (async () => { - const pointsConfig = await getPointsSystemConfig(); - return prepareFeaturePointsCharge({ - userId: session.user.id, - unitCost: getImageToolCreditCost(pointsConfig, billingFeature), - reason: - billingFeature === "multi-view" - ? "image_multi_view" - : "image_repaint", - modelId: "canvas-generate-image", - note: - billingFeature === "multi-view" - ? "图片多视角生成消耗积分" - : "图片重绘消耗积分", - pointsConfig, - }); - })() - : null; - if (pointsCharge && !pointsCharge.ok) { - return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); - } - - const asyncMode = body.async === true; - if (asyncMode) { - const task = await createCanvasImageTask({ - userId: session.user.id, - request: payload as unknown as Record, - }); - await enqueueCanvasImageTask(task.id); - const currentTask = await getCanvasImageTaskForUser({ - taskId: task.id, - userId: session.user.id, - }); - if (pointsCharge?.chargeable) { - try { - await pointsCharge.consume({ - note: - billingFeature === "multi-view" - ? `图片多视角生成消耗积分（${task.id}）` - : `图片重绘消耗积分（${task.id}）`, - }); - } catch (pointsError) { - console.error("[Points] Failed to consume canvas image task points:", pointsError); - } - } - return NextResponse.json({ - success: true, - taskId: task.id, - status: currentTask?.status || task.status, - queueStatus: currentTask?.queueStatus, - }); - } - - const result = await generateCanvasImage(payload); - if (pointsCharge?.chargeable) { - try { - await pointsCharge.consume(); - } catch (pointsError) { - console.error("[Points] Failed to consume canvas image points:", pointsError); - } - } - return NextResponse.json({ - success: true, - data: result, - }); - } catch (error) { - console.error("[Canvas Generate Image] Error:", error); - return NextResponse.json( - { success: false, error: error instanceof Error ? error.message : "生成失败" }, - { status: 500 } - ); - } + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const body = await request.json().catch(() => ({})); + return executeCanvasImageResponse(body, session.user.id); } diff --git a/app/api/canvas/generate/video/route.ts b/app/api/canvas/generate/video/route.ts index aac3f4a9f..860817982 100644 --- a/app/api/canvas/generate/video/route.ts +++ b/app/api/canvas/generate/video/route.ts @@ -1,7 +1,8 @@ import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; import { executeDirectVideoGeneration } from "@/lib/ai/tools/generate-video"; -import { filterReachableRemoteUrls } from "@/lib/net/url-access"; +import { probeRemoteUrl } from "@/lib/net/url-access"; +import { validateVideoReferences } from "@/lib/video-canvas/reference-input"; import type { VideoGenerationMode, VideoRatio, VideoResolution } from "@/types/ai"; export const maxDuration = 300; @@ -10,6 +11,8 @@ interface GenerateVideoRequest { prompt?: string; mode?: VideoGenerationMode; images?: string[]; + audioUrls?: string[]; + generateAudio?: boolean; resolution?: VideoResolution; ratio?: VideoRatio; duration?: number; @@ -46,23 +49,18 @@ export async function POST(request: Request) { ); } - const inputImages = Array.isArray(body.images) - ? body.images.map((item) => String(item || "").trim()).filter(Boolean).slice(0, 4) - : []; - const { kept: reachableImages, dropped: unreachableImages } = - await filterReachableRemoteUrls(inputImages, { timeoutMs: 3000 }); - - if (inputImages.length > 0 && reachableImages.length === 0) { - return NextResponse.json( - { success: false, error: "参考图片链接不可访问，请重新上传后再试" }, - { status: 400 } - ); + const referenceCheck = await validateVideoReferences(body.images, url => probeRemoteUrl(url, { timeoutMs: 8000 })); + if (referenceCheck.error) { + return NextResponse.json({ success: false, error: referenceCheck.error }, { status: referenceCheck.status }); } + const reachableImages = referenceCheck.images; const mode = body.mode || inferMode(reachableImages); const result = await executeDirectVideoGeneration({ prompt, mode, + audioUrls: Array.isArray(body.audioUrls) ? body.audioUrls.filter(url => typeof url === "string" && /^https?:\/\//i.test(url)).slice(0, 3) : undefined, + generateAudio: body.generateAudio, images: reachableImages, resolution: body.resolution, ratio: body.ratio, @@ -90,13 +88,13 @@ export async function POST(request: Request) { success: true, data: { url: result.videoUrl, - metadata: result.metadata || null, + metadata: { ...result.metadata, temporaryVideoUrl: result.videoUrl, storageStatus: result.metadata?.permanentVideoUrl === result.videoUrl ? "archived" : "temporary" }, modelDisplayName: result.modelDisplayName, modelId: result.modelId, modelRef: result.modelRef, mode, images: reachableImages, - droppedImages: unreachableImages, + droppedImages: [], }, }); } catch (error) { diff --git a/app/api/canvas/generation-tasks/route.ts b/app/api/canvas/generation-tasks/route.ts new file mode 100644 index 000000000..d34e3d45e --- /dev/null +++ b/app/api/canvas/generation-tasks/route.ts @@ -0,0 +1,130 @@ +import { videoUpscaleSchema } from "@/lib/video-canvas/video-upscale-contract"; +import { smartEditRequestSchema } from "@/lib/video-canvas/smart-edit/contract"; +import { audioRequestSchema } from "@/lib/video-canvas/audio-contract"; +import { canResumeCanvasTask } from "@/lib/video-canvas/task-recovery"; +import { imageOperationSchema } from "@/lib/video-canvas/image-operation"; +import { after, NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { z } from "zod"; +import { canvasTaskRequestSchema } from "@/lib/video-canvas/task-contract"; +import { + cancelCanvasGenerationTask, + createCanvasGenerationTask, + listCanvasGenerationTasks, + ownsTaskProject, +} from "@/lib/video-canvas/task-store"; +import { runCanvasGenerationQueue } from "@/lib/video-canvas/task-worker"; +import { preflightCanvasProductionRequests } from "@/lib/video-canvas/production-services"; +export const runtime = "nodejs"; +export const maxDuration = 900; + +async function owner(projectId: unknown) { + const session = await auth(); + if (!session?.user?.id) throw Object.assign(Error("未登录"), { status: 401 }); + if (!z.string().uuid().safeParse(projectId).success) + throw Object.assign(Error("画布 ID 无效"), { status: 400 }); + if (!(await ownsTaskProject(session.user.id, projectId as string))) + throw Object.assign(Error("画布不存在"), { status: 404 }); + return session.user.id; +} +const fail = (error: unknown) => + NextResponse.json( + { success: false, error: error instanceof Error ? error.message : "任务操作失败", code: (error as any)?.code }, + { status: Number((error as any)?.status) || (error instanceof SyntaxError ? 400 : 500) }, + ); +const kick = (userId: string, projectId: string) => + after(() => + runCanvasGenerationQueue(userId, projectId).catch((error) => + console.error("[Canvas tasks]", error), + ), + ); +export async function POST(request: Request) { + try { + const text = await request.text(); + if (new TextEncoder().encode(text).byteLength > 262144) + throw Object.assign(Error("任务数据过大，请使用已上传的素材链接"), { status: 413 }); + const parsed = canvasTaskRequestSchema.safeParse(JSON.parse(text)); + if (!parsed.success) throw Object.assign(Error("任务参数不合法"), { status: 400 }); + if(parsed.data.workflow)throw Object.assign(Error("编组任务必须使用整组提交接口"),{status:400}); + if ( + parsed.data.kind === "image-operation" && + !imageOperationSchema.safeParse(parsed.data.params).success + ) + throw Object.assign(Error("图片处理参数不合法"), { status: 400 }); + if ( + parsed.data.attachment && + ((parsed.data.attachment.type === "story-asset" && parsed.data.kind !== "image") || + (parsed.data.attachment.type === "story-prompts" && parsed.data.kind !== "analysis")) + ) + throw Object.assign(Error("脚本结果类型不匹配"), { status: 400 }); + if (parsed.data.kind === "audio" && !audioRequestSchema.safeParse(parsed.data.params).success) + throw Object.assign(Error("音频参数不合法"), { status: 400 }); + if ( + parsed.data.kind === "smart-edit" && + !smartEditRequestSchema.safeParse(parsed.data.params).success + ) + throw Object.assign(Error("剪辑参数无效"), { status: 400 }); + if (parsed.data.kind === "video-upscale" && !videoUpscaleSchema.safeParse(parsed.data.params).success) throw Object.assign(Error("视频高清参数无效"), {status:400}); + const userId = await owner(parsed.data.projectId); + const task = await createCanvasGenerationTask(userId, parsed.data, + () => preflightCanvasProductionRequests([parsed.data])); + kick(userId, parsed.data.projectId); + return NextResponse.json({ success: true, task }, { status: 202 }); + } catch (error) { + return fail(error); + } +} +export async function GET(request: Request) { + try { + const projectId = new URL(request.url).searchParams.get("projectId") || ""; + const userId = await owner(projectId); + const tasks = await listCanvasGenerationTasks(userId, projectId); + if (tasks.some((task) => task.status === "pending" || canResumeCanvasTask(task))) + kick(userId, projectId); + return NextResponse.json( + { success: true, tasks }, + { headers: { "Cache-Control": "no-store" } }, + ); + } catch (error) { + return fail(error); + } +} +export async function DELETE(request: Request) { + try { + const { projectId, taskId } = await request.json(); + const userId = await owner(projectId); + if (!z.string().uuid().safeParse(taskId).success) + throw Object.assign(Error("任务 ID 无效"), { status: 400 }); + const result = await cancelCanvasGenerationTask(userId, projectId, taskId); + if (!result.task) + return NextResponse.json({ success: false, error: "任务不存在" }, { status: 404 }); + if (!result.ok) + return NextResponse.json( + { + success: false, + error: + result.task.status === "running" + ? "任务已提交供应商，当前线路不支持中止。结果会继续保存，请勿重复生成。" + : "任务已结束或状态待核实，请先刷新任务状态。", + task: result.task, + }, + { status: 409 }, + ); + return NextResponse.json({ success: true, task: result.task }); + } catch (error) { + return fail(error); + } +} +export async function PATCH(request: Request) { + try { + const { projectId, taskId, nodeId } = await request.json(); + const userId = await owner(projectId); + if (!z.string().uuid().safeParse(taskId).success || typeof nodeId !== "string") + return NextResponse.json({ success: false }, { status: 400 }); + const { hideCanvasGenerationNode } = await import("@/lib/video-canvas/task-store"); + const success = await hideCanvasGenerationNode(userId, projectId, taskId, nodeId); + return NextResponse.json({ success }, { status: success ? 200 : 404 }); + } catch (error) { + return fail(error); + } +} diff --git a/app/api/canvas/group-runs/route.ts b/app/api/canvas/group-runs/route.ts new file mode 100644 index 000000000..b24277815 --- /dev/null +++ b/app/api/canvas/group-runs/route.ts @@ -0,0 +1,41 @@ +import { auth } from "@/app/(auth)/auth"; +import { after, NextResponse } from "next/server"; +import { canvasTaskRequestSchema } from "@/lib/video-canvas/task-contract"; +import { + createCanvasGroupTasks, + ownsTaskProject, +} from "@/lib/video-canvas/task-store"; +import { runCanvasGenerationQueue } from "@/lib/video-canvas/task-worker"; +import { preflightCanvasProductionRequests } from "@/lib/video-canvas/production-services"; +export const runtime = "nodejs", + maxDuration = 900; +export async function POST(request: Request) { + try { + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ error: "未登录" }, { status: 401 }); + const text = await request.text(); + if (Buffer.byteLength(text) > 1048576) + return NextResponse.json({ error: "编组数据过大" }, { status: 413 }); + const parsed = canvasTaskRequestSchema + .array() + .min(1) + .max(40) + .safeParse(JSON.parse(text)); + if (!parsed.success) + return NextResponse.json({ error: "编组参数无效" }, { status: 400 }); + const projectId = parsed.data[0].projectId, + userId = session.user.id; + if (!(await ownsTaskProject(userId, projectId))) + return NextResponse.json({ error: "画布不存在" }, { status: 404 }); + const tasks = await createCanvasGroupTasks(userId, parsed.data, + () => preflightCanvasProductionRequests(parsed.data)); + after(() => runCanvasGenerationQueue(userId, projectId)); + return NextResponse.json({ success: true, tasks }, { status: 202 }); + } catch (e) { + return NextResponse.json( + { error: e instanceof Error ? e.message : "编组执行失败", code: (e as any)?.code }, + { status: Number((e as any)?.status) || 400 } + ); + } +} diff --git a/app/api/canvas/group-templates/route.ts b/app/api/canvas/group-templates/route.ts new file mode 100644 index 000000000..aa7dec7c0 --- /dev/null +++ b/app/api/canvas/group-templates/route.ts @@ -0,0 +1,40 @@ +import { auth } from "@/app/(auth)/auth"; +import { db } from "@/lib/db"; +import { canvasProject } from "@/lib/db/schema"; +import { eq, and, sql, desc } from "drizzle-orm"; +import { NextResponse } from "next/server"; +export async function GET() { + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ error: "未登录" }, { status: 401 }); + const rows = await db + .select({ + projectId: canvasProject.id, + title: canvasProject.title, + templates: sql`${canvasProject.canvasData}->'groupTemplates'`, + }) + .from(canvasProject) + .where( + and( + eq(canvasProject.userId, session.user.id), + eq(canvasProject.workspaceType, "video"), + sql`${canvasProject.canvasData} ? 'groupTemplates'` + ) + ) + .orderBy(desc(canvasProject.updatedAt)) + .limit(100); + return NextResponse.json( + { + templates: rows.flatMap((row) => + Array.isArray(row.templates) + ? row.templates.map((template) => ({ + ...template, + projectId: row.projectId, + projectTitle: row.title, + })) + : [] + ), + }, + { headers: { "Cache-Control": "no-store" } } + ); +} diff --git a/app/api/canvas/layout-replace/batches/[batchId]/route.ts b/app/api/canvas/layout-replace/batches/[batchId]/route.ts new file mode 100644 index 000000000..0d51c3a12 --- /dev/null +++ b/app/api/canvas/layout-replace/batches/[batchId]/route.ts @@ -0,0 +1,143 @@ +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { + getCanvasLayoutReplaceBatchActiveTaskId, + getCanvasLayoutReplaceBatchForUser, + markCanvasLayoutReplaceBatchItemsAdded, + retryCanvasLayoutReplaceBatchItem, +} from "@/lib/canvas/layout-replace-batch"; +import { + enqueueCanvasImageTask, + ensureCanvasImageTaskActive, +} from "@/lib/canvas/image-task-queue"; + +function normalizeUuid(value: unknown) { + const id = String(value || "").trim(); + return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id) + ? id + : ""; +} + +export async function GET( + _request: Request, + { params }: { params: Promise<{ batchId: string }> } +) { + try { + const session = await auth(); + if (!session?.user?.id) { + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + } + const batchId = normalizeUuid((await params).batchId); + if (!batchId) { + return NextResponse.json({ success: false, error: "批次ID无效" }, { status: 400 }); + } + + let batch = await getCanvasLayoutReplaceBatchForUser({ + batchId, + userId: session.user.id, + }); + if (!batch) { + return NextResponse.json({ success: false, error: "批次不存在" }, { status: 404 }); + } + const activeTaskId = getCanvasLayoutReplaceBatchActiveTaskId(batch); + if (activeTaskId) { + await ensureCanvasImageTaskActive({ + taskId: activeTaskId, + userId: session.user.id, + }); + batch = await getCanvasLayoutReplaceBatchForUser({ + batchId, + userId: session.user.id, + }); + } + + return NextResponse.json({ success: true, batch }); + } catch (error) { + console.error("[Canvas Layout Replace Batch][Detail] Error:", error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : "查询批次失败", + }, + { status: 500 } + ); + } +} + +export async function PATCH( + request: Request, + { params }: { params: Promise<{ batchId: string }> } +) { + try { + const session = await auth(); + if (!session?.user?.id) { + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + } + const batchId = normalizeUuid((await params).batchId); + const body = await request.json().catch(() => ({})); + if (body?.action === "retry") { + const itemId = normalizeUuid(body?.itemId); + if (!batchId || !itemId) { + return NextResponse.json( + { success: false, error: "缺少需要重试的结果" }, + { status: 400 } + ); + } + const taskId = await retryCanvasLayoutReplaceBatchItem({ + batchId, + itemId, + userId: session.user.id, + }); + if (!taskId) { + return NextResponse.json( + { success: false, error: "该结果当前无法重试" }, + { status: 409 } + ); + } + await enqueueCanvasImageTask(taskId); + return NextResponse.json({ success: true, taskId }); + } + + const items = Array.isArray(body?.items) + ? body.items + .map((item: unknown) => { + const record = + item && typeof item === "object" && !Array.isArray(item) + ? (item as Record) + : {}; + return { + itemId: normalizeUuid(record.itemId), + canvasNodeId: String(record.canvasNodeId || "").trim(), + }; + }) + .filter( + (item: { itemId: string; canvasNodeId: string }) => + Boolean(item.itemId && item.canvasNodeId) + ) + : []; + if (!batchId || items.length === 0) { + return NextResponse.json( + { success: false, error: "缺少已添加的结果信息" }, + { status: 400 } + ); + } + const updated = await markCanvasLayoutReplaceBatchItemsAdded({ + batchId, + userId: session.user.id, + items, + }); + if (!updated) { + return NextResponse.json({ success: false, error: "批次不存在" }, { status: 404 }); + } + return NextResponse.json({ success: true }); + } catch (error) { + console.error("[Canvas Layout Replace Batch][PATCH] Error:", error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : "更新批次失败", + }, + { status: 500 } + ); + } +} diff --git a/app/api/canvas/layout-replace/batches/route.ts b/app/api/canvas/layout-replace/batches/route.ts new file mode 100644 index 000000000..2bbd27e09 --- /dev/null +++ b/app/api/canvas/layout-replace/batches/route.ts @@ -0,0 +1,201 @@ +import { and, eq } from "drizzle-orm"; +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { + CANVAS_LAYOUT_REPLACE_BATCH_LIMIT, + createCanvasLayoutReplaceBatch, + getCanvasLayoutReplaceBatchActiveTaskId, + getLatestCanvasLayoutReplaceBatchForUser, + type CanvasLayoutReplaceBatchSource, +} from "@/lib/canvas/layout-replace-batch"; +import { + enqueueCanvasImageTask, + ensureCanvasImageTaskActive, +} from "@/lib/canvas/image-task-queue"; +import { db } from "@/lib/db"; +import { canvasProject } from "@/lib/db/schema"; +import { getPointsSystemConfig } from "@/lib/db/queries"; +import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; +import { getImageToolCreditCost } from "@/lib/points/config"; + +export const maxDuration = 1200; + +function normalizeHttpUrl(value: unknown) { + const url = String(value || "").trim(); + return /^https?:\/\//i.test(url) ? url : ""; +} + +function normalizeUuid(value: unknown) { + const id = String(value || "").trim(); + return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id) + ? id + : ""; +} + +function normalizeSources(value: unknown): CanvasLayoutReplaceBatchSource[] { + if (!Array.isArray(value)) return []; + const seen = new Set(); + const sources: CanvasLayoutReplaceBatchSource[] = []; + for (const item of value) { + if (!item || typeof item !== "object" || Array.isArray(item)) continue; + const record = item as Record; + const sourceImageUrl = normalizeHttpUrl(record.sourceImageUrl); + if (!sourceImageUrl || seen.has(sourceImageUrl)) continue; + seen.add(sourceImageUrl); + sources.push({ + sourceImageUrl, + sourceKind: record.sourceKind === "upload" ? "upload" : "canvas", + sourceNodeId: String(record.sourceNodeId || "").trim() || undefined, + sourceTitle: String(record.sourceTitle || "").trim().slice(0, 256) || undefined, + }); + if (sources.length >= CANVAS_LAYOUT_REPLACE_BATCH_LIMIT) break; + } + return sources; +} + +async function getOwnedProject(projectId: string, userId: string) { + const [project] = await db + .select({ id: canvasProject.id }) + .from(canvasProject) + .where(and(eq(canvasProject.id, projectId), eq(canvasProject.userId, userId))) + .limit(1); + return project || null; +} + +export async function GET(request: Request) { + try { + const session = await auth(); + if (!session?.user?.id) { + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + } + + const { searchParams } = new URL(request.url); + const projectId = normalizeUuid(searchParams.get("projectId")); + const sourceNodeId = String(searchParams.get("sourceNodeId") || "").trim(); + if (!projectId) { + return NextResponse.json( + { success: false, error: "缺少有效的画布项目" }, + { status: 400 } + ); + } + if (!(await getOwnedProject(projectId, session.user.id))) { + return NextResponse.json({ success: false, error: "画布不存在" }, { status: 404 }); + } + + let batch = await getLatestCanvasLayoutReplaceBatchForUser({ + projectId, + userId: session.user.id, + sourceNodeId: sourceNodeId || undefined, + }); + const activeTaskId = batch + ? getCanvasLayoutReplaceBatchActiveTaskId(batch) + : null; + if (activeTaskId) { + await ensureCanvasImageTaskActive({ + taskId: activeTaskId, + userId: session.user.id, + }); + batch = await getLatestCanvasLayoutReplaceBatchForUser({ + projectId, + userId: session.user.id, + sourceNodeId: sourceNodeId || undefined, + }); + } + + return NextResponse.json({ success: true, batch: batch || null }); + } catch (error) { + console.error("[Canvas Layout Replace Batch][GET] Error:", error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : "查询批次失败", + }, + { status: 500 } + ); + } +} + +export async function POST(request: Request) { + try { + const session = await auth(); + if (!session?.user?.id) { + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + } + + const body = await request.json().catch(() => ({})); + const projectId = normalizeUuid(body?.projectId); + const referenceImageUrl = normalizeHttpUrl(body?.referenceImageUrl); + const sources = normalizeSources(body?.sources); + if (!projectId || !(await getOwnedProject(projectId, session.user.id))) { + return NextResponse.json({ success: false, error: "画布不存在" }, { status: 404 }); + } + if (sources.length === 0) { + return NextResponse.json( + { success: false, error: "请至少添加一张待处理图片" }, + { status: 400 } + ); + } + if (!referenceImageUrl) { + return NextResponse.json( + { success: false, error: "请上传目标模版图" }, + { status: 400 } + ); + } + + const pointsConfig = await getPointsSystemConfig(); + const perImageCost = getImageToolCreditCost(pointsConfig, "repaint"); + const totalCost = perImageCost * sources.length; + const pointsCharge = await prepareFeaturePointsCharge({ + userId: session.user.id, + unitCost: totalCost, + reason: "image_repaint", + modelId: "canvas-layout-replace-batch", + note: `批量排版替换 ${sources.length} 张`, + pointsConfig, + }); + if (!pointsCharge.ok) { + return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); + } + + const created = await createCanvasLayoutReplaceBatch({ + userId: session.user.id, + projectId, + sources, + referenceImageUrl, + supplementalPrompt: + typeof body?.supplementalPrompt === "string" + ? body.supplementalPrompt + : undefined, + imageSize: typeof body?.imageSize === "string" ? body.imageSize : "2K", + model: typeof body?.model === "string" ? body.model : undefined, + }); + await enqueueCanvasImageTask(created.firstTaskId); + + if (pointsCharge.chargeable) { + try { + await pointsCharge.consume({ + note: `批量排版替换 ${sources.length} 张（${created.batchId}）`, + }); + } catch (error) { + console.error("[Points] Failed to consume layout batch points:", error); + } + } + + return NextResponse.json({ + success: true, + batchId: created.batchId, + firstTaskId: created.firstTaskId, + sourceCount: sources.length, + pointsCost: totalCost, + }); + } catch (error) { + console.error("[Canvas Layout Replace Batch][POST] Error:", error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : "创建批次失败", + }, + { status: 500 } + ); + } +} diff --git a/app/api/canvas/layout-replace/route.ts b/app/api/canvas/layout-replace/route.ts new file mode 100644 index 000000000..386f979c0 --- /dev/null +++ b/app/api/canvas/layout-replace/route.ts @@ -0,0 +1,124 @@ +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { + createCanvasImageTask, + enqueueCanvasImageTask, +} from "@/lib/canvas/image-task-queue"; +import { getPointsSystemConfig } from "@/lib/db/queries"; +import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; +import { getImageToolCreditCost } from "@/lib/points/config"; +import type { CanvasLayoutReplaceTaskRequest } from "@/lib/canvas/layout-replace-task"; + +export const maxDuration = 1200; + +function normalizeHttpUrl(value: unknown) { + const url = String(value || "").trim(); + return /^https?:\/\//i.test(url) ? url : ""; +} + +function normalizeUuid(value: unknown) { + const uuid = String(value || "").trim(); + return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(uuid) + ? uuid + : undefined; +} + +export async function POST(request: Request) { + try { + const session = await auth(); + if (!session?.user?.id) { + return NextResponse.json( + { success: false, error: "未登录" }, + { status: 401 }, + ); + } + + const body = await request.json().catch(() => ({})); + const sourceImageUrl = normalizeHttpUrl(body?.sourceImageUrl); + const referenceImageUrl = normalizeHttpUrl(body?.referenceImageUrl); + if (!sourceImageUrl) { + return NextResponse.json( + { success: false, error: "缺少当前产品图片" }, + { status: 400 }, + ); + } + if (!referenceImageUrl) { + return NextResponse.json( + { success: false, error: "缺少排版参考图" }, + { status: 400 }, + ); + } + + const pointsConfig = await getPointsSystemConfig(); + const pointsCharge = await prepareFeaturePointsCharge({ + userId: session.user.id, + unitCost: getImageToolCreditCost(pointsConfig, "repaint"), + reason: "image_repaint", + modelId: "canvas-layout-replace", + note: "排版替换消耗积分", + pointsConfig, + }); + if (!pointsCharge.ok) { + return NextResponse.json(pointsCharge.payload, { + status: pointsCharge.status, + }); + } + + const taskRequest: CanvasLayoutReplaceTaskRequest = { + taskKind: "canvas-layout-replace", + sourceImageUrl, + referenceImageUrl, + supplementalPrompt: + typeof body?.supplementalPrompt === "string" + ? body.supplementalPrompt + : undefined, + model: typeof body?.model === "string" ? body.model : undefined, + imageSize: + typeof body?.imageSize === "string" && body.imageSize.trim() + ? body.imageSize.trim() + : "2K", + sourceNodeId: + typeof body?.sourceNodeId === "string" ? body.sourceNodeId : undefined, + agentProjectId: normalizeUuid(body?.agentProjectId), + referenceLabel: + typeof body?.referenceLabel === "string" ? body.referenceLabel : undefined, + debugMeta: + body?.debugMeta && + typeof body.debugMeta === "object" && + !Array.isArray(body.debugMeta) + ? body.debugMeta + : undefined, + }; + + const task = await createCanvasImageTask({ + userId: session.user.id, + request: taskRequest as unknown as Record, + }); + await enqueueCanvasImageTask(task.id); + + if (pointsCharge.chargeable) { + try { + await pointsCharge.consume({ + note: `排版替换消耗积分（${task.id}）`, + }); + } catch (pointsError) { + console.error("[Points] Failed to consume canvas layout replace points:", pointsError); + } + } + + return NextResponse.json({ + success: true, + taskId: task.id, + status: task.status, + }); + } catch (error) { + console.error("[Canvas Layout Replace] Error:", error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : "排版替换提交失败", + }, + { status: 500 }, + ); + } +} diff --git a/app/api/canvas/media/upload/route.ts b/app/api/canvas/media/upload/route.ts new file mode 100644 index 000000000..b00fa6a09 --- /dev/null +++ b/app/api/canvas/media/upload/route.ts @@ -0,0 +1,65 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { uploadFile } from "@/lib/storage/service"; + +export const runtime = "nodejs"; +export const maxDuration = 180; +const probe = promisify(execFile); +const running = new Set(); +const formats: Record = { + "video/mp4": "mp4", "video/quicktime": "mov", "video/webm": "webm", + "audio/mpeg": "mp3", "audio/mp3": "mp3", "audio/mp4": "m4a", "audio/x-m4a": "m4a", + "audio/wav": "wav", "audio/x-wav": "wav", "audio/ogg": "ogg", "audio/flac": "flac", "audio/webm": "webm", +}; + +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + if (running.has(session.user.id) || running.size >= 4) return NextResponse.json({ success: false, error: "正在上传，请稍后重试" }, { status: 429 }); + running.add(session.user.id); + let dir = ""; + try { + const form = await request.formData(); + const file = form.get("file"); + if (!(file instanceof File) || !formats[file.type]) return NextResponse.json({ success: false, error: "请上传 MP4、MOV、WebM 视频，或 MP3、M4A、WAV、OGG、FLAC 音频" }, { status: 400 }); + if (!file.size || file.size > 125 * 1024 * 1024) return NextResponse.json({ success: false, error: "素材必须非空且不超过 125 MiB" }, { status: 413 }); + dir = await mkdtemp(join(tmpdir(), "canvas-upload-")); + const input = join(dir, "input"); + const buffer = await file.arrayBuffer(); + await writeFile(input, Buffer.from(buffer)); + let stdout: string; + try { + ({ stdout } = await probe("ffprobe", ["-v", "error", "-protocol_whitelist", "file", "-format_whitelist", "mov,matroska,webm,mp3,wav,ogg,flac", "-show_streams", "-show_format", "-of", "json", input], { timeout: 15000, maxBuffer: 1024 * 1024 })); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return NextResponse.json({ success: false, error: "服务器缺少 ffprobe 视频解析组件，请联系管理员安装 FFmpeg" }, { status: 503 }); + } + return NextResponse.json({ success: false, error: "视频文件无法解析，请确认素材完整且格式受支持" }, { status: 400 }); + } + const info = JSON.parse(stdout); + const kind = file.type.startsWith("video/") ? "video" : "audio"; + const stream = info.streams?.find((item: { codec_type: string }) => item.codec_type === kind); + const duration = Number(info.format?.duration); + if (!stream || !Number.isFinite(duration) || duration <= 0 || duration > 600) return NextResponse.json({ success: false, error: "请上传内容完整、10 分钟以内的音视频" }, { status: 400 }); + let uploaded; + try { + uploaded = await uploadFile(`canvas-${randomUUID()}.${formats[file.type]}`, buffer); + } catch (error) { + console.error("[Canvas media upload] Storage unavailable:", error); + return NextResponse.json({ success: false, error: "媒体存储暂不可用，请稍后重试" }, { status: 503 }); + } + return NextResponse.json({ success: true, data: { url: uploaded.url, duration, width: stream.width, height: stream.height } }); + } catch (error) { + console.error("[Canvas media upload] Failed:", error); + return NextResponse.json({ success: false, error: "素材上传失败，请确认文件完整或稍后重试" }, { status: 400 }); + } finally { + running.delete(session.user.id); + if (dir) await rm(dir, { recursive: true, force: true }); + } +} diff --git a/app/api/canvas/prompt-tools/route.ts b/app/api/canvas/prompt-tools/route.ts new file mode 100644 index 000000000..38d9476b4 --- /dev/null +++ b/app/api/canvas/prompt-tools/route.ts @@ -0,0 +1,61 @@ +import { generateText } from "ai"; +import { createLogger } from "@/lib/observability/logger"; +const logger = createLogger("canvas-prompt-tools"); +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; +import { getEnabledAiModelsWithProvider, getSystemSetting } from "@/lib/db/queries"; +import { VIDEO_CANVAS_FUNCTION_CONFIG_KEY, normalizeVideoCanvasFunctionConfig } from "@/lib/video-canvas/config"; +import { promptToolErrorMessage, promptToolFunctionKey, promptToolSchema, selectPromptToolModel } from "@/lib/video-canvas/prompt-tools"; + +export const runtime = "nodejs"; +export const maxDuration = 120; +const running = new Set(); + +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const parsed = promptToolSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) return NextResponse.json({ success: false, error: parsed.error.issues[0]?.message || "参数无效" }, { status: 400 }); + if (running.has(session.user.id)) return NextResponse.json({ success: false, error: "提示词处理中，请稍候" }, { status: 429 }); + running.add(session.user.id); + try { + const [setting, available] = await Promise.all([ + getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }), + getEnabledAiModelsWithProvider(false), + ]); + const config = normalizeVideoCanvasFunctionConfig(setting?.value); + const definition = config.functions[promptToolFunctionKey(parsed.data.mode)]; + if (!config.enabled || !definition?.enabled) return NextResponse.json({ success: false, error: "此提示词功能已停用" }, { status: 403 }); + const requested = parsed.data.model && parsed.data.model !== "auto" + ? parsed.data.model : definition.model && definition.model !== "auto" + ? definition.model : config.defaultMultimodalModel; + let selected; + try { + selected = selectPromptToolModel(available + .filter(item => item.provider.isEnabled && item.model.isEnabled && !item.model.isSubModel) + .map(item => ({ ...item.model, providerType: item.provider.type })), requested); + } catch (error) { + return NextResponse.json({ success: false, error: (error as Error).message }, { status: 400 }); + } + const model = await getLanguageModelById(selected.id); + if (!model) return NextResponse.json({ success: false, error: "提示词模型初始化失败" }, { status: 502 }); + const { text } = await generateText({ + model, + system: definition.prompt, + prompt: parsed.data.prompt, + temperature: parsed.data.mode === "translate" ? 0.1 : 0.4, + maxOutputTokens: 4000, + maxRetries: 0, + abortSignal: AbortSignal.any([request.signal, AbortSignal.timeout(90_000)]), + }); + if (!text.trim()) throw new Error("empty model output"); + return NextResponse.json({ success: true, data: { text: text.trim(), model: selected.id, modelName: selected.displayName || selected.modelId } }); + } catch (error) { + const detail = error as { name?: string; statusCode?: number; cause?: { code?: string }; data?: { error?: { code?: string } } }; + logger.warn("Prompt tool request failed", { name: detail?.name, statusCode: detail?.statusCode, code: detail?.data?.error?.code || detail?.cause?.code }); + return NextResponse.json({ success: false, error: promptToolErrorMessage(error) }, { status: 502 }); + } finally { + running.delete(session.user.id); + } +} diff --git a/app/api/canvas/route.ts b/app/api/canvas/route.ts index 621e14182..b51ce00a8 100644 --- a/app/api/canvas/route.ts +++ b/app/api/canvas/route.ts @@ -100,6 +100,8 @@ export async function POST(request: NextRequest) { .insert(canvasProject) .values({ userId: session.user.id, + createdAt: new Date(), + updatedAt: new Date(), title: title || getDefaultWorkspaceTitle(workspaceType), description, workspaceType, diff --git a/app/api/canvas/smart-edit/capabilities/route.ts b/app/api/canvas/smart-edit/capabilities/route.ts new file mode 100644 index 000000000..38cb6e797 --- /dev/null +++ b/app/api/canvas/smart-edit/capabilities/route.ts @@ -0,0 +1,68 @@ +import { remoteAudioCapabilities } from "@/lib/video-canvas/smart-edit/remote-audio"; +import { auth } from "@/app/(auth)/auth"; +import { NextResponse } from "next/server"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import path from "node:path"; +import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; +import { isCanvasVisionModel } from "@/lib/video-canvas/model-matching"; +import { remoteRenderConfig } from "@/lib/video-canvas/smart-edit/remote-render"; +export async function GET() { + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + let capabilities = { + separation: false, + scenes: false, + beats: false, + asr: false, + ffmpeg: false, + ffprobe: false, + }; + try { + const { stdout } = await promisify(execFile)( + process.env.SMART_EDIT_PYTHON || "python3", + [path.resolve("services/smart-edit/analyze.py"), "--check"], + { timeout: 10000, maxBuffer: 65536 } + ); + capabilities = JSON.parse(stdout); + } catch {} + const audio = await remoteAudioCapabilities(); + if (audio) capabilities = { ...capabilities, ...audio }; + let renderAvailable = capabilities.ffmpeg; + const remote = remoteRenderConfig(); + if (remote) { + try { + const response = await fetch(remote.base + "/health", { + headers: { authorization: "Bearer " + remote.token }, + signal: AbortSignal.timeout(3000), + redirect: "error", + cache: "no-store", + }); + renderAvailable = + response.ok && (await response.json()).status === "ok"; + } catch { + renderAvailable = false; + } + } + const models = (await getEnabledAiModelsWithProvider(false)) + .filter((item) => + isCanvasVisionModel({ ...item.model, providerType: item.provider.type }) + ) + .map(({ model }) => ({ + id: model.id, + name: model.displayName, + modelId: model.modelId, + })); + return NextResponse.json( + { capabilities: { ...capabilities, render: renderAvailable }, models }, + { headers: { "Cache-Control": "no-store" } } + ); + } catch { + return NextResponse.json( + { error: "剪辑服务配置读取失败" }, + { status: 503 } + ); + } +} diff --git a/app/api/canvas/smart-edit/fonts/[id]/route.ts b/app/api/canvas/smart-edit/fonts/[id]/route.ts new file mode 100644 index 000000000..1e2e33be1 --- /dev/null +++ b/app/api/canvas/smart-edit/fonts/[id]/route.ts @@ -0,0 +1,22 @@ +import { auth } from "@/app/(auth)/auth"; +import { captionFont } from "@/lib/video-canvas/smart-edit/fonts"; +import { captionFontFile } from "@/lib/video-canvas/smart-edit/font-files"; +import { readFile } from "node:fs/promises"; +export const runtime = "nodejs"; +export const maxDuration = 180; +export async function GET(_request: Request, { params }: { params: Promise<{ id: string }> }) { + if (!(await auth())?.user?.id) return Response.json({ error: "未登录" }, { status: 401 }); + const { id } = await params; + if (!captionFont(id)) return Response.json({ error: "字体不存在" }, { status: 404 }); + try { + const data = await readFile(await captionFontFile(id)); + return new Response(new Uint8Array(data), { + headers: { + "Content-Type": "font/ttf", + "Cache-Control": "private, max-age=31536000, immutable", + }, + }); + } catch { + return Response.json({ error: "字体加载失败，请检查字体服务配置后重试" }, { status: 503 }); + } +} diff --git a/app/api/canvas/text-edit/route.ts b/app/api/canvas/text-edit/route.ts index a014043db..1154546db 100644 --- a/app/api/canvas/text-edit/route.ts +++ b/app/api/canvas/text-edit/route.ts @@ -29,8 +29,9 @@ export const runtime = "nodejs"; export const maxDuration = 1200; const logger = createLogger("canvas-text-edit"); -const TEXT_EDIT_SOURCE_ARCHIVE_TIMEOUT_MS = 20_000; +const TEXT_EDIT_SOURCE_ARCHIVE_TIMEOUT_MS = 120_000; const TEXT_EDIT_MODEL_FALLBACK_LIMIT = 3; +const TEXT_EDIT_GENERATION_TIMEOUT_MS = 180_000; const textItemSchema = z.object({ id: z.string().min(1), @@ -68,6 +69,13 @@ const SUPPORTED_TEXT_EDIT_PROVIDER_TYPES = new Set([ type TextEditModelCandidate = Awaited>[number]; +function isExplicitTextEditModel(candidate: TextEditModelCandidate) { + const capabilities = candidate.model.capabilities as Record | null; + const kind = String(capabilities?.kind || "").toLowerCase(); + const modelId = String(candidate.model.modelId || "").toLowerCase(); + return kind === "image-edit" || /(?:^|[/_-])edit(?:$|[/_-])/.test(modelId); +} + function getTextEditModelCandidates( models: Awaited> ) { @@ -82,16 +90,19 @@ function getTextEditModelCandidates( }) && item.model.isEnabled && item.provider.isEnabled && + (!item.model.excludeFromFailover || item.model.isDefaultTextEditModel) && SUPPORTED_TEXT_EDIT_PROVIDER_TYPES.has(item.provider.type), ) .sort((a, b) => { const aScore = (a.model.isDefaultTextEditModel ? 100 : 0) + + (isExplicitTextEditModel(a) ? 30 : 0) + (a.model.isDefault ? 10 : 0) + (a.model.modelId?.toLowerCase().includes("seedream") ? 5 : 0) + (a.model.displayName?.includes("火山") ? 3 : 0); const bScore = (b.model.isDefaultTextEditModel ? 100 : 0) + + (isExplicitTextEditModel(b) ? 30 : 0) + (b.model.isDefault ? 10 : 0) + (b.model.modelId?.toLowerCase().includes("seedream") ? 5 : 0) + (b.model.displayName?.includes("火山") ? 3 : 0); @@ -220,6 +231,13 @@ async function resolveTextEditSourceImageUrl(imageUrl: string) { } catch (error) { logger.warn("source_image_archive_failed", { error: error instanceof Error ? error.message : String(error), + sourceHost: (() => { + try { + return new URL(imageUrl).host; + } catch { + return "invalid-url"; + } + })(), }); return imageUrl; } @@ -495,6 +513,7 @@ async function generateTextEditImageWithFailover(params: { ], imageParams: { imageCount: 1, + timeoutMs: TEXT_EDIT_GENERATION_TIMEOUT_MS, }, }; if (selected.provider.type === "atlascloud" || selected.provider.type === "vip98k") { diff --git a/app/api/canvas/video-agent/route.ts b/app/api/canvas/video-agent/route.ts new file mode 100644 index 000000000..95cc9fa22 --- /dev/null +++ b/app/api/canvas/video-agent/route.ts @@ -0,0 +1,78 @@ +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { auth } from "@/app/(auth)/auth"; +import { getChatById, getMessagesByChatId } from "@/lib/db/queries"; +import { + readOwnedVideoCanvas, + saveVideoAgentCanvas, +} from "@/lib/ai/canvas-agent-runtime/video-store"; +import { AgentRunError } from "@/lib/ai/canvas-agent-runtime/store"; +import { publicAgentError } from "@/lib/ai/canvas-agent-runtime/protocol"; + +export const dynamic = "force-dynamic"; +export async function GET(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const projectId = z.string().uuid().parse(new URL(request.url).searchParams.get("projectId")); + await readOwnedVideoCanvas(session.user.id, projectId); + // One stable conversation per video workspace survives browser reloads. + const chat = await getChatById({ id: projectId }); + if (chat && chat.userId !== session.user.id) throw new AgentRunError("对话不存在", 404); + const messages = chat ? (await getMessagesByChatId({ id: projectId })).slice(-40) : []; + return NextResponse.json( + { messages: messages.map(({ id, role, parts }) => ({ id, role, parts })) }, + { headers: { "Cache-Control": "private, no-store" } }, + ); + } catch (error) { + return NextResponse.json( + { error: publicAgentError(error) }, + { + status: + error instanceof AgentRunError ? error.status : error instanceof z.ZodError ? 400 : 503, + }, + ); + } +} + +const saveSchema = z.object({ + projectId: z.string().uuid(), + baseRevision: z.string().datetime(), + canvasData: z.object({ + nodes: z.array(z.record(z.unknown())).max(2000), + edges: z + .array( + z.object({ + id: z.string(), + source: z.string(), + target: z.string(), + type: z.string().optional(), + }), + ) + .max(4000), + selectedNodeIds: z.array(z.string()).max(20), + viewport: z.object({ + x: z.number().finite(), + y: z.number().finite(), + zoom: z.number().positive(), + }), + tool: z.literal("select"), + }), +}); +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const body = saveSchema.parse(await request.json()); + await saveVideoAgentCanvas(session.user.id, body.projectId, body.baseRevision, body.canvasData); + return NextResponse.json({ success: true }); + } catch (error) { + return NextResponse.json( + { error: publicAgentError(error) }, + { + status: + error instanceof AgentRunError ? error.status : error instanceof z.ZodError ? 400 : 503, + }, + ); + } +} diff --git a/app/api/canvas/video-upscale/capabilities/route.ts b/app/api/canvas/video-upscale/capabilities/route.ts new file mode 100644 index 000000000..9bdc36754 --- /dev/null +++ b/app/api/canvas/video-upscale/capabilities/route.ts @@ -0,0 +1,39 @@ +import { auth } from "@/app/(auth)/auth"; +import { upscaleConfig } from "@/lib/video-canvas/video-upscale-remote"; +import { NextResponse } from "next/server"; +export async function GET() { + if (!(await auth())?.user?.id) + return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const config = upscaleConfig(); + if (!config) + return NextResponse.json({ + available: false, + message: "视频高清服务尚未接通", + }); + const response = await fetch(config.base + "/health", { + headers: { authorization: "Bearer " + config.token }, + signal: AbortSignal.timeout(5000), + redirect: "error", + cache: "no-store", + }); + if (!response.ok) throw Error("unavailable"); + const data = await response.json(); + return NextResponse.json( + { + available: data.status === "ok", + engine: "Real-ESRGAN", + resolutions: ["1080P", "2K", "4K"], + maxScale: 4, + maxSeconds: 120, + queued: data.queued, + }, + { headers: { "Cache-Control": "no-store" } } + ); + } catch { + return NextResponse.json( + { available: false, message: "视频高清服务暂时不可用，请稍后重试" }, + { headers: { "Cache-Control": "no-store" } } + ); + } +} diff --git a/app/api/canvas/video/archive/route.ts b/app/api/canvas/video/archive/route.ts new file mode 100644 index 000000000..f76790464 --- /dev/null +++ b/app/api/canvas/video/archive/route.ts @@ -0,0 +1,23 @@ +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { auth } from "@/app/(auth)/auth"; +import { archiveCanvasVideo } from "@/lib/video-canvas/archive-video"; + +export const runtime = "nodejs"; +export const maxDuration = 300; +const running = new Set(); +const schema = z.object({ url: z.string().url().max(12000) }); +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const parsed = schema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) return NextResponse.json({ success: false, error: "视频地址无效" }, { status: 400 }); + if (running.has(session.user.id)) return NextResponse.json({ success: false, error: "视频正在保存，请稍候" }, { status: 429 }); + running.add(session.user.id); + try { + const url = await archiveCanvasVideo(parsed.data.url); + return NextResponse.json({ success: true, data: { url, storageStatus: "archived", permanentVideoUrl: url } }); + } catch { + return NextResponse.json({ success: false, error: "视频转存失败，原视频仍保留；请稍后重试保存，无需重新生成" }, { status: 502 }); + } finally { running.delete(session.user.id); } +} diff --git a/app/api/canvas/video/process/route.ts b/app/api/canvas/video/process/route.ts new file mode 100644 index 000000000..9fd412245 --- /dev/null +++ b/app/api/canvas/video/process/route.ts @@ -0,0 +1,96 @@ +import { auth } from "@/app/(auth)/auth"; +import { NextResponse } from "next/server"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { randomUUID } from "node:crypto"; +import { uploadFile } from "@/lib/storage/service"; +import { downloadCanvasMedia } from "@/lib/video-canvas/download-media"; +import { processVideoFile } from "@/lib/video-canvas/media-processing"; +import { videoProcessSchema } from "@/lib/video-canvas/media-contract"; +export const runtime = "nodejs"; +export const maxDuration = 300; +const running = new Set(); +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const parsed = videoProcessSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) + return NextResponse.json({ success: false, error: "视频处理参数无效" }, { status: 400 }); + if (running.has(session.user.id) || running.size >= 2) + return NextResponse.json( + { success: false, error: "视频处理中，请稍后再提交" }, + { status: 429 }, + ); + running.add(session.user.id); + let dir = ""; + try { + dir = await mkdtemp(join(tmpdir(), "canvas-video-")); + const input = join(dir, "input"); + await downloadCanvasMedia(parsed.data.url, input); + const save = async (name: string) => { + const buffer = await readFile(join(dir, name)); + return ( + await uploadFile(`canvas-video-${randomUUID()}-${name}`, new Uint8Array(buffer).buffer) + ).url; + }; + if (parsed.data.operation === "frame" || parsed.data.operation === "slow") { + const name = parsed.data.operation === "frame" ? "frame.png" : "slow.mp4"; + const metadata = await processVideoFile( + input, + join(dir, name), + parsed.data.operation, + undefined, + undefined, + parsed.data, + ); + return NextResponse.json({ success: true, data: { url: await save(name), ...metadata } }); + } + if (parsed.data.operation === "trim") { + const metadata = await processVideoFile( + input, + join(dir, "clip.mp4"), + "trim", + undefined, + parsed.data.range, + ); + return NextResponse.json({ + success: true, + data: { url: await save("clip.mp4"), ...metadata }, + }); + } + if (parsed.data.operation === "crop") { + const metadata = await processVideoFile( + input, + join(dir, "crop.mp4"), + "crop", + parsed.data.rect, + ); + return NextResponse.json({ + success: true, + data: { url: await save("crop.mp4"), ...metadata }, + }); + } + const metadata = await processVideoFile(input, join(dir, "audio.m4a"), "extract-audio"); + await processVideoFile(input, join(dir, "silent.mp4"), "silent-video"); + return NextResponse.json({ + success: true, + data: { audioUrl: await save("audio.m4a"), url: await save("silent.mp4"), ...metadata }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : "视频处理失败"; + return NextResponse.json( + { + success: false, + error: message.startsWith("Command failed") + ? "视频处理失败，请确认文件完整且格式受支持" + : message, + }, + { status: 400 }, + ); + } finally { + running.delete(session.user.id); + if (dir) await rm(dir, { recursive: true, force: true }); + } +} diff --git a/app/api/canvas/workflow/analyze/route.ts b/app/api/canvas/workflow/analyze/route.ts index a2c4b744e..3d832b2be 100644 --- a/app/api/canvas/workflow/analyze/route.ts +++ b/app/api/canvas/workflow/analyze/route.ts @@ -1,1010 +1,12 @@ -import { generateText } from "ai"; -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; +import { executeWorkflowAnalysisResponse } from "@/lib/video-canvas/workflow-analysis-response"; import { NextResponse } from "next/server"; -import sharp from "sharp"; -import { z } from "zod"; import { auth } from "@/app/(auth)/auth"; -import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; -import { inlineVisionImages } from "@/lib/ai/vision-inline"; -import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags"; -import { - getAgentAiModelWithProvider, - getEnabledAiModelsWithProvider, -} from "@/lib/db/queries"; - export const runtime = "nodejs"; export const maxDuration = 180; - -const execFileAsync = promisify(execFile); -const VIDEO_ANALYSIS_FRAME_COUNT = 4; -const VIDEO_ANALYSIS_MAX_SIDE = 960; -const VIDEO_ANALYSIS_TIMEOUT_MS = 15000; - -const requestSchema = z.object({ - mode: z.enum(["video-analysis", "character-scene-extraction", "story-script", "agent-plan"]), - prompt: z.string().trim().optional(), - imageUrls: z.array(z.string().trim()).optional(), - videoUrls: z.array(z.string().trim()).optional(), - nodeTitle: z.string().trim().optional(), - preferredModelRef: z.string().trim().optional(), - model: z.string().trim().optional(), -}); - -function sanitizeUrls(value: unknown, limit = 6) { - if (!Array.isArray(value)) return []; - return value - .map((item) => String(item || "").trim()) - .filter((url) => /^https?:\/\//i.test(url)) - .slice(0, limit); -} - -function supportsVision(model: { - supportsVision?: boolean | null; - capabilities?: unknown; - defaultParams?: unknown; -}) { - return ( - Boolean(model.supportsVision) || - isAiModelMultimodal({ - capabilities: model.capabilities, - defaultParams: model.defaultParams, - }) - ); -} - -function extractJsonBlock>(text: string): T | null { - const matched = String(text || "").match(/\{[\s\S]*\}/); - if (!matched) return null; - try { - return JSON.parse(matched[0]) as T; - } catch { - return null; - } -} - -function normalizeText(value: unknown) { - return String(value || "").trim(); -} - -function shouldUseLocalWorkflowE2E(request: Request) { - const hostname = new URL(request.url).hostname; - return ( - process.env.CANVAS_E2E_LOCAL === "true" && - (hostname === "127.0.0.1" || hostname === "localhost" || hostname === "::1") - ); -} - -function normalizeStringList(value: unknown, limit = 8) { - if (!Array.isArray(value)) return []; - return Array.from( - new Set( - value - .map((item) => String(item || "").trim()) - .filter(Boolean) - .slice(0, limit) - ) - ); -} - -function normalizeShotIdeas(value: unknown) { - if (!Array.isArray(value)) return []; - return value - .map((item, index) => { - const raw = - item && typeof item === "object" && !Array.isArray(item) - ? (item as Record) - : {}; - const title = normalizeText(raw.title || `镜头 ${index + 1}`); - const description = normalizeText(raw.description || raw.prompt || raw.summary); - if (!title && !description) return null; - return { - title: title || `镜头 ${index + 1}`, - description, - }; - }) - .filter( - (item): item is { title: string; description: string } => Boolean(item) - ) - .slice(0, 6); -} - -function normalizeVideoAnalysisRows( - value: unknown, - framePreviews: string[], -) { - if (!Array.isArray(value)) return []; - return value - .map((item, index) => { - const raw = - item && typeof item === "object" && !Array.isArray(item) - ? (item as Record) - : {}; - const sceneDescription = normalizeText( - raw.sceneDescription || raw.scene || raw.title || raw.description, - ); - const narrative = normalizeText( - raw.narrative || raw.story || raw.storyContent || raw.narration || raw.summary, - ); - const shot = normalizeText(raw.shot || raw.shotNumber || raw.index || index + 1); - const startTime = normalizeText(raw.startTime || raw.start || raw.start_time); - const endTime = normalizeText(raw.endTime || raw.end || raw.end_time); - const duration = normalizeText(raw.duration || raw.durationSeconds || raw.duration_seconds); - const shotSize = normalizeText(raw.shotSize || raw.frameSize || raw.shot_type || raw.sceneScale); - const cameraAngle = normalizeText(raw.cameraAngle || raw.angle || raw.camera_angle); - const cameraMovement = normalizeText( - raw.cameraMovement || raw.cameraMove || raw.camera_motion || raw.motionType, - ); - const focusDepth = normalizeText( - raw.focusDepth || raw.depthOfField || raw.focalDepth || raw.focus_depth, - ); - const lighting = normalizeText(raw.lighting || raw.light || raw.lightDesign); - const backgroundMusic = normalizeText( - raw.backgroundMusic || raw.music || raw.bgm, - ); - const voiceAndSfx = normalizeText( - raw.voiceAndSfx || raw.sound || raw.audio || raw.sfx, - ); - const imagePrompt = normalizeText( - raw.imagePrompt || raw.image_generation_prompt || raw.image, - ); - const motionPrompt = normalizeText( - raw.motionPrompt || raw.motion || raw.videoPrompt || raw.video_motion_prompt, - ); - const keyframeIndex = Number( - raw.keyframeIndex || raw.frameIndex || raw.key_frame_index || index + 1, - ); - if ( - !sceneDescription && - !narrative && - !backgroundMusic && - !voiceAndSfx && - !imagePrompt && - !motionPrompt && - !shotSize && - !cameraAngle && - !cameraMovement && - !focusDepth && - !lighting - ) { - return null; - } - const normalizedFrameIndex = - Number.isFinite(keyframeIndex) && keyframeIndex > 0 - ? Math.max(0, Math.min(framePreviews.length - 1, Math.round(keyframeIndex) - 1)) - : index % Math.max(framePreviews.length, 1); - return { - shot: shot || String(index + 1), - startTime: startTime || (index === 0 ? "0" : ""), - endTime, - duration, - sceneDescription: sceneDescription || `镜头 ${index + 1}`, - narrative: narrative || sceneDescription || `镜头 ${index + 1}`, - shotSize: shotSize || "中景至近景", - cameraAngle: cameraAngle || "平视", - cameraMovement: cameraMovement || "缓慢推镜头 (Slow Zoom In)", - focusDepth: focusDepth || "浅景深", - lighting: lighting || "冷调低沉照明，主体光线柔和。", - backgroundMusic: backgroundMusic || "无", - voiceAndSfx: voiceAndSfx || "无", - imagePrompt: imagePrompt || sceneDescription || "延续当前视频气质生成关键帧", - motionPrompt: motionPrompt || "保持镜头运动自然、连贯、符合前后节奏。", - keyframeUrl: framePreviews[normalizedFrameIndex] || "", - }; - }) - .filter( - ( - item, - ): item is { - shot: string; - startTime: string; - endTime: string; - duration: string; - sceneDescription: string; - narrative: string; - shotSize: string; - cameraAngle: string; - cameraMovement: string; - focusDepth: string; - lighting: string; - backgroundMusic: string; - voiceAndSfx: string; - imagePrompt: string; - motionPrompt: string; - keyframeUrl: string; - } => Boolean(item), - ) - .slice(0, 8); -} - -function normalizeStoryShot(value: unknown, index: number) { - const text = normalizeText(value); - if (/^\d+$/.test(text)) return text.padStart(2, "0"); - return text || String(index + 1).padStart(2, "0"); -} - -type NormalizedStoryScriptRow = { - shot: string; - duration: string; - scene: string; - sceneDescription: string; - shotSize: string; - lighting: string; - dialogue: string; - backgroundMusic: string; - voiceAndSfx: string; - cameraMovement: string; - imagePrompt: string; - motionPrompt: string; - keyframeUrl: string; -}; - -function normalizeStoryScriptRows(value: unknown): NormalizedStoryScriptRow[] { - if (!Array.isArray(value)) return []; - return value - .map((item, index) => { - if (Array.isArray(item)) { - const cells = item.map((cell) => normalizeText(cell)); - const shot = normalizeStoryShot(cells[0], index); - const secondCellLooksLikeDuration = /^\d+(\.\d+)?\s*(s|秒)?$/i.test(cells[1] || ""); - const duration = secondCellLooksLikeDuration ? cells[1] : "5s"; - const scene = secondCellLooksLikeDuration ? cells[2] || `镜头 ${shot}` : cells[1] || `镜头 ${shot}`; - const sceneDescription = secondCellLooksLikeDuration ? cells[2] || scene : cells[2] || cells[3] || scene; - return { - shot, - duration, - scene, - sceneDescription, - shotSize: (secondCellLooksLikeDuration ? cells[3] : "") || "中景", - lighting: (secondCellLooksLikeDuration ? cells[4] : cells[4]) || "电影感光影", - dialogue: (secondCellLooksLikeDuration ? cells[5] : "") || "+", - backgroundMusic: "无", - voiceAndSfx: (secondCellLooksLikeDuration ? cells[6] : cells[3]) || "无", - cameraMovement: (secondCellLooksLikeDuration ? cells[7] : "") || "缓慢推镜头", - imagePrompt: (secondCellLooksLikeDuration ? cells[8] : cells[5]) || sceneDescription, - motionPrompt: - (secondCellLooksLikeDuration ? cells[9] || cells[7] : cells[6]) || - "保持镜头运动自然、连贯，围绕主体动作推进。", - keyframeUrl: "", - }; - } - - const raw = - item && typeof item === "object" && !Array.isArray(item) - ? (item as Record) - : {}; - const shot = normalizeStoryShot(raw.shot || raw.shotNumber || raw.index || raw.id, index); - const duration = normalizeText(raw.duration || raw.length || raw.time || raw.seconds); - const scene = normalizeText(raw.scene || raw.location || raw.title || `镜头 ${shot}`); - const sceneDescription = normalizeText( - raw.sceneDescription || - raw.description || - raw.visual || - raw.picture || - raw.screen || - raw.summary, - ); - const shotSize = normalizeText( - raw.shotSize || raw.frameSize || raw.shot_type || raw.sceneScale || raw.cameraShot, - ); - const lighting = normalizeText( - raw.lighting || raw.light || raw.atmosphere || raw.mood || raw.lightMood, - ); - const dialogue = normalizeText( - raw.dialogue || raw.dialog || raw.narration || raw.voiceover || raw.monologue || raw.line, - ); - const cameraMovement = normalizeText( - raw.cameraMovement || raw.cameraMove || raw.movement || raw.lensMovement || raw.motion, - ); - const imagePrompt = normalizeText( - raw.imagePrompt || - raw.image_generation_prompt || - raw.image || - raw.keyframePrompt || - raw.keyFramePrompt, - ); - const motionPrompt = normalizeText( - raw.motionPrompt || - raw.videoPrompt || - raw.video_motion_prompt || - raw.motion || - raw.cameraMovement, - ); - const backgroundMusic = normalizeText( - raw.backgroundMusic || raw.music || raw.bgm || raw.soundtrack, - ); - const voiceAndSfx = normalizeText( - raw.voiceAndSfx || raw.voice || raw.sound || raw.audio || raw.sfx, - ); - - if ( - !scene && - !sceneDescription && - !imagePrompt && - !motionPrompt && - !backgroundMusic && - !voiceAndSfx - ) { - return null; - } - - return { - shot, - duration: duration || "5s", - scene, - sceneDescription: sceneDescription || imagePrompt || scene || `镜头 ${shot}`, - shotSize: shotSize || "中景", - lighting: lighting || "电影感光影", - dialogue: dialogue || "+", - backgroundMusic: backgroundMusic || "无", - voiceAndSfx: voiceAndSfx || "无", - cameraMovement: cameraMovement || "缓慢推镜头", - imagePrompt: imagePrompt || sceneDescription || scene || `镜头 ${shot}关键帧`, - motionPrompt: motionPrompt || cameraMovement || "保持镜头运动自然、连贯，围绕主体动作推进。", - keyframeUrl: "", - }; - }) - .filter( - ( - item, - ): item is NormalizedStoryScriptRow => Boolean(item), - ) - .slice(0, 18); -} - -const storyExpansionPhases = [ - { - description: "建立镜头，交代环境、人物位置和当前压力", - cameraMovement: "缓慢推镜头", - imagePrompt: "广角或中景建立画面，环境信息清晰，主体位置明确", - lighting: "环境主光明确，氛围基调清晰", - motionPrompt: "镜头缓慢推进或轻微横移，建立空间关系", - shotSize: "中景", - }, - { - description: "动作推进，突出人物的关键行为或剧情转折", - cameraMovement: "跟随主体动作推进", - imagePrompt: "中景表现人物动作，服装、道具和光影保持连续", - lighting: "主体光影连续，动作区域清晰", - motionPrompt: "跟随主体动作推进，节奏自然连贯", - shotSize: "中景", - }, - { - description: "情绪反应，用近景或特写强化人物心理和冲突", - cameraMovement: "轻微推近或稳定特写", - imagePrompt: "近景或特写，突出表情、眼神、手部或关键道具细节", - lighting: "面部光线突出，情绪反差增强", - motionPrompt: "轻微推近或稳定特写，强调情绪变化", - shotSize: "特写", - }, - { - description: "关系与氛围，表现周围人物、环境反馈或声画张力", - cameraMovement: "横移扫过环境或切换视角", - imagePrompt: "加入环境反馈、群像反应或空间层次，电影感光影", - lighting: "场景氛围增强，空间层次明显", - motionPrompt: "镜头扫过环境或切换视角，保持节奏递进", - shotSize: "全景", - }, - { - description: "转场收束，为下一镜头保留动作、视线或情绪钩子", - cameraMovement: "顺势后移、上摇或转场", - imagePrompt: "构图留出转场方向，动作或视线指向下一段剧情", - lighting: "光影引导视线并保留转场方向", - motionPrompt: "镜头后移、上摇或顺势转场，衔接下一镜头", - shotSize: "大远景", - }, -]; - -function expandStoryScriptRowsToTarget( - rows: NormalizedStoryScriptRow[], - targetCount = 18, -): NormalizedStoryScriptRow[] { - if (rows.length === 0) return []; - if (rows.length >= targetCount) { - return rows.slice(0, targetCount).map((row, index) => ({ - ...row, - shot: String(index + 1).padStart(2, "0"), - })); - } - - const baseCount = Math.floor(targetCount / rows.length); - const extraCount = targetCount % rows.length; - const expanded: NormalizedStoryScriptRow[] = []; - - rows.forEach((row, rowIndex) => { - const segmentCount = baseCount + (rowIndex < extraCount ? 1 : 0); - for (let segmentIndex = 0; segmentIndex < segmentCount; segmentIndex += 1) { - const phase = - storyExpansionPhases[ - Math.min( - storyExpansionPhases.length - 1, - Math.floor((segmentIndex / Math.max(segmentCount, 1)) * storyExpansionPhases.length), - ) - ]; - expanded.push({ - ...row, - shot: String(expanded.length + 1).padStart(2, "0"), - sceneDescription: `${phase.description}：${row.sceneDescription}`, - shotSize: row.shotSize || phase.shotSize, - lighting: row.lighting || phase.lighting, - cameraMovement: row.cameraMovement || phase.cameraMovement, - imagePrompt: `${row.imagePrompt}；${phase.imagePrompt}`, - motionPrompt: `${phase.motionPrompt}；${row.motionPrompt}`, - }); - } - }); - - return expanded.slice(0, targetCount); -} - -function normalizeCharacters(value: unknown) { - if (!Array.isArray(value)) return []; - return value - .map((item) => { - const raw = - item && typeof item === "object" && !Array.isArray(item) - ? (item as Record) - : {}; - const name = normalizeText(raw.name || raw.title); - const description = normalizeText(raw.description || raw.summary); - const prompt = normalizeText(raw.prompt); - if (!name && !description && !prompt) return null; - return { - name: name || "未命名角色", - description, - prompt, - look: normalizeText(raw.look) || undefined, - wardrobe: normalizeText(raw.wardrobe) || undefined, - signatureAction: normalizeText(raw.signatureAction) || undefined, - traits: normalizeStringList(raw.traits, 6), - }; - }) - .filter((item): item is NonNullable => Boolean(item)) - .slice(0, 8); -} - -function normalizeScenes(value: unknown) { - if (!Array.isArray(value)) return []; - return value - .map((item) => { - const raw = - item && typeof item === "object" && !Array.isArray(item) - ? (item as Record) - : {}; - const name = normalizeText(raw.name || raw.title); - const description = normalizeText(raw.description || raw.summary); - const prompt = normalizeText(raw.prompt); - if (!name && !description && !prompt) return null; - return { - name: name || "未命名场景", - description, - prompt, - mood: normalizeText(raw.mood) || undefined, - lighting: normalizeText(raw.lighting) || undefined, - props: normalizeStringList(raw.props, 6), - }; - }) - .filter((item): item is NonNullable => Boolean(item)) - .slice(0, 8); -} - -function buildVideoAnalysisPrompt(params: { - nodeTitle?: string; - prompt: string; - frameCount: number; - imageCount: number; - videoCount: number; -}) { - return [ - "你是视频画布工作流里的视频分析节点，需要为后续分镜和生成节点提供结构化输出。", - params.nodeTitle ? `当前节点：${params.nodeTitle}` : "", - `上游参考图数量：${params.imageCount}`, - `上游视频数量：${params.videoCount}`, - `已提供按时间顺序抽取的视频关键帧数量：${params.frameCount}`, - params.prompt ? `上游脚本/说明：\n${params.prompt}` : "上游没有额外脚本，请重点依据参考画面分析。", - "", - "请只输出 JSON，不要输出 markdown，不要输出解释。", - "JSON 结构如下：", - `{ - "summary": "一句话总结当前视频/镜头风格，不超过120字", - "highlights": ["3-5条镜头分析要点"], - "analysisRows": [ - { - "shot": "1", - "startTime": "0", - "endTime": "5", - "duration": "5", - "sceneDescription": "画面描述，说明主体、环境、动作、构图和色调", - "narrative": "叙事内容，说明这一镜头承担的剧情/情绪/信息作用", - "shotSize": "景别，例如中景至近景", - "cameraAngle": "摄影机角度，例如平视", - "cameraMovement": "摄影机运动，例如缓慢推镜头 (Slow Zoom In)", - "focusDepth": "焦距与景深，例如浅景深", - "lighting": "光线，例如冷调低沉照明，屏幕光作为侧逆光", - "backgroundMusic": "这一段适合的背景音乐氛围", - "voiceAndSfx": "人声/环境声/音效建议，没有就写无", - "imagePrompt": "后续生成关键帧时可直接使用的英文图像提示词", - "motionPrompt": "后续生成视频时可直接使用的英文运动提示词", - "keyframeIndex": 1 - } - ], - "shotIdeas": [ - { "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" } - ], - "visualMotifs": ["2-4个可复用的视觉母题"], - "recommendedNextSteps": ["2-4条建议的下游动作"] -}`, - "analysisRows 请按镜头时间顺序输出；短视频可以输出 1 行，复杂视频输出 2-6 行。", - "每行必须覆盖镜号、开始时间、结束时间、时长、画面描述、叙事内容、景别、摄影机角度、摄影机运动、焦距与景深、光线、背景音乐、人声/音效、图像生成提示词、视频运动提示词和关键帧。", - "imagePrompt 和 motionPrompt 优先使用英文；其他字段使用中文；keyframeIndex 使用 1 开始的序号，对应你看到的关键帧顺序。", - "如果没有真实视频帧，只能依据参考图和脚本推断，也要正常输出，但措辞要稳健。", - ] - .filter(Boolean) - .join("\n"); -} - -async function getVideoDurationSeconds(videoUrl: string) { - try { - const { stdout } = await execFileAsync( - "ffprobe", - [ - "-v", - "error", - "-show_entries", - "format=duration", - "-of", - "default=noprint_wrappers=1:nokey=1", - videoUrl, - ], - { - timeout: VIDEO_ANALYSIS_TIMEOUT_MS, - maxBuffer: 1024 * 1024, - }, - ); - const duration = Number.parseFloat(String(stdout || "").trim()); - return Number.isFinite(duration) && duration > 0 ? duration : null; - } catch { - return null; - } -} - -function buildVideoFrameTimestamps(durationSeconds: number | null, frameCount: number) { - const safeCount = Math.max(1, frameCount); - if (!durationSeconds || !Number.isFinite(durationSeconds) || durationSeconds <= 0.5) { - return Array.from({ length: safeCount }).map((_, index) => - Number((0.3 + index * 0.45).toFixed(2)), - ); - } - const start = Math.min(0.35, durationSeconds * 0.12); - const end = Math.max(start + 0.6, durationSeconds * 0.88); - if (safeCount === 1) return [Number(((start + end) / 2).toFixed(2))]; - const step = (end - start) / (safeCount - 1); - return Array.from({ length: safeCount }).map((_, index) => - Number((start + step * index).toFixed(2)), - ); -} - -async function downscaleVideoFrameForVision(inputBuffer: Buffer) { - return sharp(inputBuffer, { failOn: "none" }) - .rotate() - .resize({ - width: VIDEO_ANALYSIS_MAX_SIDE, - height: VIDEO_ANALYSIS_MAX_SIDE, - fit: "inside", - withoutEnlargement: true, - }) - .jpeg({ quality: 82, mozjpeg: true }) - .toBuffer(); -} - -async function extractVideoFrameDataUrls(videoUrl: string, frameCount = VIDEO_ANALYSIS_FRAME_COUNT) { - const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-")); - try { - const durationSeconds = await getVideoDurationSeconds(videoUrl); - const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount); - const frames = await Promise.all( - timestamps.map(async (timestamp, index) => { - const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`); - await execFileAsync( - "ffmpeg", - [ - "-hide_banner", - "-loglevel", - "error", - "-ss", - String(timestamp), - "-i", - videoUrl, - "-frames:v", - "1", - "-q:v", - "4", - "-y", - outputPath, - ], - { - timeout: VIDEO_ANALYSIS_TIMEOUT_MS, - maxBuffer: 2 * 1024 * 1024, - }, - ); - const buffer = await readFile(outputPath); - const processed = await downscaleVideoFrameForVision(buffer); - return `data:image/jpeg;base64,${processed.toString("base64")}`; - }), - ); - return frames.filter(Boolean); - } catch (error) { - console.warn("[Canvas Workflow Analyze] extract video frames failed:", { - message: error instanceof Error ? error.message : String(error), - videoUrl, - }); - return []; - } finally { - await rm(tempDir, { force: true, recursive: true }).catch(() => undefined); - } -} - -function buildCharacterScenePrompt(params: { - nodeTitle?: string; - prompt: string; - imageCount: number; -}) { - return [ - "你是视频画布工作流里的角色场景提取节点，需要从脚本、设定和参考图里提取角色与场景资产。", - params.nodeTitle ? `当前节点：${params.nodeTitle}` : "", - `参考图数量：${params.imageCount}`, - params.prompt ? `输入内容：\n${params.prompt}` : "没有额外脚本，请根据参考图提取角色和场景线索。", - "", - "请只输出 JSON，不要输出 markdown，不要输出解释。", - "JSON 结构如下：", - `{ - "summary": "一句话总结当前故事世界与资产线索", - "characters": [ - { - "name": "角色名", - "description": "角色外观与身份简介", - "look": "外貌特征", - "wardrobe": "服装/道具", - "traits": ["性格关键词"], - "signatureAction": "代表动作", - "prompt": "可直接用于生成角色图/角色视频的提示词" - } - ], - "scenes": [ - { - "name": "场景名", - "description": "空间、布景和氛围简介", - "mood": "情绪与气氛", - "lighting": "布光特征", - "props": ["核心道具"], - "prompt": "可直接用于生成场景图/场景视频的提示词" - } - ], - "relationships": ["角色与场景或角色之间的关系"], - "recommendedNextSteps": ["建议连接的后续节点动作"] -}`, - "角色和场景都要尽量具体，避免空泛形容词。", - ] - .filter(Boolean) - .join("\n"); -} - -function buildAgentPlanPrompt(params: { - nodeTitle?: string; - prompt: string; - imageCount: number; - videoCount: number; -}) { - return [ - "你是视频画布里的自定义代理节点，需要把当前创作目标拆成可执行的工作流计划。", - params.nodeTitle ? `当前节点：${params.nodeTitle}` : "", - `参考图数量：${params.imageCount}`, - `视频数量：${params.videoCount}`, - params.prompt ? `上游需求：\n${params.prompt}` : "没有额外脚本，请基于当前上下文给出稳妥计划。", - "", - "请只输出 JSON，不要输出 markdown，不要输出解释。", - "JSON 结构如下：", - `{ - "summary": "一句话说明这个代理链的目标", - "agentPlan": ["3-6个执行步骤"], - "deliverables": ["2-4个预期交付物"], - "recommendedNodes": ["建议继续连接的节点类型或能力名"] -}`, - "步骤要偏工作流视角，而不是泛泛而谈。", - ] - .filter(Boolean) - .join("\n"); -} - -function buildStoryScriptPrompt(params: { - nodeTitle?: string; - prompt: string; - imageCount: number; - videoCount: number; -}) { - return [ - "你是视频画布里的故事脚本生成节点，需要把用户输入、参考图和参考视频整理成可直接用于后续分镜/生图/生视频的结构化脚本表。", - params.nodeTitle ? `当前节点：${params.nodeTitle}` : "", - `参考图数量：${params.imageCount}`, - `视频数量：${params.videoCount}`, - params.prompt ? `输入内容：\n${params.prompt}` : "没有额外脚本，请生成一个节奏清晰的短视频脚本。", - "", - "请只输出 JSON，不要输出 markdown，不要输出解释。", - "JSON 结构如下：", - `{ - "summary": "一句话概括故事主题、情绪和视频方向，不超过120字", - "analysisRows": [ - { - "shot": "01", - "duration": "5s", - "scene": "场景名或地点，如 盛唐·金銮殿·日", - "sceneDescription": "画面内容、人物动作和剧情推进，不要重复 scene 字段", - "shotSize": "景别，如中景、特写、全景、大远景", - "lighting": "光影氛围，如暖金主光带阴影，气氛紧张压抑", - "dialogue": "对白或旁白，没有就写 +", - "backgroundMusic": "背景音乐或情绪氛围，没有就写无", - "voiceAndSfx": "环境声、动作声或音效，没有就写无", - "cameraMovement": "运镜，如缓慢推镜头、固定、横移、上摇、环绕轨道跟拍", - "imagePrompt": "后续生成该镜头关键帧时可直接使用的中文图像提示词", - "motionPrompt": "后续生成视频时可直接使用的中文视频运动提示词" - } - ], - "assetCandidates": { - "style": "全局视觉风格，如 盛唐·国风二次元", - "characters": ["需要生成的角色资产名"], - "scenes": ["需要生成的场景资产名"], - "props": ["需要生成的道具资产名"] - }, - "shotIdeas": [ - { "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" } - ], - "recommendedNextSteps": ["2-4条建议的下游动作"] -}`, - "analysisRows 必须直接输出 18 行，按镜头时间顺序编号 01-18；每行默认 5s 或 6s，剧情高潮可用 6s。", - "每行要覆盖竞品脚本表同等信息：镜号、时长、画面描述、景别、光影氛围、对白·旁白、音效、运镜、最终图像提示词和视频运动提示词。", - "画面描述中用 @角色名 或 @场景/道具名 标出可复用资产，例如 @沈昭昭、@金銮殿、@侍卫长刀。", - "imagePrompt 要像可直接生成分镜图的中文提示词，包含主体、环境、构图、光影、风格与一致性要求；motionPrompt 要按起始状态、动作过程、结束状态、音效组织。", - "assetCandidates 根据 18 镜头抽取 3-6 个角色、2-5 个场景和 1-4 个道具，方便后续准备资产。", - "每行要像专业视频画布脚本表：场景具体、画面可生成、声音可执行、镜头节奏明确，不要只写剧情摘要。", - ] - .filter(Boolean) - .join("\n"); -} - export async function POST(request: Request) { - try { - const body = await request.json().catch(() => ({})); - const parsed = requestSchema.safeParse(body); - if (!parsed.success) { - return NextResponse.json( - { success: false, error: "请求参数不合法" }, - { status: 400 } - ); - } - - const prompt = normalizeText(parsed.data.prompt); - const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4); - const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4); - const extractedVideoFrames = - parsed.data.mode === "video-analysis" && videoUrls.length > 0 - ? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT) - : []; - const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model); - if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) { - return NextResponse.json( - { success: false, error: "请至少提供脚本、图片或视频上下文" }, - { status: 400 } - ); - } - - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); - } - - const [allModels, agentModel] = await Promise.all([ - getEnabledAiModelsWithProvider(false), - getAgentAiModelWithProvider(), - ]); - const compatibleModels = allModels.filter( - (item) => !item.model.isSubModel && item.model.isEnabled - ); - - const preferredModel = - preferredModelRef - ? compatibleModels.find( - (item) => - item.model.id === preferredModelRef || - item.model.modelId === preferredModelRef, - ) - : null; - - const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0; - - let selectedModel = - preferredModel && (!hasVisionInputs || supportsVision(preferredModel.model)) - ? preferredModel - : hasVisionInputs && - agentModel && - supportsVision(agentModel.model) - ? agentModel - : hasVisionInputs - ? compatibleModels.find((item) => supportsVision(item.model)) - : agentModel || compatibleModels[0]; - - if (!selectedModel) { - selectedModel = compatibleModels[0]; - } - - if (!selectedModel) { - return NextResponse.json( - { success: false, error: "当前站点没有可用的分析模型" }, - { status: 500 } - ); - } - - const useVision = hasVisionInputs && supportsVision(selectedModel.model); - const codingModelId = getAiModelAgentCodingModelId(selectedModel.model); - const model = await getLanguageModelById(selectedModel.model.id, codingModelId); - if (!model) { - return NextResponse.json( - { success: false, error: "分析模型初始化失败" }, - { status: 500 } - ); - } - - const promptText = - parsed.data.mode === "video-analysis" - ? buildVideoAnalysisPrompt({ - frameCount: extractedVideoFrames.length, - nodeTitle: normalizeText(parsed.data.nodeTitle), - prompt, - imageCount: imageUrls.length, - videoCount: videoUrls.length, - }) - : parsed.data.mode === "character-scene-extraction" - ? buildCharacterScenePrompt({ - nodeTitle: normalizeText(parsed.data.nodeTitle), - prompt, - imageCount: imageUrls.length, - }) - : parsed.data.mode === "story-script" - ? buildStoryScriptPrompt({ - nodeTitle: normalizeText(parsed.data.nodeTitle), - prompt, - imageCount: imageUrls.length, - videoCount: videoUrls.length, - }) - : buildAgentPlanPrompt({ - nodeTitle: normalizeText(parsed.data.nodeTitle), - prompt, - imageCount: imageUrls.length, - videoCount: videoUrls.length, - }); - - const inlinedImageReferences = useVision - ? ( - await inlineVisionImages(imageUrls, { - maxImages: 4, - maxSide: 960, - timeoutMs: 12000, - retries: 1, - loggerPrefix: "[Canvas Workflow Analyze]", - }) - ).inlineDataUrls - : []; - const referenceImages = useVision - ? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6) - : []; - - const { text } = await generateText({ - model, - messages: [ - { - role: "user", - content: [ - { type: "text", text: promptText }, - ...referenceImages.map((image) => ({ type: "image" as const, image })), - ], - }, - ], - temperature: 0.2, - maxTokens: parsed.data.mode === "story-script" ? 4200 : parsed.data.mode === "video-analysis" ? 2200 : 1600, - } as any); - - const parsedJson = extractJsonBlock>(text) || {}; - const summary = - normalizeText(parsedJson.summary) || - normalizeText(text).slice(0, 160); - - if (parsed.data.mode === "video-analysis") { - const analysisRows = normalizeVideoAnalysisRows( - parsedJson.analysisRows, - extractedVideoFrames, - ); - const shotIdeas = - normalizeShotIdeas(parsedJson.shotIdeas).length > 0 - ? normalizeShotIdeas(parsedJson.shotIdeas) - : analysisRows.map((row, index) => ({ - title: `镜头 ${index + 1}`, - description: row.sceneDescription, - })); - return NextResponse.json({ - success: true, - data: { - summary, - analysisRows, - framePreviews: extractedVideoFrames, - highlights: normalizeStringList(parsedJson.highlights, 6), - shotIdeas, - visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6), - recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), - model: selectedModel.model.displayName, - raw: text, - }, - }); - } - - if (parsed.data.mode === "character-scene-extraction") { - return NextResponse.json({ - success: true, - data: { - summary, - characters: normalizeCharacters(parsedJson.characters), - scenes: normalizeScenes(parsedJson.scenes), - relationships: normalizeStringList(parsedJson.relationships, 8), - recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), - model: selectedModel.model.displayName, - raw: text, - }, - }); - } - - if (parsed.data.mode === "story-script") { - const analysisRows = expandStoryScriptRowsToTarget(normalizeStoryScriptRows( - parsedJson.analysisRows || parsedJson.storyRows || parsedJson.rows || parsedJson.shots, - )); - return NextResponse.json({ - success: true, - data: { - summary, - analysisRows, - shotIdeas: - normalizeShotIdeas(parsedJson.shotIdeas).length > 0 - ? normalizeShotIdeas(parsedJson.shotIdeas) - : analysisRows.map((row, index) => ({ - title: `镜头 ${index + 1}`, - description: row.sceneDescription, - })), - recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), - model: selectedModel.model.displayName, - raw: text, - }, - }); - } - - return NextResponse.json({ - success: true, - data: { - summary, - agentPlan: normalizeStringList(parsedJson.agentPlan, 8), - deliverables: normalizeStringList(parsedJson.deliverables, 6), - recommendedNodes: normalizeStringList(parsedJson.recommendedNodes, 6), - model: selectedModel.model.displayName, - raw: text, - }, - }); - } catch (error) { - console.error("[Canvas Workflow Analyze] Error:", error); - return NextResponse.json( - { - success: false, - error: error instanceof Error ? error.message : "工作流分析失败", - }, - { status: 500 } - ); - } + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const body = await request.json().catch(() => ({})); + return executeWorkflowAnalysisResponse(body, request.signal); } diff --git a/app/api/canvas/world/tasks/[taskId]/route.ts b/app/api/canvas/world/tasks/[taskId]/route.ts index a96b2948f..057c03d63 100644 --- a/app/api/canvas/world/tasks/[taskId]/route.ts +++ b/app/api/canvas/world/tasks/[taskId]/route.ts @@ -190,6 +190,48 @@ function assertArchivedAssetBuffer(params: { } } +async function verifyArchivedAssetUpload(params: { + url: string; + expectedBytes: number; + validateAs?: ArchiveValidationKind; +}) { + const headResponse = await fetch(params.url, { + method: "HEAD", + cache: "no-store", + }); + if (!headResponse.ok) { + throw new Error(`归档文件校验失败（HTTP ${headResponse.status}）`); + } + + const archivedLength = Number(headResponse.headers.get("content-length")); + if ( + Number.isFinite(archivedLength) && + archivedLength > 0 && + archivedLength !== params.expectedBytes + ) { + throw new Error( + `归档文件大小不一致（源文件 ${params.expectedBytes}，归档文件 ${archivedLength}）` + ); + } + + if (!params.validateAs) return; + + const rangeResponse = await fetch(params.url, { + method: "GET", + headers: { Range: "bytes=0-63" }, + cache: "no-store", + }); + if (!rangeResponse.ok) { + throw new Error(`归档文件头校验失败（HTTP ${rangeResponse.status}）`); + } + const headerBuffer = await rangeResponse.arrayBuffer(); + assertArchivedAssetBuffer({ + buffer: headerBuffer, + contentType: rangeResponse.headers.get("content-type"), + validateAs: params.validateAs, + }); +} + async function archiveRemoteAssetIfNeeded(params: { cacheKey: string; sourceUrl?: string | null; @@ -249,6 +291,11 @@ async function archiveRemoteAssetIfNeeded(params: { ); const filename = `${params.filenamePrefix}.${ext}`; const uploaded = await uploadFile(filename, assetBuffer); + await verifyArchivedAssetUpload({ + url: uploaded.url, + expectedBytes: assetBuffer.byteLength, + validateAs: params.validateAs, + }); cache.set(params.cacheKey, { sourceUrl, archivedUrl: uploaded.url, @@ -299,7 +346,8 @@ async function archiveRemoteAssetMapIfNeeded(params: { ) ).filter((entry): entry is readonly [string, string] => Boolean(entry)); - return archivedEntries.length > 0 ? Object.fromEntries(archivedEntries) : null; + if (archivedEntries.length !== entries.length) return null; + return Object.fromEntries(archivedEntries); } function getStringValue(...values: unknown[]) { @@ -368,33 +416,96 @@ async function updatePersistedWorldAsset(params: { function extractWorldId(operation: WorldLabsOperation) { const response = getObjectValue(operation.response); const metadata = getObjectValue(operation.metadata); - return getStringValue( - response?.world_id, - response?.worldId, - metadata?.world_id, - metadata?.worldId - ); + const nestedWorldId = + getStringValue(response?.world_id, response?.worldId) || + findNestedString(response, ["world_id", "worldId"]); + const metadataWorldId = + getStringValue(metadata?.world_id, metadata?.worldId) || + findNestedString(metadata, ["world_id", "worldId"]); + + return getStringValue(nestedWorldId, metadataWorldId); +} + +function isWorldLikePayload(value: unknown): value is WorldLabsWorld { + const record = getObjectValue(value); + if (!record) return false; + + const hasCoreWorldField = + typeof record.world_id === "string" || + typeof record.world_marble_url === "string" || + typeof record.display_name === "string" || + typeof record.worldId === "string"; + const hasPossibleWorldContainerFields = record.assets || record.permission || record.tags || record.world_prompt; + const hasModelAliasOnly = + typeof record.model === "string" && /world/i.test(record.model); + + return Boolean(hasCoreWorldField || hasPossibleWorldContainerFields || hasModelAliasOnly); +} + +function extractWorldCandidate(value: unknown, depth = 0): WorldLabsWorld | null { + if (depth > 8) return null; + const record = getObjectValue(value); + if (!record) return null; + + if (isWorldLikePayload(record)) return record; + + const childCandidates: Array | null> = [ + getObjectValue(record.world), + getObjectValue(record.result), + getObjectValue(record.output), + getObjectValue(record.data), + getObjectValue(record.payload), + ]; + + for (const child of childCandidates) { + if (!child) continue; + if (isWorldLikePayload(child)) return child; + const nested = extractWorldCandidate(child, depth + 1); + if (nested) return nested; + } + + for (const child of Object.values(record)) { + const nested = extractWorldCandidate(child, depth + 1); + if (nested) return nested; + } + + return null; } function extractWorldFromOperation(operation: WorldLabsOperation) { - const response = operation.response; - if (response && typeof response === "object" && "world_marble_url" in (response as Record)) { - return response as WorldLabsWorld; - } - return null; + return extractWorldCandidate(operation.response); } function hasUsableWorldAssets(world?: WorldLabsWorld | null) { const assets = world?.assets || null; if (!assets) return false; - const spzUrls = assets.splats?.spz_urls; + const meshRecord = getObjectValue(assets.mesh); + const imageryRecord = getObjectValue(assets.imagery); + const splatsRecord = getObjectValue(assets.splats); + const spzUrls = + splatsRecord?.spz_urls || splatsRecord?.spzUrls || splatsRecord?.splat_urls; return Boolean( getStringValue(assets.thumbnail_url) || - getStringValue(assets.imagery?.pano_url) || - getStringValue(assets.mesh?.collider_mesh_url) || + getStringValue( + imageryRecord?.pano_url, + imageryRecord?.panoUrl, + imageryRecord?.panorama_url, + imageryRecord?.panoramaUrl + ) || + getStringValue( + meshRecord?.collider_mesh_url, + meshRecord?.colliderMeshUrl, + meshRecord?.collider_mesh, + meshRecord?.mesh_url, + meshRecord?.meshUrl, + meshRecord?.model_url, + meshRecord?.modelUrl + ) || (spzUrls && typeof spzUrls === "object" && - Object.values(spzUrls).some((value) => typeof value === "string" && value.trim())) + Object.values(spzUrls).some( + (value) => typeof value === "string" && value.trim() + )) ); } @@ -667,185 +778,224 @@ export async function GET(request: Request, { params }: RouteParams) { }); providerForLog = config.providerId || null; const modelAliasConfig = await getThreeDModelAliasConfigForRuntime(); - const persistedAssetPayload = + let persistedAssetPayload = getObjectValue(persistedWorldAsset?.assetPayload) || {}; - const shouldUseExternalTaskResultApi = + let shouldUseExternalTaskResultApi = isWorldLabsExternalTaskApi(config.baseUrl) && persistedAssetPayload.externalTaskApi !== false && persistedAssetPayload.marbleApi !== true; if (shouldUseExternalTaskResultApi) { - const externalPayload = await worldLabsRequest< - WorldLabsExternalTaskResponse - >(config, { - method: "GET", - path: `/api/external/task/result?taskId=${encodeURIComponent(normalizedTaskId)}`, - timeoutMs: resolveWorldLabsStatusTimeoutMs(), - }); - const externalData = - unwrapExternalTaskData(externalPayload); - const externalStatusText = getStringValue(externalData.status) || "running"; - const externalStatus = normalizeExternalTaskStatus( - externalStatusText, - externalData.done === true - ); - const externalAssetPayloadBase = { - ...persistedAssetPayload, - providerId: config.providerId || null, - providerName: config.providerName || null, - externalTaskApi: true, - externalTaskStatus: externalStatusText, - requestId: getStringValue(externalPayload.requestId, externalPayload.request_id), - }; - - if (externalStatus === "running") { - return NextResponse.json({ - success: true, - data: { - taskId: normalizedTaskId, - type: "world", - status: "running", - progress: null, - output: externalData, - assetPayload: externalAssetPayloadBase, - }, + try { + const externalPayload = await worldLabsRequest< + WorldLabsExternalTaskResponse + >(config, { + method: "GET", + path: `/api/external/task/result?taskId=${encodeURIComponent(normalizedTaskId)}`, + timeoutMs: resolveWorldLabsStatusTimeoutMs(), }); - } - - if (externalStatus === "failed") { - const errorMessage = extractExternalTaskErrorMessage( - externalData, - externalPayload + const externalData = + unwrapExternalTaskData(externalPayload); + const externalStatusText = getStringValue(externalData.status) || "running"; + const externalStatus = normalizeExternalTaskStatus( + externalStatusText, + externalData.done === true ); + const externalAssetPayloadBase = { + ...persistedAssetPayload, + providerId: config.providerId || null, + providerName: config.providerName || null, + externalTaskApi: true, + externalTaskStatus: externalStatusText, + requestId: getStringValue(externalPayload.requestId, externalPayload.request_id), + }; + + if (externalStatus === "running") { + return NextResponse.json({ + success: true, + data: { + taskId: normalizedTaskId, + type: "world", + status: "running", + progress: null, + output: externalData, + assetPayload: externalAssetPayloadBase, + }, + }); + } + + if (externalStatus === "failed") { + const errorMessage = extractExternalTaskErrorMessage( + externalData, + externalPayload + ); + await updatePersistedWorldAsset({ + id: persistedWorldAsset?.id, + status: "failed", + progress: null, + error: errorMessage, + assetPayload: externalAssetPayloadBase, + }); + logger.warn("external task failed status", { + elapsedMs: Date.now() - startedAt, + taskIdLength: normalizedTaskId.length, + providerId: config.providerId || null, + error: errorMessage, + }); + return NextResponse.json({ + success: true, + data: { + taskId: normalizedTaskId, + type: "world", + status: "failed", + progress: null, + errorMessage, + output: externalData, + assetPayload: externalAssetPayloadBase, + }, + }); + } + const extractedAssetPayload = extractExternalWorldAssetPayload(externalData); + const rawMeshUrl = extractedAssetPayload.meshUrl; + const rawThumbnailUrl = extractedAssetPayload.thumbnailUrl; + const rawPanoUrl = extractedAssetPayload.panoUrl; + const stableWorldUrl: string | null = null; + const stableModelUrl = await archiveRemoteAssetIfNeeded({ + cacheKey: `worldlabs:${normalizedTaskId}:mesh`, + sourceUrl: rawMeshUrl, + filenamePrefix: `worldlabs-mesh-${normalizedTaskId}`, + fallbackExtension: "glb", + validateAs: "glb", + }); + const stableThumbnailUrl = await archiveRemoteAssetIfNeeded({ + cacheKey: `worldlabs:${normalizedTaskId}:thumb`, + sourceUrl: rawThumbnailUrl, + filenamePrefix: `worldlabs-thumb-${normalizedTaskId}`, + fallbackExtension: "jpg", + }); + const stablePanoUrl = await archiveRemoteAssetIfNeeded({ + cacheKey: `worldlabs:${normalizedTaskId}:pano`, + sourceUrl: rawPanoUrl, + filenamePrefix: `worldlabs-pano-${normalizedTaskId}`, + fallbackExtension: "jpg", + }); + const stableSpzUrls = await archiveRemoteAssetMapIfNeeded({ + cacheKeyPrefix: `worldlabs:${normalizedTaskId}:spz`, + sourceMap: extractedAssetPayload.spzUrls, + filenamePrefix: `worldlabs-splat-${normalizedTaskId}`, + fallbackExtension: "spz", + validateAs: "spz", + }); + const sourceSpzEntryCount = Object.keys( + extractedAssetPayload.spzUrls || {} + ).length; + const stableSpzEntryCount = Object.keys(stableSpzUrls || {}).length; + const assetsArchivingPending = Boolean( + (rawMeshUrl && !stableModelUrl) || + (rawThumbnailUrl && !stableThumbnailUrl) || + (rawPanoUrl && !stablePanoUrl) || + (sourceSpzEntryCount > 0 && stableSpzEntryCount !== sourceSpzEntryCount) + ); + const storageArchivedAt = assetsArchivingPending + ? null + : new Date().toISOString(); + const assetPayload = { + ...externalAssetPayloadBase, + worldId: extractedAssetPayload.worldId, + worldUrl: stableWorldUrl, + displayName: extractedAssetPayload.displayName, + model: resolveThreeDModelAliasId( + extractedAssetPayload.model, + modelAliasConfig.worldModels + ), + caption: extractedAssetPayload.caption, + thumbnailUrl: stableThumbnailUrl || extractedAssetPayload.thumbnailUrl, + panoUrl: stablePanoUrl || extractedAssetPayload.panoUrl, + meshUrl: stableModelUrl, + spzUrls: stableSpzUrls, + semanticsMetadata: extractedAssetPayload.semanticsMetadata, + spzArchiveValidatedAt: stableSpzUrls ? storageArchivedAt : null, + storageArchivedAt, + assetsArchivingPending, + sourceWorldUrl: extractedAssetPayload.worldUrl || null, + sourceMeshUrl: + rawMeshUrl && stableModelUrl && rawMeshUrl !== stableModelUrl + ? rawMeshUrl + : null, + sourceThumbnailUrl: + rawThumbnailUrl && stableThumbnailUrl && rawThumbnailUrl !== stableThumbnailUrl + ? rawThumbnailUrl + : null, + sourcePanoUrl: + rawPanoUrl && stablePanoUrl && rawPanoUrl !== stablePanoUrl + ? rawPanoUrl + : null, + sourceSpzUrls: + extractedAssetPayload.spzUrls && + stableSpzUrls !== extractedAssetPayload.spzUrls + ? extractedAssetPayload.spzUrls + : null, + }; + await updatePersistedWorldAsset({ id: persistedWorldAsset?.id, - status: "failed", - progress: null, - error: errorMessage, - assetPayload: externalAssetPayloadBase, + status: "success", + progress: 100, + modelUrl: stableModelUrl, + previewImageUrl: stableThumbnailUrl || stablePanoUrl || null, + error: null, + assetPayload, }); - logger.warn("external task failed status", { + + logger.info("external task success status", { elapsedMs: Date.now() - startedAt, taskIdLength: normalizedTaskId.length, providerId: config.providerId || null, - error: errorMessage, + hasMesh: Boolean(stableModelUrl), + hasPano: Boolean(stablePanoUrl), + hasThumbnail: Boolean(stableThumbnailUrl), + hasSplat: Boolean(stableSpzUrls && Object.keys(stableSpzUrls).length > 0), + assetsArchivingPending, }); + return NextResponse.json({ success: true, data: { taskId: normalizedTaskId, type: "world", - status: "failed", - progress: null, - errorMessage, + status: "success", + progress: 100, output: externalData, - assetPayload: externalAssetPayloadBase, + modelUrl: stableModelUrl, + previewImageUrl: stableThumbnailUrl || stablePanoUrl || null, + worldId: assetPayload.worldId, + worldUrl: stableWorldUrl, + panoUrl: stablePanoUrl, + caption: assetPayload.caption, + assetPayload, }, }); + } catch (error) { + const fallbackAssetPayload = { + ...persistedAssetPayload, + providerId: config.providerId || null, + providerName: config.providerName || null, + marbleApi: true, + externalTaskApi: false, + }; + shouldUseExternalTaskResultApi = false; + persistedAssetPayload = fallbackAssetPayload; + await updatePersistedWorldAsset({ + id: persistedWorldAsset?.id, + status: "running", + assetPayload: fallbackAssetPayload, + }).catch(() => {}); + logger.warn("external task result failed, fallback to marble status", { + elapsedMs: Date.now() - startedAt, + taskIdLength: normalizedTaskId.length, + providerId: config.providerId || null, + error: error instanceof Error ? error.message : String(error), + }); } - - const extractedAssetPayload = extractExternalWorldAssetPayload(externalData); - const rawMeshUrl = extractedAssetPayload.meshUrl; - const rawThumbnailUrl = extractedAssetPayload.thumbnailUrl; - const rawPanoUrl = extractedAssetPayload.panoUrl; - const stableWorldUrl: string | null = null; - const stableModelUrl = await archiveRemoteAssetIfNeeded({ - cacheKey: `worldlabs:${normalizedTaskId}:mesh`, - sourceUrl: rawMeshUrl, - filenamePrefix: `worldlabs-mesh-${normalizedTaskId}`, - fallbackExtension: "glb", - validateAs: "glb", - }); - const stableThumbnailUrl = await archiveRemoteAssetIfNeeded({ - cacheKey: `worldlabs:${normalizedTaskId}:thumb`, - sourceUrl: rawThumbnailUrl, - filenamePrefix: `worldlabs-thumb-${normalizedTaskId}`, - fallbackExtension: "jpg", - }); - const stablePanoUrl = await archiveRemoteAssetIfNeeded({ - cacheKey: `worldlabs:${normalizedTaskId}:pano`, - sourceUrl: rawPanoUrl, - filenamePrefix: `worldlabs-pano-${normalizedTaskId}`, - fallbackExtension: "jpg", - }); - const stableSpzUrls = await archiveRemoteAssetMapIfNeeded({ - cacheKeyPrefix: `worldlabs:${normalizedTaskId}:spz`, - sourceMap: extractedAssetPayload.spzUrls, - filenamePrefix: `worldlabs-splat-${normalizedTaskId}`, - fallbackExtension: "spz", - validateAs: "spz", - }); - const assetPayload = { - ...externalAssetPayloadBase, - worldId: extractedAssetPayload.worldId, - worldUrl: stableWorldUrl, - displayName: extractedAssetPayload.displayName, - model: resolveThreeDModelAliasId( - extractedAssetPayload.model, - modelAliasConfig.worldModels - ), - caption: extractedAssetPayload.caption, - thumbnailUrl: stableThumbnailUrl || extractedAssetPayload.thumbnailUrl, - panoUrl: stablePanoUrl || extractedAssetPayload.panoUrl, - meshUrl: stableModelUrl, - spzUrls: stableSpzUrls, - semanticsMetadata: extractedAssetPayload.semanticsMetadata, - storageArchivedAt: new Date().toISOString(), - sourceWorldUrl: extractedAssetPayload.worldUrl || null, - sourceMeshUrl: - rawMeshUrl && stableModelUrl && rawMeshUrl !== stableModelUrl - ? rawMeshUrl - : null, - sourceThumbnailUrl: - rawThumbnailUrl && stableThumbnailUrl && rawThumbnailUrl !== stableThumbnailUrl - ? rawThumbnailUrl - : null, - sourcePanoUrl: - rawPanoUrl && stablePanoUrl && rawPanoUrl !== stablePanoUrl - ? rawPanoUrl - : null, - sourceSpzUrls: - extractedAssetPayload.spzUrls && stableSpzUrls !== extractedAssetPayload.spzUrls - ? extractedAssetPayload.spzUrls - : null, - }; - - await updatePersistedWorldAsset({ - id: persistedWorldAsset?.id, - status: "success", - progress: 100, - modelUrl: stableModelUrl, - previewImageUrl: stableThumbnailUrl || stablePanoUrl || null, - error: null, - assetPayload, - }); - - logger.info("external task success status", { - elapsedMs: Date.now() - startedAt, - taskIdLength: normalizedTaskId.length, - providerId: config.providerId || null, - hasMesh: Boolean(stableModelUrl), - hasPano: Boolean(stablePanoUrl), - hasThumbnail: Boolean(stableThumbnailUrl), - hasSplat: Boolean(stableSpzUrls && Object.keys(stableSpzUrls).length > 0), - }); - - return NextResponse.json({ - success: true, - data: { - taskId: normalizedTaskId, - type: "world", - status: "success", - progress: 100, - output: externalData, - modelUrl: stableModelUrl, - previewImageUrl: stableThumbnailUrl || stablePanoUrl || null, - worldId: assetPayload.worldId, - worldUrl: stableWorldUrl, - panoUrl: stablePanoUrl, - caption: assetPayload.caption, - assetPayload, - }, - }); } const operation = await worldLabsRequest(config, { @@ -908,22 +1058,68 @@ export async function GET(request: Request, { params }: RouteParams) { let world = extractWorldFromOperation(operation); const worldId = extractWorldId(operation); if ((!world || !hasUsableWorldAssets(world)) && worldId) { - const detailedWorld = await worldLabsRequest(config, { - method: "GET", - path: `/marble/v1/worlds/${encodeURIComponent(worldId)}`, - timeoutMs: resolveWorldLabsStatusTimeoutMs(), - }); - world = world - ? { - ...world, - ...detailedWorld, - assets: detailedWorld.assets || world.assets || null, - } - : detailedWorld; + try { + const detailedWorld = await worldLabsRequest(config, { + method: "GET", + path: `/marble/v1/worlds/${encodeURIComponent(worldId)}`, + timeoutMs: resolveWorldLabsStatusTimeoutMs(), + }); + world = world + ? { + ...world, + ...detailedWorld, + assets: detailedWorld.assets || world.assets || null, + } + : detailedWorld; + } catch (error) { + logger.warn("world detail fetch failed, retry with running status", { + elapsedMs: Date.now() - startedAt, + taskIdLength: normalizedTaskId.length, + providerId: config.providerId || null, + worldId, + error: error instanceof Error ? error.message : "unknown", + }); + } } if (!world) { - throw new Error("World Labs 任务已完成，但未返回 world 数据"); + const pendingPayload = { + ...(getObjectValue(persistedWorldAsset?.assetPayload) || {}), + providerId: config.providerId || null, + providerName: config.providerName || null, + worldId: worldId || null, + worldUrl: null, + displayName: null, + model: null, + assetsPending: true, + operationMetadata: operation.metadata ?? null, + }; + await updatePersistedWorldAsset({ + id: persistedWorldAsset?.id, + status: "running", + progress: 95, + assetPayload: pendingPayload, + }); + logger.warn("task completed but world payload is missing", { + elapsedMs: Date.now() - startedAt, + taskIdLength: normalizedTaskId.length, + providerId: config.providerId || null, + worldId: worldId || null, + }); + return NextResponse.json({ + success: true, + data: { + taskId: normalizedTaskId, + type: "world", + status: "running", + progress: 95, + output: operation.response ?? null, + worldId: worldId || null, + worldUrl: null, + previewImageUrl: null, + assetPayload: pendingPayload, + }, + }); } if (!hasUsableWorldAssets(world)) { @@ -1039,24 +1235,15 @@ export async function GET(request: Request, { params }: RouteParams) { : 0; const hasMissingSpzArchive = rawSpzUrls.length > 0 && stableSpzEntryCount < rawSpzUrls.length; - const storageArchivedAt = - [ - [rawWorldUrl, stableWorldUrl], - [rawMeshUrl, stableModelUrl], - [rawThumbnailUrl, stableThumbnailUrl], - [rawPanoUrl, stablePanoUrl], - ].some(([rawUrl, stableUrl]) => rawUrl && stableUrl && rawUrl !== stableUrl) || - Boolean( - stableSpzUrls && - assetPayload.spzUrls && - Object.entries(stableSpzUrls).some( - ([key, value]) => - typeof assetPayload.spzUrls?.[key] === "string" && - assetPayload.spzUrls[key] !== value - ) - ) - ? new Date().toISOString() - : null; + const assetsArchivingPending = Boolean( + (rawMeshUrl && !stableModelUrl) || + (rawThumbnailUrl && !stableThumbnailUrl) || + (rawPanoUrl && !stablePanoUrl) || + hasMissingSpzArchive + ); + const storageArchivedAt = assetsArchivingPending + ? null + : new Date().toISOString(); logger.info("task success status", { elapsedMs: Date.now() - startedAt, @@ -1068,10 +1255,7 @@ export async function GET(request: Request, { params }: RouteParams) { hasThumbnail: Boolean(stableThumbnailUrl), hasSplat: Boolean(stableSpzUrls && Object.keys(stableSpzUrls).length > 0), storageArchivedAt, - assetsArchivingPending: Boolean( - (rawMeshUrl && !stableModelUrl) || - hasMissingSpzArchive - ), + assetsArchivingPending, }); const responseAssetPayload = { @@ -1081,12 +1265,9 @@ export async function GET(request: Request, { params }: RouteParams) { panoUrl: stablePanoUrl || assetPayload.panoUrl, meshUrl: stableModelUrl, spzUrls: stableSpzUrls, - spzArchiveValidatedAt: stableSpzUrls ? new Date().toISOString() : null, + spzArchiveValidatedAt: stableSpzUrls ? storageArchivedAt : null, storageArchivedAt, - assetsArchivingPending: Boolean( - (rawMeshUrl && !stableModelUrl) || - hasMissingSpzArchive - ), + assetsArchivingPending, sourceWorldUrl: rawWorldUrl || null, sourceMeshUrl: rawMeshUrl && stableModelUrl && rawMeshUrl !== stableModelUrl diff --git a/app/api/canvas/world/tasks/route.ts b/app/api/canvas/world/tasks/route.ts index ae3961682..1bb4e05df 100644 --- a/app/api/canvas/world/tasks/route.ts +++ b/app/api/canvas/world/tasks/route.ts @@ -142,7 +142,27 @@ function shouldUseExternalTaskCreateApi( config: Awaited>, mode: WorldTaskMode ) { - return isWorldLabsExternalTaskApi(config.baseUrl) && mode === "text"; + return isWorldLabsExternalTaskApi(config.baseUrl) && mode === "image"; +} + +function isWorldLabsModeUnsupportedByExternalProvider( + config: Awaited>, + mode: WorldTaskMode +) { + return isWorldLabsExternalTaskApi(config.baseUrl) && mode !== "image"; +} + +function assertWorldModeSupportedByCandidate( + config: Awaited>, + mode: WorldTaskMode, +) { + if (isWorldLabsModeUnsupportedByExternalProvider(config, mode)) { + const normalizedBaseUrl = String(config.baseUrl || "").trim() || "未知服务地址"; + const normalizedMode = String(mode || "").trim() || "未知模式"; + throw new Error( + `${normalizedBaseUrl} 的 world-generation 当前仅支持图生世界（image）。当前提交的是 ${normalizedMode}，请先切换供应商或使用图生世界。` + ); + } } function resolveOfficialMarbleModel(model?: string | null) { @@ -153,6 +173,49 @@ function resolveOfficialMarbleModel(model?: string | null) { return normalized; } +function buildExternalWorldTaskPayload( + mode: WorldTaskMode, + worldPrompt: Record, + prompt: string | null, + displayName: string, +) { + const normalizedPrompt = readOptionalTrimmedText(prompt); + if (!normalizedPrompt && mode === "text") { + throw new Error("请输入世界描述"); + } + const payload: Record = { + model: WORLDLABS_EXTERNAL_TASK_MODEL, + displayName, + }; + if (normalizedPrompt) { + payload.prompt = normalizedPrompt; + } else if (mode !== "text") { + payload.prompt = "图生世界"; + } + + if (mode === "image" || mode === "multi-image") { + payload.world_prompt = worldPrompt; + } + + return payload; +} + +function buildOfficialWorldTaskPayload( + displayName: string, + worldPrompt: Record, + officialMarbleModel: string, + body: CreateWorldTaskRequest +) { + return { + display_name: displayName, + model: officialMarbleModel, + permission: { + public: body.makePublic === true, + }, + world_prompt: worldPrompt, + }; +} + function buildWorldPrompt(body: CreateWorldTaskRequest, mode: WorldTaskMode) { const textPrompt = readOptionalTrimmedText(body.prompt); @@ -356,41 +419,73 @@ export async function POST(request: Request) { for (const candidate of candidates) { try { const useExternalTaskApi = shouldUseExternalTaskCreateApi(candidate, mode); + assertWorldModeSupportedByCandidate(candidate, mode); + let requestUsedExternalTaskApi = false; if (useExternalTaskApi) { - const prompt = readOptionalTrimmedText(body.prompt); - if (!prompt) { - throw new Error("请输入世界描述"); + const payload = buildExternalWorldTaskPayload( + mode, + worldPrompt, + readOptionalTrimmedText(body.prompt), + displayName, + ); + try { + const externalResult = await worldLabsRequest< + WorldLabsExternalTaskResponse + >(candidate, { + method: "POST", + path: "/api/external/task/generate", + timeoutMs: resolveWorldLabsCreateTimeoutMs(), + json: payload, + }); + result = normalizeExternalTaskCreateOperation(externalResult); + requestUsedExternalTaskApi = true; + } catch (error) { + const externalError = error instanceof Error ? error.message : String(error); + logger.warn("external world create failed, fallback to marble endpoint", { + providerId: candidate.providerId || null, + error: externalError, + }); + if (isCredentialOrBillingProviderError(error)) { + await reportWorldLabsProviderRuntimeFailure( + candidate.providerId, + externalError + ); + } + + result = await worldLabsRequest(candidate, { + method: "POST", + path: "/marble/v1/worlds:generate", + timeoutMs: resolveWorldLabsCreateTimeoutMs(), + json: buildOfficialWorldTaskPayload( + displayName, + worldPrompt, + officialMarbleModel, + body + ), + }); + requestUsedExternalTaskApi = false; } - const externalResult = await worldLabsRequest< - WorldLabsExternalTaskResponse - >(candidate, { - method: "POST", - path: "/api/external/task/generate", - timeoutMs: resolveWorldLabsCreateTimeoutMs(), - json: { - model: WORLDLABS_EXTERNAL_TASK_MODEL, - displayName, - prompt, - }, - }); - result = normalizeExternalTaskCreateOperation(externalResult); } else { + requestUsedExternalTaskApi = false; result = await worldLabsRequest(candidate, { method: "POST", path: "/marble/v1/worlds:generate", timeoutMs: resolveWorldLabsCreateTimeoutMs(), - json: { - display_name: displayName, - model: officialMarbleModel, - permission: { - public: body.makePublic === true, - }, - world_prompt: worldPrompt, - }, + json: buildOfficialWorldTaskPayload( + displayName, + worldPrompt, + officialMarbleModel, + body + ), }); } + + if (!result?.operation_id) { + throw new Error("创建 World Labs 世界任务返回缺少 operation_id"); + } + usedProvider = candidate; - usedExternalTaskApi = useExternalTaskApi; + usedExternalTaskApi = requestUsedExternalTaskApi; providerForLog = candidate.providerId || null; break; } catch (error) { diff --git a/app/api/enhance/route.ts b/app/api/enhance/route.ts index ea888b293..bba47dd54 100644 --- a/app/api/enhance/route.ts +++ b/app/api/enhance/route.ts @@ -1,1411 +1,15 @@ -import { NextRequest, NextResponse } from "next/server"; -import { - createMediaAssetRelation, - getAiProvidersByType, - getMediaAssetsByUserAndUrls, - getSystemSetting, - upsertMediaAsset, -} from "@/lib/db/queries"; +import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; -import { uploadFile } from "@/lib/storage/service"; -import { db } from "@/lib/db"; -import { generatedImage } from "@/lib/db/schema"; -import { executeConfiguredVip98kTask } from "@/lib/ai/vip98k-runtime"; -import { createFalClient } from "@fal-ai/client"; - -// 变清晰API配置类型 -type EnhanceApiConfig = { - provider?: "vip98k" | "aoscdn"; - apiUrl?: string; - apiKey?: string; - type?: string; // "clean" | "face" - scaleFactor?: number | string; // 1 | 2 | 4 | "auto" - // 备用API配置 (Real-ESRGAN) - useBackupApi?: boolean; - backupApiUrl?: string; - backupApiKey?: string; - backupScale?: string; // "2" | "4" | "8" - backupModel?: string; -}; - -const AOS_MAX_RESOLUTION = 4096; -const ENHANCE_CREATE_TIMEOUT_MS = 120_000; -const ENHANCE_CREATE_RETRY_COUNT = 1; -const ENHANCE_SYNC_WAIT_TIMEOUT_MS = 20_000; -const ENHANCE_TASK_POLL_INTERVAL_MS = 1200; - -function getAosFriendlyErrorMessage(detail: string): string | null { - const resolutionMatch = detail.match(/resolution excess,\s*width=(\d+),\s*height=(\d+)/i); - if (resolutionMatch) { - const [, width, height] = resolutionMatch; - return `图片分辨率超限（${width}x${height}），AOS要求宽高都不超过 ${AOS_MAX_RESOLUTION}px。请缩小后重试。`; - } - - if (/invalid file/i.test(detail)) { - return "图片文件无法识别，请尝试转换为 JPG/PNG 后重试。"; - } - - return null; -} - -function isRetryableNetworkError(error: unknown): boolean { - if (!(error instanceof Error)) return false; - if (error.name === "AbortError") return true; - - const message = error.message || ""; - const cause = (error as Error & { cause?: { message?: string; code?: string } }).cause; - const causeMessage = cause?.message || ""; - const causeCode = cause?.code || ""; - const combined = `${message} ${causeMessage} ${causeCode}`; - - return /EHOSTUNREACH|ENOTFOUND|ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|No route to host/i.test(combined); -} - -async function ensureAosResolutionLimit( - buffer: Buffer, - logPrefix: string -): Promise { - try { - const sharp = await import("sharp"); - const image = sharp.default(buffer); - const metadata = await image.metadata(); - const width = metadata.width; - const height = metadata.height; - - if (!width || !height) { - return buffer; - } - - if (width <= AOS_MAX_RESOLUTION && height <= AOS_MAX_RESOLUTION) { - return buffer; - } - - const resizedBuffer = await image - .resize(AOS_MAX_RESOLUTION, AOS_MAX_RESOLUTION, { - fit: "inside", - withoutEnlargement: true, - }) - .toBuffer(); - const resizedMeta = await sharp.default(resizedBuffer).metadata(); - - console.warn( - `[${logPrefix}] 输入图片分辨率超限，已自动缩放: ${width}x${height} -> ${resizedMeta.width}x${resizedMeta.height}` - ); - return resizedBuffer; - } catch (error) { - console.error(`[${logPrefix}] 自动缩放失败，继续使用原图:`, error); - return buffer; - } -} - -/** - * 压缩图片生成缩略图（使用 sharp） - */ -async function createThumbnail(buffer: Buffer, mimeType: string): Promise { - try { - const sharp = await import("sharp"); - - // 创建缩略图（宽度 500px，高度按比例，质量 80%） - const thumbnailBuffer = await sharp.default(buffer) - .resize(500, null, { - fit: "inside", - withoutEnlargement: true, - }) - .jpeg({ quality: 80 }) // JPEG格式 - .toBuffer(); - - console.log(`[Enhance] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`); - return thumbnailBuffer; - } catch (error) { - console.error("[Enhance] Failed to create thumbnail:", error); - // 如果失败，返回原图 - return buffer; - } -} - -/** - * 将 base64 图片上传到 S3，返回缩略图和原图的 URL - */ -async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jpeg"): Promise<{ thumbnail: string; original: string }> { - try { - // 解析 base64 数据 - const buffer = Buffer.from(base64Data, "base64"); - - // 生成文件名 - const ext = mimeType.split("/")[1] || "jpg"; - const timestamp = Date.now(); - const randomStr = Math.random().toString(36).slice(2, 8); - const thumbnailFilename = `enhance-thumb-${timestamp}-${randomStr}.${ext}`; - const originalFilename = `enhance-orig-${timestamp}-${randomStr}.${ext}`; - - console.log(`[Enhance] Original size: ${buffer.length} bytes`); - - // 生成缩略图 - const thumbnailBuffer = await createThumbnail(buffer, mimeType); - - // 转换为 ArrayBuffer - const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( - thumbnailBuffer.byteOffset, - thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength - ) as ArrayBuffer; - const originalArrayBuffer = buffer.buffer.slice( - buffer.byteOffset, - buffer.byteOffset + buffer.byteLength - ) as ArrayBuffer; - - // 并行上传缩略图和原图到 S3 - console.log(`[Enhance] Uploading thumbnail and original to S3...`); - const [thumbnailResult, originalResult] = await Promise.all([ - uploadFile(thumbnailFilename, thumbnailArrayBuffer), - uploadFile(originalFilename, originalArrayBuffer), - ]); - - console.log(`[Enhance] Uploaded to S3:`); - console.log(` - Thumbnail: ${thumbnailResult.url}`); - console.log(` - Original: ${originalResult.url}`); - - return { - thumbnail: thumbnailResult.url, - original: originalResult.url, - }; - } catch (error) { - console.error("[Enhance] Failed to upload to S3:", error); - throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`); - } -} - -// 设置API路由最大执行时间为3分钟（180秒） +import { executeEnhanceGET, executeEnhancePOST } from "@/lib/canvas/operations/enhance"; +export const runtime = "nodejs"; export const maxDuration = 180; - -/** - * 使用Real-ESRGAN备用API处理图片（通过URL方式） - * @param imageUrl 图片URL（需要公开可访问） - * @param config 备用API配置 - * @returns 处理后的图片Base64和S3 URL - */ -async function processWithRealESRGAN( - imageUrl: string, - config: { - backupApiUrl: string; - backupApiKey: string; - backupScale: string; - backupModel: string; - } -): Promise<{ imageBase64: string; imageUrl: string; processingTime: number }> { - const startTime = Date.now(); - const apiUrl = config.backupApiUrl.endsWith("/") - ? config.backupApiUrl.slice(0, -1) - : config.backupApiUrl; - - const upscaleUrl = `${apiUrl}/upscale/url`; - console.log(`[Enhance-RealESRGAN] 调用URL方式API: ${upscaleUrl}`); - console.log(`[Enhance-RealESRGAN] 图片URL: ${imageUrl}`); - console.log(`[Enhance-RealESRGAN] 放大倍数: ${config.backupScale}x, 模型: ${config.backupModel}`); - - const response = await fetch(upscaleUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-API-Key": config.backupApiKey, - }, - body: JSON.stringify({ - image_url: imageUrl, - scale: Number.parseInt(config.backupScale) || 4, - model_name: config.backupModel || "RealESRGAN_x4plus", - }), - }); - - if (!response.ok) { - const errorText = await response.text(); - console.error(`[Enhance-RealESRGAN] API请求失败: ${response.status} ${response.statusText}`); - console.error(`[Enhance-RealESRGAN] 错误详情: ${errorText}`); - throw new Error(`Real-ESRGAN API请求失败: ${response.status} - ${errorText || response.statusText}`); - } - - const result = await response.json(); - console.log(`[Enhance-RealESRGAN] API响应:`, { - success: result.success, - hasImageUrl: !!result.image_url, - processingTime: result.processing_time, - totalTime: result.total_time, - }); - - if (!result.success || !result.image_url) { - throw new Error(`Real-ESRGAN处理失败: ${JSON.stringify(result)}`); - } - - // Real-ESRGAN API直接返回S3 URL，我们需要下载图片并转换为Base64 - console.log(`[Enhance-RealESRGAN] 下载处理后的图片: ${result.image_url}`); - const imageResponse = await fetch(result.image_url); - if (!imageResponse.ok) { - throw new Error(`无法下载Real-ESRGAN处理后的图片: ${imageResponse.status}`); - } - - const imageBuffer = await imageResponse.arrayBuffer(); - const imageBase64 = Buffer.from(imageBuffer).toString("base64"); - - const processingTime = (Date.now() - startTime) / 1000; - console.log(`[Enhance-RealESRGAN] ✅ 处理完成，总耗时: ${processingTime.toFixed(2)}秒`); - console.log(`[Enhance-RealESRGAN] ✅ 处理后图片大小: ${imageBuffer.byteLength} bytes`); - - return { - imageBase64, - imageUrl: result.image_url, - processingTime, - }; +export async function GET(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + return executeEnhanceGET(request, session.user.id); } - -/** - * 使用Real-ESRGAN备用API处理图片（通过Base64方式，用于没有公开URL的图片） - * @param imageBase64 图片Base64数据 - * @param config 备用API配置 - * @returns 处理后的图片Base64 - */ -async function processWithRealESRGANBase64( - imageBase64: string, - config: { - backupApiUrl: string; - backupApiKey: string; - backupScale: string; - backupModel: string; - } -): Promise<{ imageBase64: string; processingTime: number }> { - const startTime = Date.now(); - const apiUrl = config.backupApiUrl.endsWith("/") - ? config.backupApiUrl.slice(0, -1) - : config.backupApiUrl; - - const upscaleUrl = `${apiUrl}/upscale/base64`; - console.log(`[Enhance-RealESRGAN] 调用Base64方式API: ${upscaleUrl}`); - console.log(`[Enhance-RealESRGAN] 放大倍数: ${config.backupScale}x, 模型: ${config.backupModel}`); - console.log(`[Enhance-RealESRGAN] 输入图片Base64长度: ${imageBase64.length}`); - - const response = await fetch(upscaleUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-API-Key": config.backupApiKey, - }, - body: JSON.stringify({ - image_base64: imageBase64, - scale: Number.parseInt(config.backupScale) || 4, - model_name: config.backupModel || "RealESRGAN_x4plus", - }), - }); - - if (!response.ok) { - const errorText = await response.text(); - console.error(`[Enhance-RealESRGAN] API请求失败: ${response.status} ${response.statusText}`); - console.error(`[Enhance-RealESRGAN] 错误详情: ${errorText}`); - throw new Error(`Real-ESRGAN API请求失败: ${response.status} - ${errorText || response.statusText}`); - } - - const result = await response.json(); - console.log(`[Enhance-RealESRGAN] API响应:`, { - success: result.success, - hasImageBase64: !!result.image_base64, - processingTime: result.processing_time, - }); - - if (!result.success || !result.image_base64) { - throw new Error(`Real-ESRGAN处理失败: ${JSON.stringify(result)}`); - } - - const processingTime = (Date.now() - startTime) / 1000; - console.log(`[Enhance-RealESRGAN] ✅ 处理完成，总耗时: ${processingTime.toFixed(2)}秒`); - console.log(`[Enhance-RealESRGAN] ✅ 处理后图片Base64长度: ${result.image_base64.length}`); - - return { - imageBase64: result.image_base64, - processingTime, - }; -} - -async function processWithFalEsrgan( - imageBase64: string, - requestedScaleFactor: number, -): Promise<{ imageBase64: string; processingTime: number }> { - const providers = await getAiProvidersByType({ - type: "fal-image", - enabledOnly: true, - }); - const provider = providers.find((item) => item.apiKey?.trim()); - if (!provider?.apiKey) { - throw new Error("未配置可用的 FAL 专用高清服务"); - } - - const scale = Math.max(2, Math.min(8, requestedScaleFactor || 4)); - const model = scale <= 2 ? "RealESRGAN_x2plus" : "RealESRGAN_x4plus"; - const credentials = provider.apiKey - .trim() - .replace(/^(?:Key|Bearer)\s+/i, ""); - const fal = createFalClient({ credentials }); - const startedAt = Date.now(); - const result: any = await fal.subscribe("fal-ai/esrgan", { - input: { - image_url: `data:image/jpeg;base64,${imageBase64}`, - scale, - model, - output_format: "png", - }, - logs: true, - timeout: 180_000, - }); - const outputUrl = result?.data?.image?.url || result?.image?.url; - if (!outputUrl) { - throw new Error("FAL ESRGAN 专用高清服务未返回图片"); - } - - const outputResponse = await fetch(outputUrl, { - cache: "no-store", - signal: AbortSignal.timeout(120_000), - }); - if (!outputResponse.ok) { - throw new Error( - `FAL ESRGAN 高清结果下载失败（HTTP ${outputResponse.status}）`, - ); - } - - return { - imageBase64: Buffer.from(await outputResponse.arrayBuffer()).toString( - "base64", - ), - processingTime: (Date.now() - startedAt) / 1000, - }; -} - -async function processWithVip98kEnhance(params: { - userId: string; - imageBase64: string; - prompt: string; - chatId: string | null; - messageId: string | null; - sourceImageUrl: string | null; - scaleFactor: number; -}) { - const startedAt = Date.now(); - const result = await executeConfiguredVip98kTask({ - modelId: "enhance", - kind: "image-tool", - prompt: `${params.prompt || "Enhance image clarity"}. Upscale ${params.scaleFactor}x while preserving composition, color and texture.`, - imageUrls: [`data:image/jpeg;base64,${params.imageBase64}`], - imageParams: { scaleFactor: params.scaleFactor }, - }); - const outputUrl = result.outputUrls[0]; - if (!outputUrl) throw new Error("VIP98K 变清晰未返回图片"); - - const response = await fetch(outputUrl, { - cache: "no-store", - signal: AbortSignal.timeout(120_000), - }); - if (!response.ok) { - throw new Error(`VIP98K 变清晰结果下载失败（HTTP ${response.status}）`); - } - const outputBuffer = Buffer.from(await response.arrayBuffer()); - const outputBase64 = outputBuffer.toString("base64"); - const uploaded = await uploadBase64ToS3(outputBase64, "image/png"); - const [savedImage] = await db - .insert(generatedImage) - .values({ - userId: params.userId, - chatId: params.chatId, - messageId: params.messageId, - thumbnailUrl: uploaded.thumbnail, - originalUrl: uploaded.original, - imageSize: `${params.scaleFactor}x放大`, - prompt: params.prompt, - modelId: "enhance", - }) - .returning(); - - try { - const sourceAssets = params.sourceImageUrl - ? await getMediaAssetsByUserAndUrls({ - userId: params.userId, - urls: [params.sourceImageUrl], - }) - : []; - const media = await upsertMediaAsset({ - userId: params.userId, - chatId: params.chatId, - mediaType: "image", - sourceType: "edited", - status: "ready", - originalUrl: uploaded.original, - previewUrl: uploaded.original, - thumbnailUrl: uploaded.thumbnail, - displayName: params.prompt.slice(0, 80), - prompt: params.prompt, - modelId: "enhance", - sourceMessageId: params.messageId, - sourceTable: "GeneratedImage", - sourceRowId: savedImage?.id || null, - primaryParentAssetId: sourceAssets[0]?.id ?? null, - metadata: { provider: "vip98k", scaleFactor: params.scaleFactor }, - }); - if (media) { - for (const sourceAsset of sourceAssets) { - await createMediaAssetRelation({ - userId: params.userId, - chatId: params.chatId, - fromAssetId: sourceAsset.id, - toAssetId: media.id, - relationType: "edited_from", - sourceMessageId: params.messageId, - metadata: { prompt: params.prompt, provider: "vip98k" }, - }); - } - } - } catch (error) { - console.warn("[Enhance-VIP98K] MediaAsset registration failed", error); - } - - return { - success: true, - image_base64: outputBase64, - thumbnail_url: uploaded.thumbnail, - original_url: uploaded.original, - processing_time: (Date.now() - startedAt) / 1000, - api_type: "vip98k", - }; -} - -/** - * 轮询获取任务结果（根据官方示例） - */ -async function pollTaskResult( - apiUrl: string, - apiKey: string, - taskId: string, - delay: number = 1000, - timeout: number = 120000 // 增加到120秒（2分钟） -): Promise { - const pollUrl = `${apiUrl}/api/tasks/visual/scale/${taskId}`; - - const getTaskResult = async (): Promise => { - const response = await fetch(pollUrl, { - method: "GET", - headers: { - "X-API-KEY": apiKey, - }, - }); - - if (!response.ok) { - throw new Error(`轮询请求失败: ${response.status} ${response.statusText}`); - } - - const result = await response.json(); - - if (!result.data) { - throw result; - } - - const { progress, state } = result.data; - console.log(`[Enhance] 轮询进度: progress=${progress}%, state=${state}, task_id=${taskId}`); - - // state < 0 表示失败 - if (state < 0) { - const errMessage = String( - result.data?.err_message || result.data?.err_info || result.data?.state_detail || `state=${state}` - ); - throw new Error(`AOS任务失败: ${errMessage}`); - } - - // progress >= 100 表示完成 - if (progress >= 100) { - console.log(`[Enhance] 任务完成，task_id=${taskId}`); - return result; - } - - // 否则继续轮询 - throw null; - }; - - try { - const result = await getTaskResult(); - return result; - } catch (error) { - if (error && typeof error === "object" && "data" in error) { - throw new Error(JSON.stringify(error, null, 2)); - } - if (timeout <= 0) { - throw new Error("轮询超时"); - } - await new Promise((resolve) => { - setTimeout(() => { - resolve(undefined); - }, delay); - }); - return pollTaskResult(apiUrl, apiKey, taskId, delay, timeout - delay); - } -} - -function isEnhanceTaskStillProcessing(error: unknown): boolean { - if (!(error instanceof Error)) return false; - return /轮询超时/i.test(error.message || ""); -} - -function buildEnhanceTaskPendingResponse(taskId: string) { - return NextResponse.json( - { - success: true, - processing: true, - status: "running", - taskId, - pollAfterMs: ENHANCE_TASK_POLL_INTERVAL_MS, - }, - { status: 202 } - ); -} - -async function finalizeEnhanceTaskResult(params: { - userId: string; - taskResult: any; - prompt?: string; - chatId?: string | null; - messageId?: string | null; - startedAt?: number; - sourceImageUrls?: string[]; -}) { - if (params.taskResult?.status !== 200 || !params.taskResult?.data) { - throw new Error(params.taskResult?.message || "未知错误"); - } - - let enhancedImageBase64: string | null = null; - if (typeof params.taskResult.data.image_base64 === "string" && params.taskResult.data.image_base64) { - enhancedImageBase64 = params.taskResult.data.image_base64; - } else if (typeof params.taskResult.data.image === "string" && params.taskResult.data.image) { - const imageData = params.taskResult.data.image; - if (/^https?:\/\//i.test(imageData)) { - const imageResponse = await fetch(imageData); - if (!imageResponse.ok) { - throw new Error(`无法下载处理后的图片: ${imageResponse.status}`); - } - const downloadedBuffer = await imageResponse.arrayBuffer(); - enhancedImageBase64 = Buffer.from(downloadedBuffer).toString("base64"); - } else { - enhancedImageBase64 = imageData; - } - } - - if (!enhancedImageBase64) { - throw new Error("轮询结果未包含图片数据"); - } - - const uploadResult = await uploadBase64ToS3(enhancedImageBase64, "image/jpeg"); - const finalPrompt = params.prompt?.trim() || "图片变清晰处理"; - - let saved = false; - let saveError: unknown = null; - let savedImage: - | { - id: string; - } - | null = null; - for (let attempt = 1; attempt <= 3; attempt += 1) { - try { - ([savedImage] = await db.insert(generatedImage).values({ - userId: params.userId, - chatId: params.chatId || null, - messageId: params.messageId || null, - thumbnailUrl: uploadResult.thumbnail, - originalUrl: uploadResult.original, - imageSize: "2K", - prompt: finalPrompt, - modelId: "enhance-1.0", - }).returning()); - saved = true; - break; - } catch (dbError) { - saveError = dbError; - if (attempt < 3) { - await new Promise((resolve) => setTimeout(resolve, 120 * attempt)); - } - } - } - if (!saved) { - console.error("[Enhance] 保存到相册失败:", saveError); - throw new Error("图片变清晰成功但保存到相册失败，请稍后重试"); - } - - if (savedImage) { - try { - const sourceAssets = - Array.isArray(params.sourceImageUrls) && params.sourceImageUrls.length > 0 - ? await getMediaAssetsByUserAndUrls({ - userId: params.userId, - urls: params.sourceImageUrls, - }) - : []; - const media = await upsertMediaAsset({ - userId: params.userId, - chatId: params.chatId || null, - mediaType: "image", - sourceType: "edited", - status: "ready", - originalUrl: uploadResult.original, - previewUrl: uploadResult.original, - thumbnailUrl: uploadResult.thumbnail, - displayName: finalPrompt.slice(0, 80), - prompt: finalPrompt, - modelId: "enhance-1.0", - sourceMessageId: params.messageId || null, - sourceTable: "GeneratedImage", - sourceRowId: savedImage.id, - primaryParentAssetId: sourceAssets[0]?.id ?? null, - metadata: { - imageSize: "2K", - }, - }); - if (media) { - for (const sourceAsset of sourceAssets) { - await createMediaAssetRelation({ - userId: params.userId, - chatId: params.chatId || null, - fromAssetId: sourceAsset.id, - toAssetId: media.id, - relationType: "edited_from", - sourceMessageId: params.messageId || null, - metadata: { - prompt: finalPrompt, - }, - }); - } - } - } catch (assetError) { - console.error("[Enhance] ⚠️ MediaAsset 注册失败:", assetError); - } - } - - const processingTime = - typeof params.startedAt === "number" ? (Date.now() - params.startedAt) / 1000 : undefined; - - return { - success: true, - image_base64: enhancedImageBase64, - thumbnail_url: uploadResult.thumbnail, - original_url: uploadResult.original, - processing_time: processingTime, - }; -} - -async function executeFalEsrganFallback(params: { - userId: string; - imageBase64: string; - prompt: string; - chatId: string | null; - messageId: string | null; - sourceImageUrl: string | null; - requestedScaleFactor?: number; - reason: string; -}) { - const scaleFactor = params.requestedScaleFactor || 4; - console.warn("[Enhance-FAL-ESRGAN] 启用专用高清备用链路", { - reason: params.reason.slice(0, 500), - scaleFactor, - }); - const result = await processWithFalEsrgan(params.imageBase64, scaleFactor); - const payload = await finalizeEnhanceTaskResult({ - userId: params.userId, - taskResult: { - status: 200, - data: { image_base64: result.imageBase64 }, - }, - prompt: params.prompt || "图片变清晰处理", - chatId: params.chatId, - messageId: params.messageId, - sourceImageUrls: params.sourceImageUrl - ? [params.sourceImageUrl] - : undefined, - }); - return { - ...payload, - processing_time: result.processingTime, - api_type: "fal-esrgan", - }; -} - -export async function GET(request: NextRequest) { - try { - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json({ error: "未授权，请先登录" }, { status: 401 }); - } - - const { searchParams } = new URL(request.url); - const taskId = String(searchParams.get("taskId") || "").trim(); - if (!taskId) { - return NextResponse.json({ error: "缺少 taskId 参数" }, { status: 400 }); - } - - const prompt = searchParams.get("prompt") || "图片变清晰处理"; - const chatId = searchParams.get("chatId"); - const messageId = searchParams.get("messageId"); - const sourceImageUrl = searchParams.get("sourceImageUrl"); - - const enhanceConfig = await getSystemSetting({ key: "enhance_api_config" }); - if (!enhanceConfig?.value) { - return NextResponse.json( - { error: "变清晰API配置未设置，请在系统设置中配置" }, - { status: 400 } - ); - } - - const config = enhanceConfig.value as EnhanceApiConfig; - if (config.provider === "vip98k") { - return NextResponse.json( - { error: "VIP98K 变清晰任务在 POST 请求内完成，无需 taskId 轮询" }, - { status: 400 } - ); - } - if (config.useBackupApi) { - return NextResponse.json( - { error: "当前启用了Real-ESRGAN备用模式，不支持 taskId 轮询" }, - { status: 400 } - ); - } - if (!config.apiUrl || !config.apiKey) { - return NextResponse.json( - { error: "变清晰API地址或密钥未配置" }, - { status: 400 } - ); - } - - const apiUrl = config.apiUrl.endsWith("/") ? config.apiUrl.slice(0, -1) : config.apiUrl; - - let taskResult: any; - try { - taskResult = await pollTaskResult( - apiUrl, - config.apiKey, - taskId, - ENHANCE_TASK_POLL_INTERVAL_MS, - ENHANCE_SYNC_WAIT_TIMEOUT_MS - ); - } catch (pollError) { - if (isEnhanceTaskStillProcessing(pollError)) { - return buildEnhanceTaskPendingResponse(taskId); - } - const detail = pollError instanceof Error ? pollError.message : "轮询失败"; - return NextResponse.json( - { error: "变清晰处理失败", detail }, - { status: 500 } - ); - } - - const responsePayload = await finalizeEnhanceTaskResult({ - userId: session.user.id, - taskResult, - prompt, - chatId, - messageId, - sourceImageUrls: sourceImageUrl ? [sourceImageUrl] : undefined, - }); - - return NextResponse.json(responsePayload); - } catch (error) { - console.error("[Enhance API][TaskStatus] 处理错误:", error); - return NextResponse.json( - { error: "服务器内部错误", detail: error instanceof Error ? error.message : "未知错误" }, - { status: 500 } - ); - } -} - -/** - * 图片变清晰API路由 - * 从数据库读取变清晰API配置，调用外部API进行图片增强处理 - * 将处理后的图片上传到S3并保存到数据库 - */ -export async function POST(request: NextRequest) { - console.log("[Enhance API] ===== 收到请求 ====="); - try { - // 0. 获取用户会话 - console.log("[Enhance API] 开始获取用户会话..."); - const session = await auth(); - console.log("[Enhance API] 用户会话获取完成:", session?.user?.id ? "已登录" : "未登录"); - if (!session?.user?.id) { - return NextResponse.json( - { error: "未授权，请先登录" }, - { status: 401 } - ); - } - - const userId = session.user.id; - console.log("[Enhance API] 用户ID:", userId); - // 1. 从数据库获取变清晰API配置 - console.log("[Enhance API] 开始获取配置..."); - const enhanceConfig = await getSystemSetting({ key: "enhance_api_config" }); - console.log("[Enhance API] 配置获取完成:", enhanceConfig ? "已找到" : "未找到"); - - if (!enhanceConfig || !enhanceConfig.value) { - return NextResponse.json( - { error: "变清晰API配置未设置，请在系统设置中配置" }, - { status: 400 } - ); - } - - const config = enhanceConfig.value as EnhanceApiConfig; - const useVip98k = config.provider === "vip98k"; - - // 检查是否使用备用API - const useBackupApi = !useVip98k && config.useBackupApi === true; - console.log(`[Enhance API] 使用${useBackupApi ? "备用API (Real-ESRGAN)" : "主API"}`); - - if (useVip98k) { - // VIP98K 密钥只从 AiProvider 读取。 - } else if (useBackupApi) { - // 验证备用API配置 - if (!config.backupApiUrl || !config.backupApiKey) { - return NextResponse.json( - { error: "备用变清晰API地址或密钥未配置" }, - { status: 400 } - ); - } - } else { - // 验证主API配置 - if (!config.apiUrl || !config.apiKey) { - return NextResponse.json( - { error: "变清晰API地址或密钥未配置" }, - { status: 400 } - ); - } - } - - // 2. 获取请求体（支持 imageUrl、image_base64 或 FormData） - console.log("[Enhance API] 开始获取请求体..."); - const contentType = request.headers.get("content-type") || ""; - console.log("[Enhance API] Content-Type:", contentType); - - let imageBase64: string; - let prompt = ""; // 用户提示词（可选） - let chatId: string | null = null; // 聊天ID（可选） - let messageId: string | null = null; // 消息ID（可选） - let sourceImageUrl: string | null = null; - let requestedScaleFactor: number | undefined; - - if (contentType.includes("application/json")) { - console.log("[Enhance API] 解析JSON请求体..."); - const body = await request.json(); - prompt = body.prompt || "图片变清晰处理"; - chatId = body.chatId || null; - messageId = body.messageId || null; - const parsedScaleFactor = Number.parseInt(String(body.scaleFactor || ""), 10); - requestedScaleFactor = - Number.isFinite(parsedScaleFactor) && [1, 2, 4, 8].includes(parsedScaleFactor) - ? parsedScaleFactor - : undefined; - - // 优先使用 imageUrl，其次使用 image_base64 - if (body.imageUrl) { - sourceImageUrl = String(body.imageUrl); - // 从 URL 获取图片并转换为 Base64 - console.log(`[Enhance API] 从 URL 获取图片: ${body.imageUrl}`); - try { - const imageResponse = await fetch(body.imageUrl); - if (!imageResponse.ok) { - return NextResponse.json( - { error: `无法获取图片: ${imageResponse.status} ${imageResponse.statusText}` }, - { status: 400 } - ); - } - const arrayBuffer = await imageResponse.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - imageBase64 = buffer.toString("base64"); - console.log(`[Enhance API] 图片已转换为 Base64，大小: ${buffer.length} bytes`); - } catch (fetchError) { - console.error("[Enhance API] 获取图片失败:", fetchError); - return NextResponse.json( - { error: "无法获取图片，请检查图片链接是否有效" }, - { status: 400 } - ); - } - } else if (body.image_base64) { - console.log("[Enhance API] JSON请求体解析完成，image_base64长度:", body.image_base64?.length || 0); - imageBase64 = body.image_base64; - } else { - return NextResponse.json( - { error: "缺少 imageUrl 或 image_base64 参数" }, - { status: 400 } - ); - } - } else if (contentType.includes("multipart/form-data")) { - // FormData格式：文件上传 - const formData = await request.formData(); - const file = formData.get("file") as File; - - if (!file) { - return NextResponse.json( - { error: "缺少file参数" }, - { status: 400 } - ); - } - - // 将文件转换为Base64 - const arrayBuffer = await file.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - imageBase64 = buffer.toString("base64"); - prompt = (formData.get("prompt") as string) || "图片变清晰处理"; - chatId = (formData.get("chatId") as string) || null; - messageId = (formData.get("messageId") as string) || null; - } else { - return NextResponse.json( - { error: "不支持的Content-Type，请使用application/json或multipart/form-data" }, - { status: 400 } - ); - } - - if (useVip98k) { - try { - const scaleFactor = requestedScaleFactor || 4; - const payload = await processWithVip98kEnhance({ - userId, - imageBase64, - prompt: prompt || "图片变清晰处理", - chatId, - messageId, - sourceImageUrl, - scaleFactor, - }); - return NextResponse.json(payload); - } catch (error) { - console.error("[Enhance-VIP98K] Processing failed", error); - return NextResponse.json( - { - error: "VIP98K 变清晰处理失败", - detail: error instanceof Error ? error.message : "未知错误", - }, - { status: 500 } - ); - } - } - - // ========== 备用API处理路径 (Real-ESRGAN) ========== - if (useBackupApi) { - console.log("[Enhance API] 使用Real-ESRGAN备用API处理（URL方式）..."); - - try { - // 步骤1: 先将Base64图片上传到S3获取公开可访问的URL - console.log(`[Enhance-RealESRGAN] 📤 步骤1: 上传原图到S3获取URL...`); - const tempUploadResult = await uploadBase64ToS3(imageBase64, "image/jpeg"); - const sourceImageUrl = tempUploadResult.original; - console.log(`[Enhance-RealESRGAN] ✅ 原图已上传，URL: ${sourceImageUrl}`); - - // 步骤2: 使用URL方式调用Real-ESRGAN API - console.log(`[Enhance-RealESRGAN] 🔄 步骤2: 调用Real-ESRGAN API处理...`); - const result = await processWithRealESRGAN(sourceImageUrl, { - backupApiUrl: config.backupApiUrl!, - backupApiKey: config.backupApiKey!, - backupScale: String(requestedScaleFactor || Number.parseInt(config.backupScale || "4", 10) || 4), - backupModel: config.backupModel || "RealESRGAN_x4plus", - }); - - // 步骤3: Real-ESRGAN API返回的URL就是处理后的图片 - // 需要重新上传到我们的S3（因为Real-ESRGAN返回的URL可能在另一个S3桶） - console.log(`[Enhance-RealESRGAN] 📤 步骤3: 上传处理后的图片到S3...`); - let thumbnailUrl: string; - let originalUrl: string; - - try { - const uploadResult = await uploadBase64ToS3(result.imageBase64, "image/png"); - thumbnailUrl = uploadResult.thumbnail; - originalUrl = uploadResult.original; - console.log(`[Enhance-RealESRGAN] ✅ 处理后图片已上传: ${originalUrl}`); - } catch (uploadError) { - console.error("[Enhance-RealESRGAN] ❌ 上传到S3失败:", uploadError); - // 如果上传失败，直接使用Real-ESRGAN返回的URL - console.log(`[Enhance-RealESRGAN] ⚠️ 使用Real-ESRGAN返回的URL作为备选: ${result.imageUrl}`); - thumbnailUrl = result.imageUrl; - originalUrl = result.imageUrl; - } - - // 步骤4: 保存到数据库 - let saved = false; - let saveError: unknown = null; - let savedImage: - | { - id: string; - } - | null = null; - for (let attempt = 1; attempt <= 3; attempt += 1) { - try { - ([savedImage] = await db.insert(generatedImage).values({ - userId, - chatId: chatId || null, - messageId: messageId || null, - thumbnailUrl, - originalUrl, - imageSize: `${requestedScaleFactor || Number.parseInt(config.backupScale || "4", 10) || 4}x放大`, - prompt: prompt || "Real-ESRGAN图片变清晰处理", - modelId: `realesrgan-${config.backupModel}`, - }).returning()); - console.log(`[Enhance-RealESRGAN] ✅ 保存到数据库成功，ID: ${savedImage.id}`); - saved = true; - break; - } catch (dbError) { - saveError = dbError; - if (attempt < 3) { - await new Promise((resolve) => setTimeout(resolve, 120 * attempt)); - } - } - } - if (!saved) { - console.error("[Enhance-RealESRGAN] ❌ 保存到数据库失败:", saveError); - throw new Error("图片变清晰成功但保存到相册失败，请稍后重试"); - } - - if (savedImage) { - try { - const sourceAssets = - sourceImageUrl - ? await getMediaAssetsByUserAndUrls({ - userId, - urls: [sourceImageUrl], - }) - : []; - const media = await upsertMediaAsset({ - userId, - chatId, - mediaType: "image", - sourceType: "edited", - status: "ready", - originalUrl, - previewUrl: originalUrl, - thumbnailUrl, - displayName: (prompt || "Real-ESRGAN图片变清晰处理").slice(0, 80), - prompt: prompt || "Real-ESRGAN图片变清晰处理", - modelId: `realesrgan-${config.backupModel}`, - sourceMessageId: messageId, - sourceTable: "GeneratedImage", - sourceRowId: savedImage.id, - primaryParentAssetId: sourceAssets[0]?.id ?? null, - metadata: { - imageSize: `${requestedScaleFactor || Number.parseInt(config.backupScale || "4", 10) || 4}x放大`, - apiType: "realesrgan-url", - }, - }); - if (media) { - for (const sourceAsset of sourceAssets) { - await createMediaAssetRelation({ - userId, - chatId, - fromAssetId: sourceAsset.id, - toAssetId: media.id, - relationType: "edited_from", - sourceMessageId: messageId, - metadata: { - prompt: prompt || "Real-ESRGAN图片变清晰处理", - }, - }); - } - } - } catch (assetError) { - console.error("[Enhance-RealESRGAN] ⚠️ MediaAsset 注册失败:", assetError); - } - } - - // 返回结果 - console.log(`[Enhance-RealESRGAN] ✅ 全部完成，总耗时: ${result.processingTime.toFixed(2)}秒`); - return NextResponse.json({ - success: true, - image_base64: result.imageBase64, - thumbnail_url: thumbnailUrl, - original_url: originalUrl, - processing_time: result.processingTime, - api_type: "realesrgan-url", - }); - - } catch (error) { - console.error("[Enhance-RealESRGAN] ❌ 处理失败:", error); - try { - const fallbackPayload = await executeFalEsrganFallback({ - userId, - imageBase64, - prompt, - chatId, - messageId, - sourceImageUrl, - requestedScaleFactor, - reason: error instanceof Error ? error.message : "RealESRGAN 备用服务失败", - }); - return NextResponse.json(fallbackPayload); - } catch (falError) { - console.error("[Enhance-FAL-ESRGAN] 专用高清备用链路失败:", falError); - } - return NextResponse.json( - { - error: "Real-ESRGAN变清晰处理失败", - detail: error instanceof Error ? error.message : "未知错误", - }, - { status: 500 } - ); - } - } - - // ========== 主API处理路径 ========== - // 3. 调用变清晰API（使用异步模式，根据官方示例） - console.log("[Enhance API] 开始准备主API调用..."); - const apiUrl = config.apiUrl!.endsWith("/") - ? config.apiUrl!.slice(0, -1) - : config.apiUrl!; - - const enhanceUrl = `${apiUrl}/api/tasks/visual/scale`; - console.log("[Enhance API] API URL:", enhanceUrl); - - // 将Base64转换为Buffer - console.log("[Enhance API] 转换Base64为Buffer，长度:", imageBase64.length); - const rawImageBuffer = Buffer.from(imageBase64, "base64"); - const imageBuffer = await ensureAosResolutionLimit(rawImageBuffer, "Enhance API"); - console.log("[Enhance API] Buffer创建完成，大小:", imageBuffer.length, "bytes"); - - // 创建FormData（使用form-data包，Node.js环境） - console.log("[Enhance API] 创建FormData..."); - const FormDataLib = await import("form-data"); - const formData = new FormDataLib.default(); - console.log("[Enhance API] FormData创建完成"); - formData.append("image_file", imageBuffer, { - filename: "image.jpg", - contentType: "image/jpeg", - }); - // 使用异步模式（sync=0），根据官方示例 - formData.append("sync", "0"); - formData.append("type", config.type || "clean"); // 默认通用清晰化 - const scaleFactorValue = - requestedScaleFactor || - (typeof config.scaleFactor === "number" - ? config.scaleFactor - : config.scaleFactor === "auto" - ? undefined - : Number.parseInt(String(config.scaleFactor))); - if (scaleFactorValue) { - formData.append("scale_factor", String(scaleFactorValue)); - } - formData.append("return_type", "2"); // 返回base64 - - const headers: Record = { - "X-API-KEY": config.apiKey!, - ...formData.getHeaders(), // 包含Content-Type和boundary - }; - - const startTime = Date.now(); - let taskId: string; - - try { - console.log("[Enhance API] 开始创建任务..."); - // 将FormData转换为Buffer（form-data包是stream） - console.log("[Enhance API] 转换FormData为Buffer..."); - const formDataBuffer = await new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - formData.on("data", (chunk: any) => { - // 确保chunk是Buffer类型 - if (Buffer.isBuffer(chunk)) { - chunks.push(chunk); - } else if (typeof chunk === "string") { - chunks.push(Buffer.from(chunk, "utf-8")); - } else if (chunk instanceof Uint8Array) { - chunks.push(Buffer.from(chunk)); - } else { - console.warn("[Enhance API] 未知的chunk类型:", typeof chunk, chunk); - chunks.push(Buffer.from(String(chunk), "utf-8")); - } - }); - formData.on("end", () => { - console.log("[Enhance API] FormData stream结束，chunks数量:", chunks.length); - resolve(Buffer.concat(chunks)); - }); - formData.on("error", (error) => { - console.error("[Enhance API] FormData stream错误:", error); - reject(error); - }); - - // 重要：form-data包需要被pipe或手动触发才能开始流式传输 - // 使用PassThrough来触发stream - const { PassThrough } = require("stream"); - const passThrough = new PassThrough(); - passThrough.on("data", () => {}); // 忽略数据，我们已经在formData上监听 - formData.pipe(passThrough); - }); - - console.log("[Enhance API] FormData Buffer转换完成，大小:", formDataBuffer.length, "bytes"); - const uint8Array = new Uint8Array(formDataBuffer); - - let createResponse: Response | null = null; - let lastFetchError: unknown; - - for (let attempt = 1; attempt <= ENHANCE_CREATE_RETRY_COUNT + 1; attempt++) { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), ENHANCE_CREATE_TIMEOUT_MS); - - try { - console.log(`[Enhance API] 发送创建任务请求... attempt=${attempt}`); - createResponse = await fetch(enhanceUrl, { - method: "POST", - headers, - body: uint8Array, - signal: controller.signal, - }); - console.log("[Enhance API] 创建任务响应状态:", createResponse.status, createResponse.statusText); - clearTimeout(timeoutId); - break; - } catch (fetchError) { - clearTimeout(timeoutId); - lastFetchError = fetchError; - const canRetry = attempt <= ENHANCE_CREATE_RETRY_COUNT && isRetryableNetworkError(fetchError); - if (canRetry) { - console.warn( - `[Enhance API] 创建任务失败，准备重试: attempt=${attempt}, error=${fetchError instanceof Error ? fetchError.message : String(fetchError)}` - ); - await new Promise((resolve) => setTimeout(resolve, 1500)); - continue; - } - throw fetchError; - } - } - - if (!createResponse) { - throw lastFetchError instanceof Error ? lastFetchError : new Error("创建任务请求失败"); - } - - if (!createResponse.ok) { - const errorText = await createResponse.text(); - throw new Error(`创建任务失败: ${errorText || createResponse.statusText}`); - } - - const createResult = await createResponse.json(); - - if (!createResult.data || !createResult.data.task_id) { - throw new Error(`API未返回task_id: ${JSON.stringify(createResult)}`); - } - - taskId = createResult.data.task_id; - console.log(`[Enhance] 任务已创建，task_id: ${taskId}`); - } catch (fetchError) { - const processingTime = (Date.now() - startTime) / 1000; - console.error("[Enhance API] 网络请求失败:", { - error: fetchError, - enhanceUrl, - processingTime, - }); - - try { - const fallbackPayload = await executeFalEsrganFallback({ - userId, - imageBase64, - prompt, - chatId, - messageId, - sourceImageUrl, - requestedScaleFactor, - reason: fetchError instanceof Error ? fetchError.message : "AOS 高清任务创建失败", - }); - return NextResponse.json(fallbackPayload); - } catch (falError) { - console.error("[Enhance-FAL-ESRGAN] 专用高清备用链路失败:", falError); - } - - // 检查是否是网络连接错误 - if (fetchError instanceof Error) { - if (fetchError.name === "AbortError") { - return NextResponse.json( - { - error: "变清晰API请求超时", - detail: "创建任务请求超过120秒未响应，请检查API服务器状态或网络连接", - }, - { status: 504 } - ); - } - - // 检查错误消息和cause中的网络错误 - const errorMessage = fetchError.message || ""; - const errorCause = (fetchError as any).cause; - const causeMessage = errorCause?.message || ""; - const causeCode = errorCause?.code || ""; - - // 检查各种网络连接错误 - if (errorMessage.includes("EHOSTUNREACH") || - errorMessage.includes("ENOTFOUND") || - errorMessage.includes("ECONNREFUSED") || - errorMessage.includes("No route to host") || - causeMessage.includes("EHOSTUNREACH") || - causeMessage.includes("ENOTFOUND") || - causeMessage.includes("ECONNREFUSED") || - causeMessage.includes("No route to host") || - causeCode === "EHOSTUNREACH" || - causeCode === "ENOTFOUND" || - causeCode === "ECONNREFUSED") { - return NextResponse.json( - { - error: "无法连接到变清晰API服务器", - detail: `服务器无法访问 ${enhanceUrl}，请检查：\n1. API服务器是否在线\n2. 网络连接是否正常\n3. 防火墙设置是否正确\n4. API地址和密钥是否正确`, - }, - { status: 503 } - ); - } - } - - return NextResponse.json( - { - error: "变清晰API请求失败", - detail: fetchError instanceof Error ? fetchError.message : "未知网络错误", - }, - { status: 500 } - ); - } - - // 4. 轮询获取任务结果（短等待，超时后让前端继续轮询） - console.log(`[Enhance] 开始短轮询任务结果，task_id: ${taskId}`); - let taskResult: any; - try { - taskResult = await pollTaskResult( - apiUrl, - config.apiKey!, - taskId, - ENHANCE_TASK_POLL_INTERVAL_MS, - ENHANCE_SYNC_WAIT_TIMEOUT_MS - ); - } catch (pollError) { - if (isEnhanceTaskStillProcessing(pollError)) { - return buildEnhanceTaskPendingResponse(taskId); - } - const detail = pollError instanceof Error ? pollError.message : "轮询失败"; - try { - const fallbackPayload = await executeFalEsrganFallback({ - userId, - imageBase64, - prompt, - chatId, - messageId, - sourceImageUrl, - requestedScaleFactor, - reason: detail, - }); - return NextResponse.json(fallbackPayload); - } catch (falError) { - console.error("[Enhance-FAL-ESRGAN] 专用高清备用链路失败:", falError); - } - const friendlyMessage = getAosFriendlyErrorMessage(detail); - return NextResponse.json( - { error: "变清晰处理失败", detail: friendlyMessage || detail }, - { status: friendlyMessage ? 400 : 500 } - ); - } - - const responsePayload = await finalizeEnhanceTaskResult({ - userId, - taskResult, - prompt, - chatId, - messageId, - startedAt: startTime, - sourceImageUrls: sourceImageUrl ? [sourceImageUrl] : undefined, - }); - - return NextResponse.json(responsePayload); - } catch (error) { - console.error("[Enhance API] 处理错误:", error); - return NextResponse.json( - { error: "服务器内部错误", detail: error instanceof Error ? error.message : "未知错误" }, - { status: 500 } - ); - } +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + return executeEnhancePOST(request, session.user.id); } diff --git a/app/api/extend/route.ts b/app/api/extend/route.ts index 2e44f06b1..fb215ae44 100644 --- a/app/api/extend/route.ts +++ b/app/api/extend/route.ts @@ -1,234 +1,10 @@ -import { type NextRequest, NextResponse } from "next/server"; +import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; -import { ExtendServiceError, executeExtendJob } from "@/lib/ai/extend/service"; -import { getPointsSystemConfig } from "@/lib/db/queries"; -import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; -import { getImageToolCreditCost } from "@/lib/points/config"; -import { - getFriendlyRateLimitMessage, - getUserFacingRequestErrorMessage, - isRateLimitError, -} from "@/lib/request-error"; - +import { executeExtendPOST } from "@/lib/canvas/operations/extend"; +export const runtime = "nodejs"; export const maxDuration = 180; - -function parseIncludeBase64(value: unknown): boolean { - if (typeof value === "boolean") return value; - const normalized = String(value ?? "").trim().toLowerCase(); - return normalized === "true" || normalized === "1" || normalized === "yes"; -} - -function buildExtendSuccessPayload(result: Awaited>) { - return { - success: true, - original_url: result.original_url, - thumbnail_url: result.thumbnail_url, - ...(result.image_base64 ? { base64: result.image_base64 } : {}), - processing_time: result.processing_time, - }; -} - -export async function POST(request: NextRequest) { - try { - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json({ error: "未授权，请先登录" }, { status: 401 }); - } - - const contentType = request.headers.get("content-type") || ""; - - let imageUrl: string | null = null; - let imageBase64: string | null = null; - let prompt = "图像拓展处理"; - let chatId: string | null = null; - let messageId: string | null = null; - let includeBase64 = false; - const requestId = `extend-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; - - if (contentType.includes("application/json")) { - const body = await request.json(); - imageUrl = body.image_url ? String(body.image_url).trim() : null; - imageBase64 = body.image_base64 ? String(body.image_base64).trim() : null; - prompt = body.prompt || prompt; - chatId = body.chatId || null; - messageId = body.messageId || null; - includeBase64 = parseIncludeBase64(body.includeBase64); - - if (!imageUrl && !imageBase64) { - return NextResponse.json( - { error: "缺少image_base64或image_url参数" }, - { status: 400 }, - ); - } - - console.log("[Extend API] Request received:", { - requestId, - source: imageUrl ? "url" : "base64", - includeBase64, - chatId, - messageId, - xScale: body.xScale, - yScale: body.yScale, - }); - - const pointsConfig = await getPointsSystemConfig(); - const pointsCharge = await prepareFeaturePointsCharge({ - userId: session.user.id, - unitCost: getImageToolCreditCost(pointsConfig, "extend"), - reason: "image_extend", - modelId: "image-extend", - note: "图片扩图消耗积分", - pointsConfig, - }); - if (!pointsCharge.ok) { - return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); - } - - const result = await executeExtendJob({ - userId: session.user.id, - imageUrl, - imageBase64, - prompt, - chatId, - messageId, - xScale: body.xScale ? Number(body.xScale) : undefined, - yScale: body.yScale ? Number(body.yScale) : undefined, - includeBase64, - requestId, - }); - - if (pointsCharge.chargeable) { - try { - await pointsCharge.consume(); - } catch (pointsError) { - console.error("[Points] Failed to consume extend points:", pointsError); - } - } - - console.log("[Extend API] Request completed:", { - requestId, - includeBase64, - processingTime: result.processing_time, - originalUrl: result.original_url.slice(0, 120), - }); - - return NextResponse.json(buildExtendSuccessPayload(result)); - } - - if (contentType.includes("multipart/form-data")) { - const formData = await request.formData(); - const file = formData.get("file") as File | null; - if (!file) { - return NextResponse.json({ error: "缺少file参数" }, { status: 400 }); - } - - const arrayBuffer = await file.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - const multipartChatId = String(formData.get("chatId") || "") || null; - const multipartMessageId = String(formData.get("messageId") || "") || null; - includeBase64 = parseIncludeBase64(formData.get("includeBase64")); - console.log("[Extend API] Multipart request received:", { - requestId, - includeBase64, - chatId: multipartChatId, - messageId: multipartMessageId, - bytes: buffer.length, - }); - - const pointsConfig = await getPointsSystemConfig(); - const pointsCharge = await prepareFeaturePointsCharge({ - userId: session.user.id, - unitCost: getImageToolCreditCost(pointsConfig, "extend"), - reason: "image_extend", - modelId: "image-extend", - note: "图片扩图消耗积分", - pointsConfig, - }); - if (!pointsCharge.ok) { - return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); - } - - const result = await executeExtendJob({ - userId: session.user.id, - imageBase64: buffer.toString("base64"), - prompt: String(formData.get("prompt") || prompt), - chatId: multipartChatId, - messageId: multipartMessageId, - includeBase64, - requestId, - }); - - if (pointsCharge.chargeable) { - try { - await pointsCharge.consume(); - } catch (pointsError) { - console.error("[Points] Failed to consume extend points:", pointsError); - } - } - - console.log("[Extend API] Multipart request completed:", { - requestId, - includeBase64, - processingTime: result.processing_time, - originalUrl: result.original_url.slice(0, 120), - }); - - return NextResponse.json(buildExtendSuccessPayload(result)); - } - - return NextResponse.json( - { error: "不支持的Content-Type，请使用application/json或multipart/form-data" }, - { status: 400 }, - ); - } catch (error) { - if (error instanceof ExtendServiceError) { - if (error.status === 429 || isRateLimitError(error)) { - return NextResponse.json( - { - error: getFriendlyRateLimitMessage("当前操作过于频繁，请稍后再试"), - detail: error.detail || error.message, - retryAfter: 10, - }, - { - status: 429, - headers: { - "Retry-After": "10", - }, - }, - ); - } - - return NextResponse.json( - { - error: getUserFacingRequestErrorMessage(error, "图像拓展失败"), - detail: error.detail || error.message, - }, - { status: error.status }, - ); - } - - console.error("[Extend API] 错误:", error); - if (isRateLimitError(error)) { - return NextResponse.json( - { - error: getFriendlyRateLimitMessage("当前操作过于频繁，请稍后再试"), - detail: error instanceof Error ? error.message : String(error), - retryAfter: 10, - }, - { - status: 429, - headers: { - "Retry-After": "10", - }, - }, - ); - } - return NextResponse.json( - { - error: getUserFacingRequestErrorMessage(error, "图像拓展失败"), - detail: error instanceof Error ? error.message : String(error), - }, - { status: 500 }, - ); - } +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + return executeExtendPOST(request, session.user.id); } diff --git a/app/api/flow-board/_utils.ts b/app/api/flow-board/_utils.ts index 61a5de0f4..7ac064783 100644 --- a/app/api/flow-board/_utils.ts +++ b/app/api/flow-board/_utils.ts @@ -1,4 +1,8 @@ -import type { FlowBoardDraft, FlowBoardSnapshotData } from "@/lib/db/schema"; +import type { + FlowBoardDraft, + FlowBoardSheetSnapshotData, + FlowBoardSnapshotData, +} from "@/lib/db/schema"; export function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); @@ -8,6 +12,73 @@ function normalizeRecord(value: unknown): Record | null { return isRecord(value) ? value : null; } +const DEFAULT_FLOW_BOARD_CANVAS = { + backgroundColor: "#f7f8fb", + gridColor: "#d8dde8", + gridEnabled: false, + gridSize: 24, +}; + +function normalizeCanvasColor(value: unknown, fallback: string) { + return typeof value === "string" && /^#[0-9a-f]{6}$/i.test(value) + ? value.toLowerCase() + : fallback; +} + +function normalizeFlowBoardCanvas(value: unknown): Record { + const canvas = isRecord(value) ? value : {}; + const requestedGridSize = + typeof canvas.gridSize === "number" && Number.isFinite(canvas.gridSize) + ? Math.round(canvas.gridSize) + : DEFAULT_FLOW_BOARD_CANVAS.gridSize; + + return { + backgroundColor: normalizeCanvasColor( + canvas.backgroundColor, + DEFAULT_FLOW_BOARD_CANVAS.backgroundColor + ), + gridColor: normalizeCanvasColor( + canvas.gridColor, + DEFAULT_FLOW_BOARD_CANVAS.gridColor + ), + gridEnabled: + typeof canvas.gridEnabled === "boolean" + ? canvas.gridEnabled + : DEFAULT_FLOW_BOARD_CANVAS.gridEnabled, + gridSize: Math.min(64, Math.max(8, requestedGridSize)), + }; +} + +function normalizeSheetId(value: unknown, index: number) { + const id = typeof value === "string" ? value.trim().slice(0, 128) : ""; + return id || `sheet-${index + 1}`; +} + +function normalizeSheetTitle(value: unknown, index: number) { + const title = typeof value === "string" ? value.trim().slice(0, 128) : ""; + return title || `画布 ${index + 1}`; +} + +function normalizeFlowBoardSheet( + value: unknown, + index: number +): FlowBoardSheetSnapshotData | null { + if (!isRecord(value)) { + return null; + } + + return { + id: normalizeSheetId(value.id, index), + title: normalizeSheetTitle(value.title, index), + children: Array.isArray(value.children) + ? value.children.filter(isRecord) + : [], + viewport: normalizeRecord(value.viewport), + theme: normalizeRecord(value.theme), + canvas: normalizeFlowBoardCanvas(value.canvas), + }; +} + export function normalizeFlowBoardTitle(value: unknown) { const title = String(value || "").trim(); return title.slice(0, 256) || "未命名流程白板"; @@ -17,15 +88,67 @@ export function normalizeFlowBoardSnapshot( value: unknown ): FlowBoardSnapshotData { const raw = isRecord(value) ? value : {}; + const legacyChildren = Array.isArray(raw.children) + ? raw.children.filter(isRecord) + : []; + const normalizedSheets = Array.isArray(raw.sheets) + ? raw.sheets + .map(normalizeFlowBoardSheet) + .filter( + (sheet): sheet is FlowBoardSheetSnapshotData => Boolean(sheet) + ) + : []; + const sheets = + normalizedSheets.length > 0 + ? normalizedSheets + : [ + { + id: "sheet-1", + title: "画布 1", + children: legacyChildren, + viewport: normalizeRecord(raw.viewport), + theme: normalizeRecord(raw.theme), + canvas: normalizeFlowBoardCanvas(raw.canvas), + }, + ]; + const requestedActiveSheetId = + typeof raw.activeSheetId === "string" ? raw.activeSheetId.trim() : ""; + const activeSheet = + sheets.find((sheet) => sheet.id === requestedActiveSheetId) || sheets[0]; + const activeChildren = Array.isArray(raw.children) + ? legacyChildren + : activeSheet.children; + const activeViewport = isRecord(raw.viewport) + ? raw.viewport + : activeSheet.viewport; + const activeTheme = isRecord(raw.theme) ? raw.theme : activeSheet.theme; + const activeCanvas = isRecord(raw.canvas) + ? normalizeFlowBoardCanvas(raw.canvas) + : normalizeFlowBoardCanvas(activeSheet.canvas); + const nextSheets = sheets.map((sheet) => + sheet.id === activeSheet.id + ? { + ...sheet, + children: activeChildren, + viewport: activeViewport, + theme: activeTheme, + canvas: activeCanvas, + } + : sheet + ); const updatedAt = typeof raw.updatedAt === "string" && raw.updatedAt.trim() ? raw.updatedAt : new Date().toISOString(); return { - children: Array.isArray(raw.children) ? raw.children.filter(isRecord) : [], - viewport: normalizeRecord(raw.viewport), - theme: normalizeRecord(raw.theme), + version: 2, + activeSheetId: activeSheet.id, + sheets: nextSheets, + children: activeChildren, + viewport: activeViewport, + theme: activeTheme, + canvas: activeCanvas, updatedAt, }; } diff --git a/app/api/gallery/route.ts b/app/api/gallery/route.ts index 88ccadb02..4b6711d8e 100644 --- a/app/api/gallery/route.ts +++ b/app/api/gallery/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from "next/server"; import { db } from "@/lib/db"; -import { generatedImage } from "@/lib/db/schema"; -import { eq, desc, sql } from "drizzle-orm"; +import { generatedImage, imageArchiveTask } from "@/lib/db/schema"; +import { and, desc, eq, isNull, sql } from "drizzle-orm"; import { auth } from "@/app/(auth)/auth"; export async function GET(request: NextRequest) { @@ -19,12 +19,23 @@ export async function GET(request: NextRequest) { const page = Number.parseInt(searchParams.get("page") || "1"); const pageSize = Number.parseInt(searchParams.get("pageSize") || "24"); const offset = (page - 1) * pageSize; + const visibleImageFilter = and( + eq(generatedImage.userId, userId), + isNull(imageArchiveTask.id) + ); // 查询总数 const countResult = await db .select({ count: sql`count(*)::int` }) .from(generatedImage) - .where(eq(generatedImage.userId, userId)); + .leftJoin( + imageArchiveTask, + and( + eq(imageArchiveTask.sourceUrl, generatedImage.originalUrl), + eq(imageArchiveTask.status, "abandoned") + ) + ) + .where(visibleImageFilter); const total = countResult[0]?.count || 0; @@ -42,7 +53,14 @@ export async function GET(request: NextRequest) { modelId: generatedImage.modelId, // 添加 modelId 字段 }) .from(generatedImage) - .where(eq(generatedImage.userId, userId)) + .leftJoin( + imageArchiveTask, + and( + eq(imageArchiveTask.sourceUrl, generatedImage.originalUrl), + eq(imageArchiveTask.status, "abandoned") + ) + ) + .where(visibleImageFilter) .orderBy(desc(generatedImage.createdAt)) .limit(pageSize) .offset(offset); diff --git a/app/api/generated-image-preview/[key]/route.ts b/app/api/generated-image-preview/[key]/route.ts new file mode 100644 index 000000000..58cb50330 --- /dev/null +++ b/app/api/generated-image-preview/[key]/route.ts @@ -0,0 +1,34 @@ +import { NextRequest, NextResponse } from "next/server"; +import { eq } from "drizzle-orm"; +import { db } from "@/lib/db"; +import { imageArchiveTask } from "@/lib/db/schema"; + +// The SHA-256 URL is a capability, like the provider's signed image URL. It must be +// readable by image nodes and reference-image providers without a session cookie. +export async function GET(request: NextRequest, { params }: { params: Promise<{ key: string }> }) { + const { key } = await params; + if (!/^[a-f0-9]{64}$/.test(key)) return new NextResponse(null, { status: 404 }); + const configured = + process.env.NEXT_PUBLIC_APP_URL || process.env.AUTH_URL || process.env.NEXTAUTH_URL; + const origin = configured + ? new URL(configured).origin + : `http://localhost:${process.env.PORT || "3000"}`; + const sourceUrl = `${origin}/api/generated-image-preview/${key}`; + const [task] = await db + .select({ targetUrl: imageArchiveTask.targetUrl, metadata: imageArchiveTask.metadata }) + .from(imageArchiveTask) + .where(eq(imageArchiveTask.sourceUrl, sourceUrl)) + .limit(1); + if (task?.targetUrl) return NextResponse.redirect(task.targetUrl, 307); + const inline = (task?.metadata as { inlineSource?: string } | null)?.inlineSource; + const match = inline?.match(/^data:(image\/[a-z0-9.+-]+);base64,([a-z0-9+/=]+)$/i); + if (!match) return new NextResponse(null, { status: 404 }); + return new NextResponse(Buffer.from(match[2], "base64"), { + headers: { + "Content-Type": match[1], + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + "Content-Security-Policy": "default-src 'none'; sandbox", + }, + }); +} diff --git a/app/api/messages/route.ts b/app/api/messages/route.ts index 65b4f9114..247f29bb3 100644 --- a/app/api/messages/route.ts +++ b/app/api/messages/route.ts @@ -14,6 +14,10 @@ import { updateMessageById, } from "@/lib/db/queries"; import type { DBMessage } from "@/lib/db/schema"; +import { + buildOpenClawArtifactProxyUrl, + normalizeOpenClawArtifacts, +} from "@/lib/openclaw-chat/artifacts"; import { getFalImageTaskStatus, resolveFalImageTaskToMarkdown, @@ -32,12 +36,29 @@ const FAL_IMAGE_TASK_POLL_INTERVAL_MS = 2_000; const falTaskLastPolledAt = new Map(); const falTaskSyncInFlight = new Set(); -function toPublicVideoMetadata(value: DBMessage["videoMetadata"]) { +function toPublicVideoMetadata(params: { + message: DBMessage; + requestOrigin: string; +}) { + const value = params.message.videoMetadata; if (!value || typeof value !== "object" || Array.isArray(value)) return value; const metadata = value as Record; if (metadata.taskKind !== "clawd_chat") return value; const publicMetadata = { ...metadata }; delete publicMetadata.upstreamTaskKey; + publicMetadata.artifacts = normalizeOpenClawArtifacts(metadata.artifacts).map( + (artifact, artifactIndex) => ({ + ...artifact, + downloadUrl: new URL( + buildOpenClawArtifactProxyUrl({ + chatId: params.message.chatId, + messageId: params.message.id, + artifactIndex, + }), + params.requestOrigin + ).toString(), + }) + ); return publicMetadata; } @@ -286,7 +307,11 @@ export async function GET(request: NextRequest) { createdAt: msg.createdAt, parts: parts, attachments: msg.attachments || [], - videoMetadata: toPublicVideoMetadata(msg.videoMetadata) || null, + videoMetadata: + toPublicVideoMetadata({ + message: msg, + requestOrigin: request.nextUrl.origin, + }) || null, }; }); diff --git a/app/api/rmbg/route.ts b/app/api/rmbg/route.ts index 889f44c42..2b2e3caeb 100644 --- a/app/api/rmbg/route.ts +++ b/app/api/rmbg/route.ts @@ -1,138 +1,10 @@ -import { NextRequest, NextResponse } from "next/server"; - +import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; -import { - executeRmbgJob, - fetchImageBufferFromUrl, - RmbgServiceError, -} from "@/lib/ai/rmbg/service"; -import { getPointsSystemConfig } from "@/lib/db/queries"; -import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; -import { getImageToolCreditCost } from "@/lib/points/config"; - +import { executeRemoveBackgroundPOST } from "@/lib/canvas/operations/rmbg"; export const runtime = "nodejs"; export const maxDuration = 60; - -export async function POST(request: NextRequest) { - try { - const session = await auth(); - if (!session?.user?.id) { - return NextResponse.json({ error: "未授权，请先登录" }, { status: 401 }); - } - - const contentType = request.headers.get("content-type") || ""; - - let imageBuffer: Buffer; - let imageBase64: string | undefined; - let prompt = "抠图处理"; - let chatId: string | null = null; - let messageId: string | null = null; - let skipGallerySave = false; - let sourceImageUrl: string | null = null; - - if (contentType.includes("application/json")) { - const body = await request.json(); - prompt = body.prompt || "抠图处理"; - chatId = body.chatId || null; - messageId = body.messageId || null; - skipGallerySave = body.skipGallerySave === true || body.skip_gallery_save === 1; - - const imageUrl = body.imageUrl || body.image_url; - const explicitSourceImageUrl = body.sourceImageUrl || body.source_image_url; - - if (imageUrl) { - sourceImageUrl = String(imageUrl); - imageBuffer = await fetchImageBufferFromUrl(sourceImageUrl); - } else if (body.image_base64) { - imageBase64 = String(body.image_base64); - imageBuffer = Buffer.from(imageBase64, "base64"); - if (explicitSourceImageUrl) { - sourceImageUrl = String(explicitSourceImageUrl); - } - } else { - return NextResponse.json( - { error: "缺少 imageUrl 或 image_base64 参数" }, - { status: 400 } - ); - } - } else if (contentType.includes("multipart/form-data")) { - const formData = await request.formData(); - const file = formData.get("file") as File | null; - - if (!file) { - return NextResponse.json({ error: "缺少file参数" }, { status: 400 }); - } - - const arrayBuffer = await file.arrayBuffer(); - imageBuffer = Buffer.from(arrayBuffer); - imageBase64 = imageBuffer.toString("base64"); - prompt = (formData.get("prompt") as string) || "抠图处理"; - chatId = (formData.get("chatId") as string) || null; - messageId = (formData.get("messageId") as string) || null; - skipGallerySave = - formData.get("skipGallerySave") === "true" || formData.get("skip_gallery_save") === "1"; - sourceImageUrl = - (formData.get("sourceImageUrl") as string) || - (formData.get("source_image_url") as string) || - null; - } else { - return NextResponse.json( - { error: "不支持的Content-Type，请使用application/json或multipart/form-data" }, - { status: 400 } - ); - } - - const pointsConfig = await getPointsSystemConfig(); - const pointsCharge = await prepareFeaturePointsCharge({ - userId: session.user.id, - unitCost: getImageToolCreditCost(pointsConfig, "remove-background"), - reason: "image_remove_background", - modelId: "rmbg", - note: "图片抠图消耗积分", - pointsConfig, - }); - if (!pointsCharge.ok) { - return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); - } - - const result = await executeRmbgJob({ - userId: session.user.id, - imageBuffer, - imageBase64, - prompt, - chatId, - messageId, - skipGallerySave, - sourceImageUrl, - }); - - if (pointsCharge.chargeable) { - try { - await pointsCharge.consume(); - } catch (pointsError) { - console.error("[Points] Failed to consume RMBG points:", pointsError); - } - } - - return NextResponse.json({ - success: true, - ...result, - }); - } catch (error) { - if (error instanceof RmbgServiceError) { - return NextResponse.json( - { - error: error.message, - detail: error.detail || error.message, - }, - { status: error.status } - ); - } - - console.error("[RMBG API] 处理错误:", error); - return NextResponse.json( - { error: "服务器内部错误", detail: error instanceof Error ? error.message : "未知错误" }, - { status: 500 } - ); - } +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + return executeRemoveBackgroundPOST(request, session.user.id); } diff --git a/app/api/site-settings/route.ts b/app/api/site-settings/route.ts index 36a5256f0..adfa919cb 100644 --- a/app/api/site-settings/route.ts +++ b/app/api/site-settings/route.ts @@ -399,7 +399,7 @@ function normalizeLovartComposerPreviewTexts(value: unknown): string[] { function normalizeLovartComposerPrefix(value: unknown): string { const normalized = String(value || "").trim(); if (!normalized) { - return "让「由壹点AI」"; + return "让「𝗕𝘂𝗬𝗶」"; } return normalized.slice(0, 40); } @@ -573,7 +573,7 @@ const PUBLIC_SITE_SETTINGS_SCOPES = new Set([ ]); const HOME_SITE_SETTINGS_FALLBACK: Record = { - siteName: "AI Chatbot", + siteName: "𝗕𝘂𝗬𝗶", siteLogo: null, siteLogoDark: null, favicon: null, @@ -583,11 +583,11 @@ const HOME_SITE_SETTINGS_FALLBACK: Record = { bannerText: "限时特惠：升级会员最高立享 45% OFF!", bannerLink: null, bannerActionText: "立即查看", - lovartHeroTitle: "由壹点AI 让设计更简单", - lovartHeroSubtitle: "懂你的设计代理，帮你搞定一切", + lovartHeroTitle: "𝗕𝘂𝗬𝗶 让设计触手可及", + lovartHeroSubtitle: "下达指令，即刻呈现", surpriseCategories: ["全部", "精选"], canvasDefaultModelId: null, - lovartComposerPrefix: "让「由壹点AI」", + lovartComposerPrefix: "让「𝗕𝘂𝗬𝗶」", lovartComposerPreviewTexts: [], lovartPromptTagVisibility: {}, contactConfig: normalizeContactConfig(undefined), @@ -664,7 +664,7 @@ async function readScopedSiteSettings(keys: string[]) { function createBrandingSettings(settings: Record | undefined) { return { - siteName: String(settings?.siteName || "AI Chatbot"), + siteName: String(settings?.siteName || "𝗕𝘂𝗬𝗶"), siteLogo: settings?.siteLogo || null, siteLogoDark: settings?.siteLogoDark || null, favicon: settings?.favicon || null, @@ -822,7 +822,7 @@ async function getScopedPublicSiteSettings( usageTimeStart: typedSettings?.usageTimeStart || "08:00", usageTimeEnd: typedSettings?.usageTimeEnd || "20:00", usageTimeMessage: - typedSettings?.usageTimeMessage || "由壹点累了也是需要休息的", + typedSettings?.usageTimeMessage || "BuYi 累了也是需要休息的", }); } @@ -953,9 +953,9 @@ async function getScopedPublicSiteSettings( typedSettings?.lovartPromoText || "现在升级，立享 Kling 3.0 与 Nano Banana Pro，最高 5 折优惠", lovartPromoActionText: typedSettings?.lovartPromoActionText || "立即升级", - lovartHeroTitle: typedSettings?.lovartHeroTitle || "由壹点AI 让设计更简单", + lovartHeroTitle: typedSettings?.lovartHeroTitle || "𝗕𝘂𝗬𝗶 让设计触手可及", lovartHeroSubtitle: - typedSettings?.lovartHeroSubtitle || "懂你的设计代理，帮你搞定一切", + typedSettings?.lovartHeroSubtitle || "下达指令，即刻呈现", surpriseCategories: normalizeSurpriseCategories( valueByKey.get(HOME_SURPRISE_CATEGORIES_KEY) ), @@ -1476,7 +1476,7 @@ export async function GET(request: NextRequest) { usageTimeEnabled: false, usageTimeStart: "08:00", usageTimeEnd: "20:00", - usageTimeMessage: "由壹点累了也是需要休息的", + usageTimeMessage: "BuYi 累了也是需要休息的", bannerEnabled: true, bannerText: "限时特惠：升级会员最高立享 45% OFF!", bannerLink: null, @@ -1525,8 +1525,8 @@ export async function GET(request: NextRequest) { canvasDetailAgentModelId: null, lovartPromoText: "现在升级，立享 Kling 3.0 与 Nano Banana Pro，最高 5 折优惠", lovartPromoActionText: "立即升级", - lovartHeroTitle: "由壹点AI 让设计更简单", - lovartHeroSubtitle: "懂你的设计代理，帮你搞定一切", + lovartHeroTitle: "𝗕𝘂𝗬𝗶 让设计触手可及", + lovartHeroSubtitle: "下达指令，即刻呈现", surpriseCategories, aiDetailGenerationEnabled, lovartComposerPrefix, @@ -1546,8 +1546,8 @@ export async function GET(request: NextRequest) { baseSettings.lovartPromoText || "现在升级，立享 Kling 3.0 与 Nano Banana Pro，最高 5 折优惠", lovartPromoActionText: baseSettings.lovartPromoActionText || "立即升级", - lovartHeroTitle: baseSettings.lovartHeroTitle || "由壹点AI 让设计更简单", - lovartHeroSubtitle: baseSettings.lovartHeroSubtitle || "懂你的设计代理，帮你搞定一切", + lovartHeroTitle: baseSettings.lovartHeroTitle || "𝗕𝘂𝗬𝗶 让设计触手可及", + lovartHeroSubtitle: baseSettings.lovartHeroSubtitle || "下达指令，即刻呈现", notification, surpriseCategories, aiDetailGenerationEnabled, @@ -3119,7 +3119,7 @@ export async function PUT(request: NextRequest) { usageTimeEnabled: siteSettingsPayload.usageTimeEnabled ?? false, usageTimeStart: siteSettingsPayload.usageTimeStart ?? "08:00", usageTimeEnd: siteSettingsPayload.usageTimeEnd ?? "20:00", - usageTimeMessage: siteSettingsPayload.usageTimeMessage ?? "由壹点累了也是需要休息的", + usageTimeMessage: siteSettingsPayload.usageTimeMessage ?? "BuYi 累了也是需要休息的", bannerEnabled: siteSettingsPayload.bannerEnabled ?? true, bannerText: siteSettingsPayload.bannerText ?? "限时特惠：升级会员最高立享 45% OFF!", bannerLink: siteSettingsPayload.bannerLink, @@ -3144,8 +3144,8 @@ export async function PUT(request: NextRequest) { siteSettingsPayload.lovartPromoText ?? "现在升级，立享 Kling 3.0 与 Nano Banana Pro，最高 5 折优惠", lovartPromoActionText: siteSettingsPayload.lovartPromoActionText ?? "立即升级", - lovartHeroTitle: siteSettingsPayload.lovartHeroTitle ?? "由壹点AI 让设计更简单", - lovartHeroSubtitle: siteSettingsPayload.lovartHeroSubtitle ?? "懂你的设计代理，帮你搞定一切", + lovartHeroTitle: siteSettingsPayload.lovartHeroTitle ?? "𝗕𝘂𝗬𝗶 让设计触手可及", + lovartHeroSubtitle: siteSettingsPayload.lovartHeroSubtitle ?? "下达指令，即刻呈现", updatedBy: session.user.id, }) .returning(); diff --git a/app/api/tvstar/plugin/download/route.ts b/app/api/tvstar/plugin/download/route.ts new file mode 100644 index 000000000..6a5476938 --- /dev/null +++ b/app/api/tvstar/plugin/download/route.ts @@ -0,0 +1,35 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import JSZip from "jszip"; +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; + +export const runtime = "nodejs"; + +const packageFiles = [ + "package.json", "package-lock.json", "api-client.mjs", "cli.mjs", "mcp-server.mjs", + "yedir-canvas/SKILL.md", "yedir-canvas/agents/openai.yaml", + "yedir-canvas/references/operation-protocol.md", +]; + +export async function GET() { + const session = await auth(); + if (!session?.user?.id || session.user.type === "guest") { + return NextResponse.json({ error: "请先登录正式账号" }, { status: 401 }); + } + const zip = new JSZip(); + const base = path.join(process.cwd(), "integrations", "openclaw"); + try { + await Promise.all(packageFiles.map(async file => { + zip.file(`tvstar-mcp/${file}`, await readFile(path.join(base, file))); + })); + const archive = await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" }); + return new NextResponse(archive, { headers: { + "Content-Type": "application/zip", + "Content-Disposition": 'attachment; filename="tvstar-mcp.zip"', + "Cache-Control": "private, no-store", + } }); + } catch { + return NextResponse.json({ error: "插件安装包暂不可用" }, { status: 503 }); + } +} diff --git a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx index 8ebf45e4f..2069ce994 100644 --- a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx +++ b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx @@ -1,5 +1,6 @@ "use client"; +import { createCanvasSaveQueue } from "@/lib/client/canvas-save-queue"; import { AlertCircle, AlertTriangle, @@ -74,6 +75,7 @@ import { useState, } from "react"; import { toast } from "sonner"; +import { useCanvasImageArchives } from "@/hooks/use-canvas-image-archives"; import useSWR from "swr"; // 深色主题组件 V2 (新版布局) import type { AgentGenerationWorkspaceDarkV2Props } from "@/components/agents/AgentGenerationWorkspaceDarkV2"; @@ -86,7 +88,9 @@ import { CanvasAddImageDialog, type CanvasAddImageDialogImage, } from "@/components/canvas/CanvasAddImageDialog"; +import { CanvasAgentUiListener } from "@/components/canvas/chat-panel/canvas-agent-ui-listener"; import { CanvasToolbarIcon } from "@/components/canvas/CanvasToolbarIcon"; +import { CanvasSurfaceHeader } from "@/components/canvas/CanvasSurfaceHeader"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { @@ -249,7 +253,6 @@ import type { AgentWorkflowUiConfig, } from "@/types/agent"; import type { - AgentSlotNodeData, CanvasEdge, CanvasInitialProject, CanvasNode, @@ -262,7 +265,7 @@ import type { const CANVAS_SITE_SETTINGS_REFRESH_INTERVAL_MS = 15000; const CANVAS_SITE_SETTINGS_TIMEOUT_MS = 10000; -const CANVAS_SURFACE_BACKGROUND = "#f4f4f5"; +const CANVAS_SURFACE_BACKGROUND = "#f5f5f5"; const fetchCanvasSiteSettings = async (url: string) => { const separator = url.includes("?") ? "&" : "?"; @@ -1906,6 +1909,8 @@ export default function CanvasEditorLegacyPage({ [isVideoMode] ); + useCanvasImageArchives(id); + // Store state const projectTitle = useCanvasStore((state) => state.projectTitle); const viewport = useCanvasStore(canvasSelectors.viewport); @@ -3141,8 +3146,7 @@ export default function CanvasEditorLegacyPage({ const updateWidthFromClientX = (clientX: number) => { const currentViewportWidth = window.innerWidth; - const nextWidth = - currentViewportWidth - DOCKED_PANEL_INSET - clientX; + const nextWidth = currentViewportWidth - DOCKED_PANEL_INSET - clientX; setFloatingPanelWidth( clampFloatingPanelWidth(nextWidth, currentViewportWidth) @@ -5072,59 +5076,84 @@ export default function CanvasEditorLegacyPage({ workflowInputs.brand, ]); + const canvasSaveQueue = useRef | null>(null); + if (!canvasSaveQueue.current) canvasSaveQueue.current = createCanvasSaveQueue(); + const canvasAutosaveTimer = useRef | null>(null); + const canvasPersistenceScope = useRef(id); + canvasPersistenceScope.current = id; const persistCanvasState = useCallback( - async (reason: "autosave" | "delete" | "beforeunload") => { - const canvasData = buildPersistedCanvasData(); - const { canvasRevision, canvasUpdatedAt } = useCanvasStore.getState(); - const result = await canvasService.saveCanvasState( - id, - canvasData, - activeCanvasWorkspaceType, - { - baseCanvasRevision: canvasRevision, - baseCanvasUpdatedAt: canvasUpdatedAt, - } - ); - if (!result.success) { - console.error(`[Canvas] ${reason} save failed:`, result.error); - if (result.status === 409) { - externalCanvasConflictRef.current = true; - console.warn("[Canvas] stale save revision handled without user prompt", { - reason, - canvasUpdatedAt: result.canvasUpdatedAt, - }); - if (!externalCanvasConflictNotifiedRef.current) { - externalCanvasConflictNotifiedRef.current = true; - toast.warning("画布已被外部操作更新，本地改动未覆盖远端；请刷新后继续编辑"); + (reason: "autosave" | "delete" | "beforeunload") => canvasSaveQueue.current!.enqueue(async () => { + if (canvasPersistenceScope.current !== id) return false; + setIsSaving(true); + try { + const canvasData = buildPersistedCanvasData(); + const { canvasRevision, canvasUpdatedAt } = useCanvasStore.getState(); + const result = await canvasService.saveCanvasState( + id, + canvasData, + activeCanvasWorkspaceType, + { + baseCanvasRevision: canvasRevision, + baseCanvasUpdatedAt: canvasUpdatedAt, } + ); + if (canvasPersistenceScope.current !== id) return false; + if (!result.success) { + console.error(`[Canvas] ${reason} save failed:`, result.error); + if (result.status === 409) { + externalCanvasConflictRef.current = true; + console.warn("[Canvas] stale save revision handled without user prompt", { + reason, + canvasUpdatedAt: result.canvasUpdatedAt, + }); + if (!externalCanvasConflictNotifiedRef.current) { + externalCanvasConflictNotifiedRef.current = true; + toast.warning("画布已被外部操作更新，本地改动未覆盖远端；请刷新后继续编辑"); + } + } + } else if (result.data?.updatedAt) { + externalCanvasConflictRef.current = false; + externalCanvasConflictNotifiedRef.current = false; + useCanvasStore + .getState() + .setCanvasRevision(String(result.data.updatedAt), String(result.data.updatedAt)); } - } else if (result.data?.updatedAt) { - externalCanvasConflictRef.current = false; - externalCanvasConflictNotifiedRef.current = false; - useCanvasStore - .getState() - .setCanvasRevision(String(result.data.updatedAt), String(result.data.updatedAt)); + return result.success; + } finally { + if (canvasPersistenceScope.current === id) setIsSaving(false); } - return result.success; - }, + }), [activeCanvasWorkspaceType, buildPersistedCanvasData, id] ); + const persistCanvasBeforeAgent = useCallback(() => { + // Flush the pending local edit once instead of racing its scheduled autosave. + if (canvasAutosaveTimer.current) { + clearTimeout(canvasAutosaveTimer.current); + canvasAutosaveTimer.current = null; + } + return persistCanvasState("autosave"); + }, [persistCanvasState]); + // 自动保存逻辑 useEffect(() => { if (isLoading || isAgentMode || isAgentOperationRefreshActive()) return; const saveTimer = setTimeout(async () => { - setIsSaving(true); + canvasAutosaveTimer.current = null; + if (isAgentOperationRefreshActive()) return; try { await persistCanvasState("autosave"); } catch (error) { console.error("Auto save failed:", error); } - setIsSaving(false); }, 3000); // 3秒自动保存 - return () => clearTimeout(saveTimer); + canvasAutosaveTimer.current = saveTimer; + return () => { + clearTimeout(saveTimer); + if (canvasAutosaveTimer.current === saveTimer) canvasAutosaveTimer.current = null; + }; }, [isAgentMode, isAgentOperationRefreshActive, nodes, isLoading, persistCanvasState]); useEffect(() => { @@ -7515,7 +7544,7 @@ export default function CanvasEditorLegacyPage({ materialMimeType: image.mimeType || undefined, originalHeight: dimensions.originalHeight, originalWidth: dimensions.originalWidth, - source: "agent-workflow-material-search", + source: image.sourceType === "upload" ? "local-upload" : "agent-workflow-material-search", }, }, connections: [], @@ -7527,7 +7556,7 @@ export default function CanvasEditorLegacyPage({ if (createdNodeIds.length > 0) { selectNodes(createdNodeIds); - toast.success(`已添加 ${createdNodeIds.length} 张白底图到画布`); + toast.success(`已添加 ${createdNodeIds.length} 张${validImages.every((image) => image.sourceType === "upload") ? "图片" : "白底图"}到画布`); } }, [createCanvasNode, getVisibleCanvasOrigin, selectNodes] @@ -7543,64 +7572,6 @@ export default function CanvasEditorLegacyPage({ setIsAddImageDialogOpen(true); }, [handleAddWorkflowNode, isVideoMode]); - const handleAddAgentSlotNode = useCallback((slotType: AgentSlotNodeData["slotType"]) => { - if (isVideoMode) return; - - const slotDefaults: Record< - AgentSlotNodeData["slotType"], - { - label: string; - size: CanvasNode["size"]; - aspectRatio: string; - } - > = { - image: { - label: "图片槽位", - size: { width: 320, height: 320 }, - aspectRatio: "1:1", - }, - video: { - label: "视频槽位", - size: { width: 360, height: 220 }, - aspectRatio: "16:9", - }, - "ui-screen": { - label: "UI 槽位", - size: { width: 360, height: 260 }, - aspectRatio: "16:9", - }, - }; - const defaults = slotDefaults[slotType]; - const origin = getVisibleCanvasOrigin(); - createCanvasNode({ - type: "agent-slot", - position: { - x: origin.x + 40, - y: origin.y + 40, - }, - size: defaults.size, - data: { - slotType, - label: defaults.label, - status: "empty", - aspectRatio: defaults.aspectRatio, - fillPolicy: "replace-empty", - acceptedSources: ["text", "image", "selection", "annotation"], - filledNodeIds: [], - metadata: { - createdBy: "toolbar", - createdSlotType: slotType, - createdAt: new Date().toISOString(), - }, - }, - connections: [], - }); - }, [createCanvasNode, getVisibleCanvasOrigin, isVideoMode]); - - const handleAddAgentImageSlotNode = useCallback(() => { - handleAddAgentSlotNode("image"); - }, [handleAddAgentSlotNode]); - const handleAddVideoNode = useCallback(() => { if (isVideoMode) { handleAddWorkflowNode("video-input"); @@ -9374,6 +9345,7 @@ export default function CanvasEditorLegacyPage({ // 键盘快捷键 useEffect(() => { const handleKeyDown = (e: KeyboardEvent) => { + if (useCanvasStore.getState().canvasImageSelectionSession) return; // 如果正在输入则不触发快捷键（包括 input、textarea 和 contentEditable 元素） if (isCanvasTextEntryTarget(e.target)) { return; @@ -10311,7 +10283,11 @@ export default function CanvasEditorLegacyPage({ return (
+ {/* 1. Canvas Layer (Full Screen) - 使用 key 强制在 id 变化时重新挂载 */}
) : null} - {brandEntry} + {isVideoMode ? brandEntry : ( + { + setIsPanelOpen(true); + setPanelMode("fileList"); + }} + onSave={() => void handleSave()} + projectTitle={projectTitle} + rightInset={panelOcclusionRight} + /> + )} {agentDisabledNotice} {/* 2. Floating Actions (Top Right) */} {!isVideoMode && !isPanelOpen && (
) : null} -
+
{!areSecondaryPanelsReady ? ( ) : panelMode === "chat" ? ( @@ -11111,6 +11103,7 @@ export default function CanvasEditorLegacyPage({ onClose={() => setIsPanelOpen(false)} onOpenFileList={() => setPanelMode("fileList")} onRequireLogin={openLoginDialog} + onBeforeAgentRequest={persistCanvasBeforeAgent} selectedAgentPresetId={ isCanvasAgentModeEnabled ? selectedAgentPresetId : null } @@ -11200,11 +11193,11 @@ export default function CanvasEditorLegacyPage({ className={cn( isVideoMode ? VIDEO_CANVAS_TOOLBAR_CLASS - : "border bg-white shadow-2xl dark:bg-zinc-950", + : "border border-[#e9e9e9] bg-white shadow-[0_1px_3px_rgba(0,0,0,0.035)] dark:border-white/10 dark:bg-zinc-950", !isVideoMode && (shouldDockFloatingToolbarToSide ? "flex flex-col items-center gap-1 rounded-[24px] px-1.5 py-2.5" - : "flex items-center gap-1 rounded-[24px] px-2.5 py-1.5") + : "flex items-center gap-1 rounded-[14px] px-1.5 py-1.5") )} style={ shouldDockFloatingToolbarToSide @@ -11222,7 +11215,7 @@ export default function CanvasEditorLegacyPage({ tool === "select" && (isVideoMode ? VIDEO_CANVAS_TOOLBAR_ACTIVE_BUTTON_CLASS - : "bg-primary text-primary-foreground hover:bg-primary/90") + : "bg-[#ededed] text-[#333333] hover:bg-[#e5e5e5] dark:bg-white/12 dark:text-white") )} onClick={() => setTool("select")} size="icon" @@ -11232,6 +11225,8 @@ export default function CanvasEditorLegacyPage({ active={tool === "select"} className="h-4 w-4" fallback={MousePointer} + referenceName={isVideoMode ? undefined : "cursor"} + referenceSize={20} item={getCanvasToolbarItem( canvasToolbarConfig, "selectTool" @@ -11245,63 +11240,6 @@ export default function CanvasEditorLegacyPage({ - {!isVideoMode && - isCanvasToolbarItemEnabled( - canvasToolbarConfig, - "addAgentImageSlot" - ) ? ( - - - - - - - - - { - getCanvasToolbarItem( - canvasToolbarConfig, - "addAgentImageSlot" - ).label - } - - - - handleAddAgentImageSlotNode()} - > - - 图片槽位 - - handleAddAgentSlotNode("video")} - > - - - handleAddAgentSlotNode("ui-screen")} - > - - UI 槽位 - - - - ) : null} - @@ -11442,7 +11382,7 @@ export default function CanvasEditorLegacyPage({ tool === "draw" && (isVideoMode ? VIDEO_CANVAS_TOOLBAR_ACTIVE_BUTTON_CLASS - : "bg-primary text-primary-foreground hover:bg-primary/90") + : "bg-[#ededed] text-[#333333] hover:bg-[#e5e5e5] dark:bg-white/12 dark:text-white") )} onClick={() => setTool("draw")} size="icon" @@ -11483,7 +11423,7 @@ export default function CanvasEditorLegacyPage({ isAgentMode && (isVideoMode ? VIDEO_CANVAS_TOOLBAR_ACTIVE_BUTTON_CLASS - : "bg-primary text-primary-foreground hover:bg-primary/90") + : "bg-[#ededed] text-[#333333] hover:bg-[#e5e5e5] dark:bg-white/12 dark:text-white") )} onClick={handleStyleModeToolbarClick} size="icon" @@ -11499,6 +11439,14 @@ export default function CanvasEditorLegacyPage({ fallback={ shouldUseStyleModeButtonForStylePro ? Sparkles : Wand2 } + referenceName={ + isVideoMode + ? undefined + : shouldUseStyleModeButtonForStylePro + ? "style-controls" + : "creativity" + } + referenceSize={20} item={getCanvasToolbarItem( canvasToolbarConfig, "styleMode" @@ -11528,6 +11476,8 @@ export default function CanvasEditorLegacyPage({ @@ -11966,6 +11928,8 @@ export default function CanvasEditorLegacyPage({ - {/* 6. Bottom Left Compact Controls */} + {/* Compact zoom and history controls. */}
(); + return candidates + .map((value) => String(value || "").trim()) + .filter((value) => { + if (!value || seen.has(value)) return false; + seen.add(value); + return true; + }); +} + +function isPersistableTaskPreviewUrl(value?: string | null) { + return /^https?:\/\//i.test(String(value || "").trim()); +} + +function TaskAssetPreview({ task }: { task: TaskRecord }) { + const candidates = getTaskPreviewCandidates(task); + const signature = candidates.join("|"); + const [candidateIndex, setCandidateIndex] = useState(0); + + useEffect(() => { + setCandidateIndex(0); + }, [signature]); + + const previewUrl = candidates[candidateIndex] || null; + if (!previewUrl) { + return ( +
+ {task.status === "queued" || task.status === "running" ? ( + + ) : ( + + )} +
+ ); + } + + return ( + 3D 资产预览 setCandidateIndex((index) => index + 1)} + src={previewUrl} + /> + ); +} + function canImportTaskToCanvas(task?: TaskRecord | null) { if (!task || task.status !== "success") return false; if (task.provider !== "worldlabs-3d") return Boolean(resolveTaskModelUrl(task)); @@ -850,9 +906,21 @@ export default function CanvasModel3DPage() { ); const pollAttemptsRef = useRef(0); + const pollErrorCountRef = useRef(0); const multiViewSlotsRef = useRef([]); const batchImageSlotsRef = useRef([]); + const MAX_RETRYABLE_TASK_POLL_FAILURES = 12; + + const isRetryableTaskPollFailure = (message: string) => { + const normalized = String(message || "").toLowerCase(); + return ( + /http[^0-9]*5\d\d/.test(normalized) || + normalized.includes("超时") || + normalized.includes("timeout") + ); + }; + // Fetch site settings for Logo const { data: siteSettingsData } = useSWR( "/api/site-settings?scope=branding", @@ -1114,7 +1182,9 @@ export default function CanvasModel3DPage() { modelVersion: task.modelVersion, progress: task.progress ?? null, modelUrl: task.modelUrl ?? null, - previewImageUrl: task.previewImageUrl ?? null, + previewImageUrl: isPersistableTaskPreviewUrl(task.previewImageUrl) + ? task.previewImageUrl + : null, error: task.error ?? null, sourceTaskId: task.sourceTaskId ?? null, exportFormat: task.exportFormat ?? null, @@ -1434,6 +1504,33 @@ export default function CanvasModel3DPage() { }); if (!result.success || !result.data) { const failMessage = result.error || "任务查询失败"; + + if (isRetryableTaskPollFailure(failMessage)) { + pollErrorCountRef.current += 1; + if (pollErrorCountRef.current === 1) { + console.warn("[Model3DPage] 任务查询临时异常，继续重试", { + taskId: currentTask.taskId, + provider: currentTask.provider, + error: failMessage, + attempt: pollAttemptsRef.current, + }); + toast.info(`任务查询异常，正在重试...`); + } + + if (pollErrorCountRef.current >= MAX_RETRYABLE_TASK_POLL_FAILURES) { + const failedRecord: TaskRecord = { + ...currentTask, + status: "failed", + error: `${failMessage}（重试仍异常）`, + }; + setCurrentTask(failedRecord); + upsertTaskHistory(failedRecord); + } + + return; + } + + pollErrorCountRef.current = 0; const failedRecord: TaskRecord = { ...currentTask, status: "failed", @@ -1479,6 +1576,8 @@ export default function CanvasModel3DPage() { error: status === "failed" ? result.data.errorMessage || "任务失败" : null, }; + + pollErrorCountRef.current = 0; if (taskRecordChanged(currentTask, nextRecord)) { setCurrentTask(nextRecord); upsertTaskHistory(nextRecord); @@ -1652,6 +1751,7 @@ export default function CanvasModel3DPage() { sourceTaskId?: string | null; exportFormat?: string | null; assetPayload?: Model3DTaskAssetPayload | null; + previewImageUrl?: string | null; }) => { const normalizedPayload = params.provider === "tripo-3d" && @@ -1706,6 +1806,7 @@ export default function CanvasModel3DPage() { createdAt: Date.now(), sourceTaskId: params.sourceTaskId || null, exportFormat: params.exportFormat || null, + previewImageUrl: params.previewImageUrl || null, assetPayload: params.provider === "worldlabs-3d" ? worldCreateResultData?.assetPayload ?? params.assetPayload ?? null @@ -1821,6 +1922,7 @@ export default function CanvasModel3DPage() { providerId: uploadResult.data.providerId || null, providerName: uploadResult.data.providerName || null, }, + previewImageUrl: referencePreviewUrl, }); toast.success("已提交图生3D任务"); return; @@ -1855,6 +1957,8 @@ export default function CanvasModel3DPage() { providerId: uploadResult.data.providerId || null, providerName: uploadResult.data.providerName || null, }, + previewImageUrl: + batchImageSlots.find((slot) => slot.file === file)?.previewUrl || null, }); successCount += 1; } @@ -1910,6 +2014,7 @@ export default function CanvasModel3DPage() { providerId: boundTripoProviderId, providerName: boundTripoProviderName, }, + previewImageUrl: multiViewSlots.find((slot) => slot.previewUrl)?.previewUrl || null, }); toast.success("已提交多视图3D任务"); } catch (error) { @@ -1992,6 +2097,7 @@ export default function CanvasModel3DPage() { mode: "world-image", promptText: normalizedPrompt || "图生世界", modelLabel: worldModel, + previewImageUrl: sourceUrl || worldImagePreviewUrl, }); toast.success("已提交图生世界任务"); return; @@ -2076,6 +2182,10 @@ export default function CanvasModel3DPage() { mode: "world-multi-image", promptText: normalizedPrompt || "多图世界", modelLabel: worldModel, + previewImageUrl: + mediaAssets.find((item) => "sourceUrl" in item)?.sourceUrl || + selectedWorldImages.find((slot) => slot.previewUrl)?.previewUrl || + null, }); toast.success("已提交多图世界任务"); } catch (error) { @@ -4177,8 +4287,6 @@ export default function CanvasModel3DPage() { const canImportWorld = task.provider === "worldlabs-3d" && canImportTaskToCanvas(task); const isSelectedForCanvas = selectedWorldCanvasTaskIds.includes(task.taskId); - const previewUrl = resolveWorldTaskPreviewUrl(task) || task.previewImageUrl; - return (
handleSelectTask(task)} type="button" > - {previewUrl ? ( - preview - ) : ( -
- -
- )} +
diff --git a/app/dev/flow-board-preview/page.tsx b/app/dev/flow-board-preview/page.tsx new file mode 100644 index 000000000..a0e419f7d --- /dev/null +++ b/app/dev/flow-board-preview/page.tsx @@ -0,0 +1,17 @@ +import { notFound } from "next/navigation"; +import { FlowBoardClient } from "@/components/flow-board/flow-board-client"; + +export default function FlowBoardPreviewPage() { + if (process.env.NODE_ENV === "production") { + notFound(); + } + + return ( + + ); +} diff --git a/app/globals.css b/app/globals.css index 9291d28d4..accaa3229 100644 --- a/app/globals.css +++ b/app/globals.css @@ -3,6 +3,9 @@ /* include utility classes in streamdown */ @source '../node_modules/streamdown/dist/index.js'; +@source '../components/canvas/**/*.tsx'; +@source not '../.next*/**'; +@source not '../.qa/**'; @source not '../services/pascal-editor'; /* custom variant for setting dark mode programmatically */ @@ -405,6 +408,15 @@ } /* Dense, shared system-settings language. Scoped so canvas tools keep their own scale. */ +.video-canvas-workspace { + /* Match the compact typography used by the reference canvas. */ + --canvas-font-caption: 12px; + --canvas-font-body: 13px; + --canvas-font-heading: 14px; + font-size: var(--canvas-font-body); + line-height: 1.45; +} + .settings-workspace { font-size: 13px; line-height: 1.45; @@ -507,3 +519,301 @@ padding: 0 12px 12px; } } + +/* Lovart surface typography and controls, scoped to the home and image canvas. */ +.lovart-home-surface, +.lovart-canvas-surface { + font-family: Arial, "PingFang SC", "Hiragino Sans GB", "Microsoft YaHei", sans-serif; + -webkit-font-smoothing: antialiased; +} + +.lovart-suggestion { + display: inline-flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + gap: 8px; + height: 32px; + padding: 0 14px; + border: 1px solid #e5e5e5; + border-radius: 999px; + background: white; + color: #333; + font-size: 13px; + white-space: nowrap; + transition: background-color 150ms, border-color 150ms; +} +.lovart-suggestion:hover { background: #f7f7f7; border-color: #d5d5d5; } +.lovart-suggestion:focus-visible, +.lovart-start-action:focus-visible { outline: 2px solid #8c8c8c; outline-offset: 3px; } + +.lovart-start-action { + display: inline-flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + gap: 9px; + min-width: 172px; + height: 40px; + padding: 0 16px; + border: 1px solid #e5e5e5; + border-radius: 999px; + background: white; + color: #333; + font-size: 14px; + font-weight: 400; + box-shadow: 0 1px 2px rgb(0 0 0 / 4%); + transition: background-color 150ms, border-color 150ms; +} +.lovart-start-action:hover { background: #f7f7f7; border-color: #d4d4d4; } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-chat-header { border-bottom: 0; } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-chat-composer { padding: 0 8px 8px; } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-chat-composer > [data-canvas-chat-input-anchor] { + min-height: 120px; + border-color: #e5e5e5; + box-shadow: 0 1px 3px rgb(0 0 0 / 5%); +} +/* The promo backplate continues underneath the composer's rounded corners. */ +.lovart-chat-promo-connected { padding: 0 8px; } +.lovart-chat-promo-surface { padding-bottom: 24px; border-radius: 24px 24px 0 0; background: #ebebeb; } +.lovart-chat-promo-row { display: flex; align-items: center; gap: 8px; height: 36px; padding: 0 10px 0 16px; } +.lovart-chat-promo-message { display: flex; align-items: center; gap: 8px; min-width: 0; flex: 1; padding: 0; border: 0; background: transparent; color: #202020; font-size: 14px; font-weight: 400; line-height: 20px; text-align: left; cursor: pointer; } +.lovart-chat-promo-message:disabled { cursor: default; opacity: 1; } +.lovart-chat-promo-message > span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.lovart-chat-promo-badge { flex-shrink: 0; } +.lovart-chat-promo-close { display: flex; align-items: center; justify-content: center; flex-shrink: 0; width: 28px; height: 28px; border: 0; border-radius: 50%; background: transparent; color: #262626; cursor: pointer; } +.lovart-chat-promo-close:hover { background: rgb(0 0 0 / 5%); } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-chat-promo-connected + .lovart-chat-composer { position: relative; z-index: 1; margin-top: -24px; background: transparent; } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-chat-composer > [data-canvas-chat-input-anchor][data-canvas-composer-mode="image"] { min-height: 184px; border-radius: 24px; } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-reference-upload { margin: 8px 8px 0; } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-reference-tile { flex-direction: column; gap: 8px; width: 56px; height: 68px; border: 0; border-radius: 18px; background: #f5f5f5; color: #aaa; font-size: 12px; font-weight: 400; } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-reference-tile:hover { background: #ececec; color: #777; } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-reference-tile > svg { width: 14px; height: 14px; stroke-width: 1.5; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-composer-mode="image"] .lovart-composer-text-region { padding: 12px 16px 0; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-composer-mode="image"] [role="textbox"] { min-height: 40px; font-size: 14px; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-composer-mode="image"] [role="textbox"]:empty::before { color: #b0b0b0; } +.dark .lovart-chat-promo-surface { background: #27272a; } +.dark .lovart-chat-promo-message, .dark .lovart-chat-promo-close { color: #eee; } +.dark .lovart-chat-promo-close:hover { background: rgb(255 255 255 / 10%); } +.dark .lovart-canvas-surface [data-lovart-chat="true"] .lovart-reference-tile { background: #27272a; color: #a1a1aa; } +.dark .lovart-canvas-surface [data-lovart-chat="true"] .lovart-reference-tile:hover { background: #3f3f46; color: #eee; } +.dark .lovart-suggestion, +.dark .lovart-start-action { border-color: rgb(255 255 255 / 12%); background: #18181b; color: #eee; } +.dark .lovart-suggestion:hover, +.dark .lovart-start-action:hover { background: #27272a; border-color: rgb(255 255 255 / 22%); } +.dark .lovart-canvas-surface [data-lovart-chat="true"] .lovart-chat-header { border-color: rgb(255 255 255 / 10%); } +.dark [data-canvas-right-panel="true"] > .border-l { border-left-color: #27272a; } +.dark .lovart-canvas-surface [data-lovart-chat="true"] .lovart-chat-composer > [data-canvas-chat-input-anchor] { border-color: rgb(255 255 255 / 12%); } +@media (max-width: 640px) { + .lovart-home-surface main { padding-top: calc(72px + var(--home-banner-offset, 0px)); } + .lovart-hero { margin-top: 16px; } + .lovart-home-composer { border-radius: 20px; } + .lovart-suggestion { padding: 0 11px; font-size: 12px; } + .lovart-start-action { min-width: 160px; } + .lovart-chat-welcome h1 { font-size: 15px; } +} +@media (prefers-reduced-motion: reduce) { + .lovart-home-surface *, .lovart-canvas-surface * { scroll-behavior: auto !important; } +} + +.lovart-home-surface :is(button, input, textarea, select), +.lovart-canvas-surface :is(button, input, textarea, select) { font-family: inherit; } +@media (max-width: 640px) { + .lovart-canvas-surface[data-canvas-panel-open="true"] :is([data-canvas-topbar], [data-canvas-floating-toolbar], [data-canvas-compact-controls]) { display: none; } +} + +/* Reference icon sizing is independent of the application's root rem scale. */ +.lovart-suggestion { gap: 8px; padding: 0 12px; } +.lovart-start-action { min-width: 172.71875px; gap: 8px; padding: 0 12px; } +.lovart-canvas-surface [data-canvas-floating-toolbar] button { min-width: 32px; height: 32px; border-radius: 8px; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-composer-mode="image"] .lovart-composer-mode-selector { width: 90px; height: 32px; padding: 0 8px; background: transparent; font-size: 14px; font-weight: 400; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-composer-mode="image"] .lovart-composer-mode-selector > span { font-size: 13px; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-composer-mode="image"] .lovart-composer-mode-selector:hover, +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-image-params:hover, +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-model-selector:hover { background: rgb(0 0 0 / 4%); } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-image-params { width: 105.3828125px; min-width: 105.3828125px; height: 32px; padding: 0 4px 0 12px; background: transparent; font-size: 13px; gap: 4px; } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-model-selector { flex: 0 0 121px; width: 121px; height: 32px; padding: 0 4px 0 8px; gap: 0; border: 0; background: transparent; box-shadow: none; font-size: 13px; font-weight: 400; } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-model-selector > [data-reference-icon]:first-child { margin-right: 8px; } +.lovart-canvas-surface [data-lovart-chat="true"] .lovart-model-selector > span:not([data-reference-icon]) { max-width: 65px; } +.dark .lovart-canvas-surface [data-lovart-chat="true"] :is(.lovart-model-selector, .lovart-image-params) { color: #eee; } +.dark .lovart-canvas-surface [data-lovart-chat="true"] :is(.lovart-model-selector, .lovart-image-params, .lovart-composer-mode-selector):hover { background: rgb(255 255 255 / 8%); } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-composer-mode="image"] .lovart-composer-footer { padding: 0 8px 8px; gap: 0; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-composer-mode="image"] .lovart-composer-footer > div:first-child { flex: 0 0 90px; padding-right: 0; overflow: visible; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-composer-mode="image"] .lovart-composer-footer > div:last-child { flex: 1 1 0; min-width: 0; gap: 0; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-composer-mode="image"] .lovart-composer-send { min-width: 32px; width: auto; height: 32px; padding: 0 8px; gap: 2px; font-size: 13px; font-weight: 400; } + +/* Adapt to the resizable chat panel, independently of the browser width. */ +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-responsive-composer="true"] { container: canvas-composer / inline-size; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-responsive-composer="true"] .lovart-composer-footer { gap: 4px; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-responsive-composer="true"] .lovart-composer-footer > div:first-child { flex: 0 0 auto; overflow: visible; padding-right: 0; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-responsive-composer="true"] .lovart-composer-footer > div:last-child { flex: 0 1 auto; min-width: 0; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-responsive-composer="true"] .lovart-composer-mode-selector { width: auto; max-width: none; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-responsive-composer="true"] .lovart-image-params { width: auto; min-width: 0; } +.lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-responsive-composer="true"] .lovart-image-params-label { overflow: visible; text-overflow: clip; white-space: nowrap; } +.lovart-image-params-icon { display: none; } +@container canvas-composer (max-width: 480px) { + .lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-responsive-composer="true"] .lovart-composer-mode-selector { width: 32px; min-width: 32px; padding: 0; } + .lovart-composer-mode-selector .lovart-composer-mode-label, + .lovart-composer-mode-selector [data-reference-icon="chevron-down"] { display: none; } + .lovart-composer-mode-selector > span > :first-child { margin-right: 0; } +} +@container canvas-composer (max-width: 360px) { + .lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-responsive-composer="true"] .lovart-model-selector { flex: 0 0 32px; width: 32px; padding: 0; justify-content: center; } + .lovart-model-selector > span:not([data-reference-icon]), + .lovart-model-selector > [data-reference-icon="chevron-down"] { display: none; } + .lovart-canvas-surface [data-lovart-chat="true"] .lovart-model-selector > [data-reference-icon]:first-child { margin-right: 0; } +} +@container canvas-composer (max-width: 480px) { + .lovart-canvas-surface [data-lovart-chat="true"] [data-canvas-responsive-composer="true"] .lovart-image-params { width: 32px; min-width: 32px; padding: 0; justify-content: center; } + .lovart-image-params .lovart-image-params-icon { display: block; } + .lovart-image-params .lovart-image-params-label, + .lovart-image-params > [data-reference-icon="chevron-down"] { display: none; } +} + +/* The existing image toolbar is scaled to 80%; these dimensions render at + the reference's 44px bar, 32px buttons, and 20px icons. */ +.lovart-image-action-bar { + height: 55px; + gap: 5px; + padding: 0 7.5px; + border: 1.25px solid rgba(0, 0, 0, 0.07); + border-radius: 17.5px; + background: #fff !important; + color: #000; + box-shadow: 0 1.25px 2.5px rgba(0, 0, 0, 0.04); + backdrop-filter: none; +} +.lovart-image-action-bar::before { display: none; } +.lovart-image-action-bar > button { + width: 40px; + min-width: 40px; + height: 40px; + margin: 0; + gap: 0; + padding: 0; + border-radius: 10px; + background: transparent; + color: #000; + font-size: 0; + font-weight: 400; + box-shadow: none; +} +.lovart-image-action-bar > button:hover, +.lovart-image-action-bar > button[data-state="open"], +.lovart-image-action-bar > button[data-reference-active="true"] { + background: rgba(0, 0, 0, 0.04); + color: #000; +} +.lovart-image-action-bar > button:disabled { + background: transparent; + color: rgba(0, 0, 0, 0.3); +} +.lovart-image-action-bar > button > svg, +.lovart-image-action-bar > button > [data-reference-icon], +.lovart-image-action-bar > button > span[aria-hidden="true"], +.lovart-image-action-bar > button > img { + width: 25px; + height: 25px; + margin: 0; + flex-shrink: 0; +} +.lovart-image-action-bar > button > img { filter: brightness(0); } +.lovart-image-action-bar > div.h-px, +.lovart-image-action-bar > div.w-px { + width: 1.25px; + height: 55px; + margin: 0 2.5px; + background: rgba(0, 0, 0, 0.07); +} +.lovart-image-action-menu [role="menuitem"] > svg, +.lovart-image-action-menu [role="menuitem"] > [data-reference-icon] { + width: 18px; + height: 18px; +} +.dark .lovart-image-action-bar { + border-color: rgba(255, 255, 255, 0.1); + background: #18191c !important; + color: #fff; +} +.dark .lovart-image-action-bar > button { color: #fff; } +.dark .lovart-image-action-bar > button:hover, +.dark .lovart-image-action-bar > button[data-state="open"], +.dark .lovart-image-action-bar > button[data-reference-active="true"] { + background: rgba(255, 255, 255, 0.08); + color: #fff; +} +.dark .lovart-image-action-bar > button:disabled { color: rgba(255, 255, 255, 0.3); } +.dark .lovart-image-action-bar > button > img { filter: brightness(0) invert(1); } +.dark .lovart-image-action-bar > div.w-px { background: rgba(255, 255, 255, 0.1); } + +.lovart-image-action-bar[data-show-tool-names="true"] > button[data-tool-key] { width: auto; gap: 7.5px; padding: 0 12.5px; font-size: 16.25px; } +.lovart-image-tool-label { white-space: nowrap; } +.lovart-image-tool-tooltip { z-index: 10030; border: 0; border-radius: 6px; background: #1b1b1b; padding: 6px 9px; color: #fff; font-size: 13px; line-height: 18px; box-shadow: 0 4px 10px #00000012; --tw-ring-shadow: 0 0 transparent; } +.lovart-image-custom-more { min-width: 172px; padding: 6px 12px; border: 1px solid #e8e8e8; border-radius: 12px; background: #fff; } +.lovart-image-custom-more [role="menuitem"] { min-height: 32px; gap: 10px; padding: 4px 0; font-size: 14px; font-weight: 400; } +.lovart-image-custom-more .lovart-customize-entry { justify-content: center; gap: 5px; } +.lovart-toolbar-customizer-overlay { z-index: 10010; background: #0006; } +.lovart-toolbar-customizer { z-index: 10020; display: flex; flex-direction: column; width: min(720px, calc(100vw - 32px)); max-width: 720px; max-height: calc(100dvh - 48px); gap: 0; padding: 20px 24px 24px; border: 1px solid #e5e5e5; border-radius: 16px; background: #fff; color: #242424; box-shadow: 0 8px 24px #00000012; } +.lovart-customizer-header { display: flex; flex-shrink: 0; justify-content: space-between; align-items: flex-start; gap: 16px; } +.lovart-customizer-header h2 { margin: 0; font-size: 18px; line-height: 24px; font-weight: 600; } +.lovart-customizer-header p { margin-top: 2px; color: inherit; font-size: 14px; line-height: 22px; } +.lovart-customizer-header-actions { display: flex; gap: 4px; } +.lovart-customizer-header-actions button { display: flex; width: 28px; height: 28px; align-items: center; justify-content: center; border-radius: 7px; } +.lovart-customizer-header-actions button:last-child { background: #f1f1f1; } +.lovart-customizer-header-actions button:hover { background: #e9e9e9; } +.lovart-customizer-body { min-height: 0; overflow-y: auto; margin: 24px 0 20px; } +.lovart-customizer-preview { display: flex; height: 180px; align-items: center; overflow: hidden; padding: 16px; border-radius: 10px; background: #e5edef url("/icons/lovart/toolbar-preview.png") center / cover; } +.lovart-customizer-preview-bar { display: flex; align-items: center; flex: 1; min-width: 0; gap: 4px; padding: 5px 8px; border: 1px solid #e5e5e5; border-radius: 14px; background: #fff; overflow-x: auto; scrollbar-width: thin; } +.lovart-preview-fixed,.lovart-preview-chip { display: flex; align-items: center; flex-shrink: 0; height: 32px; gap: 8px; padding: 0 10px; font-size: 13px; white-space: nowrap; } +.lovart-preview-chip { position: relative; margin-top: 5px; margin-bottom: 5px; border-radius: 8px; background: #f4f4f4; cursor: grab; } +.lovart-preview-chip:active { cursor: grabbing; } +.lovart-preview-chip > button:last-child { position: absolute; top: -5px; right: -3px; display: flex; width: 16px; height: 16px; align-items: center; justify-content: center; border: 1px solid #e6e6e6; border-radius: 50%; background: #fff; color: #888; } +.lovart-customizer-tools { display: grid; grid-template-columns: repeat(3,minmax(0,1fr)); gap: 8px; margin-top: 24px; } +.lovart-customizer-tools > button { display: flex; align-items: center; gap: 10px; height: 36px; min-width: 0; padding: 0 12px; border: 1px solid #e2e2e2; border-radius: 8px; background: #fff; font-size: 14px; text-align: left; } +.lovart-customizer-tools > button > span:not([aria-hidden]) { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.lovart-customizer-tools > button > svg:last-child { flex-shrink: 0; } +.lovart-customizer-tools > button:hover { background: #f7f7f7; } +.lovart-customizer-footer { display: flex; flex-shrink: 0; align-items: center; justify-content: space-between; gap: 16px; } +.lovart-customizer-footer > label,.lovart-customizer-footer > div { display: flex; align-items: center; gap: 8px; font-size: 14px; } +.lovart-customizer-switch { width: 32px; height: 18px; padding: 2px; border-radius: 20px; background: #d5d5d5; } +.lovart-customizer-switch > span { display: block; width: 14px; height: 14px; border-radius: 50%; background: #fff; transition: transform .15s; } +.lovart-customizer-switch[aria-checked="true"] { background: #2c2c2c; } +.lovart-customizer-switch[aria-checked="true"] > span { transform: translateX(14px); } +.lovart-customizer-footer > div > button { width: 80px; height: 36px; border: 1px solid #e3e3e3; border-radius: 8px; font-size: 14px; } +.lovart-customizer-footer .lovart-customizer-save { border-color: #2c2c2c; background: #2c2c2c; color: #fff; } +.lovart-customizer-footer > div > button:hover { opacity: .8; } +.lovart-toolbar-customizer button:focus-visible,.lovart-preview-chip:focus-visible { outline: 2px solid #397dff; outline-offset: 2px; } +.dark .lovart-toolbar-customizer,.dark .lovart-image-custom-more { background: #202124; border-color: #3d3e40; color: #f5f5f5; } +.dark .lovart-customizer-preview-bar,.dark .lovart-customizer-tools > button { background: #26272a; border-color: #46474a; } +.dark .lovart-preview-chip,.dark .lovart-customizer-header-actions button:last-child { background: #35363a; } +.dark .lovart-preview-chip > button:last-child { background: #26272a; border-color: #46474a; color: #ccc; } +.dark .lovart-customizer-tools > button:hover { background: #35363a; } +@media(max-width:600px) { .lovart-toolbar-customizer { padding: 18px; } .lovart-customizer-tools { grid-template-columns: repeat(2,minmax(0,1fr)); } .lovart-customizer-preview { height: 140px; } } + +.lovart-preview-handle { display: flex; align-items: center; gap: 8px; white-space: nowrap; cursor: grab; } + +.lovart-image-tool-tooltip { display: flex; align-items: center; gap: 8px; } +.lovart-image-tool-cost { margin-left: auto; padding: 0; background: transparent; color: #999; font-size: 12px; font-weight: 400; box-shadow: none; } +.lovart-image-tool-cost > svg { width: 12px; height: 12px; } + +/* Preserve the canvas's own skills as the primary welcome content. */ +.canvas-welcome-action { + display: inline-flex; + min-height: 36px; + min-width: 0; + align-items: center; + justify-content: center; + gap: 6px; + padding: 6px 8px; + border: 1px solid #e5e5e5; + border-radius: 999px; + background: white; + color: #333; + font-size: 12px; + line-height: 16px; + transition: background-color 150ms, border-color 150ms; +} +.canvas-welcome-action:hover { background: #f7f7f7; border-color: #d4d4d4; } +.canvas-welcome-action:focus-visible { outline: 2px solid #8c8c8c; outline-offset: 3px; } +.dark .canvas-welcome-action { border-color: rgb(255 255 255 / 12%); background: #18181b; color: #eee; } +.dark .canvas-welcome-action:hover { background: #27272a; border-color: rgb(255 255 255 / 22%); } diff --git a/app/layout.tsx b/app/layout.tsx index fcee64a6a..aca22ee6f 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -94,18 +94,33 @@ const CHUNK_ERROR_RECOVERY_SCRIPT = `\ } } - function safeRemoveSessionStorage(key) { - try { - sessionStorage.removeItem(key); - return true; - } catch (e) { - return false; - } - } - - var RECOVERY_KEY = 'chunk-reload-attempted-v2'; + // Keep one attempt per route for the browser session. A cooldown alone can + // turn a recurring editor exception into an endless reload cycle. + var RECOVERY_KEY = 'chunk-reload-attempted-v3:' + window.location.pathname; var lastReloadAt = Number(safeGetSessionStorage(RECOVERY_KEY) || '0') || 0; - + var recoveryNoticeShown = false; + + function isFlowBoardEditing() { + var path = window.location.pathname; + return path === '/flow-board' || path.indexOf('/flow-board/') === 0; + } + + function showManualRecoveryNotice() { + if (recoveryNoticeShown) return; + recoveryNoticeShown = true; + function show() { + if (!document.body || !isFlowBoardEditing()) return; + var notice = document.createElement('div'); + notice.setAttribute('role', 'alert'); + notice.setAttribute('data-runtime-recovery-notice', 'true'); + notice.style.cssText = 'position:fixed;bottom:24px;left:50%;transform:translateX(-50%);z-index:2147483647;max-width:calc(100vw - 32px);padding:12px 16px;border:1px solid var(--border,#e4e4e7);border-radius:12px;background:var(--background,#fff);color:var(--foreground,#27272a);box-shadow:0 4px 20px #0002;font:14px/1.5 system-ui;'; + notice.textContent = '部分页面资源加载失败，请先保存白板后手动刷新页面。'; + document.body.appendChild(notice); + } + if (document.body) show(); + else window.addEventListener('DOMContentLoaded', show, { once: true }); + } + function shouldAutoRecover(message) { if (!message) return false; var normalized = String(message).toLowerCase(); @@ -117,28 +132,33 @@ const CHUNK_ERROR_RECOVERY_SCRIPT = `\ normalized.indexOf('error loading dynamically imported module') !== -1 || normalized.indexOf('importing a module script failed') !== -1 || normalized.indexOf('module script load failed') !== -1 || - normalized.indexOf('/_next/static/chunks/') !== -1 || - normalized.indexOf('failed to find server action') !== -1 || - normalized.indexOf('server components render') !== -1 || - normalized.indexOf('application error: a client-side exception') !== -1 + normalized.indexOf('failed to find server action') !== -1 ); } function triggerAutoRecover(reason) { + if (isFlowBoardEditing()) { + showManualRecoveryNotice(); + return; + } + if (lastReloadAt) return; var now = Date.now(); - if (lastReloadAt && now - lastReloadAt < 30000) return; + // If storage is blocked, a reload cannot remember its attempt and must not run. + if (!safeSetSessionStorage(RECOVERY_KEY, String(now))) return; lastReloadAt = now; - console.warn('[RuntimeRecover] 检测到前端运行时异常，自动刷新页面...', reason || ''); - safeSetSessionStorage(RECOVERY_KEY, String(now)); + console.warn('[RuntimeRecover] 检测到页面资源加载失败，尝试恢复一次...', reason || ''); setTimeout(function() { + if (isFlowBoardEditing()) { + showManualRecoveryNotice(); + return; + } window.location.reload(); }, 800); } - + window.addEventListener('error', function(event) { var message = ''; if (event && event.message) message += event.message; - if (event && event.filename) message += ' ' + event.filename; if (event && event.error && event.error.message) { message += ' ' + event.error.message; } @@ -157,12 +177,6 @@ const CHUNK_ERROR_RECOVERY_SCRIPT = `\ } }); - // 页面稳定一段时间后清除标记，避免旧资源异常无限刷新。 - window.addEventListener('load', function() { - setTimeout(function() { - safeRemoveSessionStorage(RECOVERY_KEY); - }, 120000); - }); })();`; export default function RootLayout({ diff --git a/app/tvstar/plugin/page.tsx b/app/tvstar/plugin/page.tsx new file mode 100644 index 000000000..1a7c02eae --- /dev/null +++ b/app/tvstar/plugin/page.tsx @@ -0,0 +1,83 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { toast } from "sonner"; + +type KeyItem = { id: string; name: string; tokenPrefix: string; scopes: string[]; revokedAt: string | null; expiresAt: string | null }; + +export default function TVstarPluginPage() { + const [keys, setKeys] = useState([]); + const [createdToken, setCreatedToken] = useState(""); + const [busy, setBusy] = useState(false); + const [origin, setOrigin] = useState("https://ai.yydd.ai"); + useEffect(() => { setOrigin(window.location.origin); }, []); + const loadKeys = useCallback(async () => { + const response = await fetch("/api/user/access-keys", { cache: "no-store" }); + const data = await response.json(); + if (!response.ok || !data.success) throw new Error(data.error || "读取密钥失败"); + setKeys(Array.isArray(data.data) ? data.data : []); + }, []); + useEffect(() => { void loadKeys().catch(error => toast.error(error.message)); }, [loadKeys]); + + const createKey = async () => { + setBusy(true); + try { + const response = await fetch("/api/user/access-keys", { method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "TVstar MCP", scopes: ["canvas:read", "canvas:write"], expiresInDays: 90 }) }); + const data = await response.json(); + if (!response.ok || !data.success || !data.data?.token) throw new Error(data.error || "创建密钥失败"); + setCreatedToken(data.data.token); + await loadKeys(); + toast.success("密钥已创建；请立即复制，关闭后不会再次显示"); + } catch (error) { toast.error(error instanceof Error ? error.message : "创建密钥失败"); } + finally { setBusy(false); } + }; + const revokeKey = async (id: string) => { + if (!window.confirm("撤销后使用此密钥的 Agent 将失去画布访问权限，确定继续吗？")) return; + setBusy(true); + try { + const response = await fetch(`/api/user/access-keys/${id}`, { method: "DELETE" }); + const data = await response.json(); + if (!response.ok || !data.success) throw new Error(data.error || "撤销失败"); + setCreatedToken(""); + await loadKeys(); + toast.success("密钥已撤销"); + } catch (error) { toast.error(error instanceof Error ? error.message : "撤销失败"); } + finally { setBusy(false); } + }; + + const example = JSON.stringify({ mcpServers: { tvstar: { command: "node", args: ["/替换为解压目录/tvstar-mcp/mcp-server.mjs"], + env: { YEDIR_BASE_URL: origin, YEDIR_ACCESS_TOKEN: "粘贴新创建的 Access Key" } } } }, null, 2); + + return
+
+

TVstar Plugin

+

让 Agent 操作视频画布

+

当前插件提供画布列表、读取和有版本保护的节点操作。视频生成与合成仍由 TVstar 画布内的任务系统执行，MCP 暂未提供直接发起生成的工具。

+
+
+

1. 下载并安装

+

需要 Node.js 20 或更新版本。解压后在 tvstar-mcp 目录运行：

+ 下载 TVstar MCP 安装包 + 
cd tvstar-mcp{"\n"}npm ci
+
+
+

2. 创建画布 Access Key

+

读取和修改画布需要对应权限。密钥只在创建时显示一次。

+ + {createdToken ?

{createdToken}

: null} +
+
+
+

3. 配置支持 STDIO MCP 的 Agent

+

将安装包的绝对路径和密钥填入客户端配置。不同 Agent 的配置文件位置和外层字段可能不同，请以所用客户端的说明为准。

+ 
{example}
+ +
+
+

已创建的 Access Key

+
{keys.length ? keys.map(key =>
{key.name} · {key.tokenPrefix} · {key.revokedAt ? "已撤销" : "有效"}{!key.revokedAt ? : null}
) :

暂无密钥

}
+
+
+
; +} diff --git a/app/tvstar/share/[token]/page.tsx b/app/tvstar/share/[token]/page.tsx new file mode 100644 index 000000000..99dc1a280 --- /dev/null +++ b/app/tvstar/share/[token]/page.tsx @@ -0,0 +1,41 @@ +import { eq } from "drizzle-orm"; +import { notFound } from "next/navigation"; +import { db } from "@/lib/db"; +import { tvstarSession } from "@/lib/db/schema"; + +export const dynamic = "force-dynamic"; + +type SharedSnapshot = { + title: string; + goal: string; + createdAt: number; + messages: { role: "user" | "assistant"; content: string; createdAt: number }[]; + plan: { summary: string; deliverables: string[] } | null; + shots: { shot: string; duration: string; sceneDescription: string }[]; +}; + +export default async function TVstarSharedConversation({ params }: { params: Promise<{ token: string }> }) { + const { token } = await params; + if (!/^[A-Za-z0-9_-]{32}$/.test(token || "")) notFound(); + const row = (await db.select({ snapshot: tvstarSession.sharedSnapshot }).from(tvstarSession) + .where(eq(tvstarSession.shareToken, token)).limit(1))[0]; + if (!row?.snapshot) notFound(); + const shared = row.snapshot as SharedSnapshot; + return
+
+

TVstar · 只读分享

+

{shared.title}

+

{new Date(shared.createdAt).toLocaleString("zh-CN")}

+

{shared.goal}

+
+ {(shared.messages || []).map((message, index) =>
+

{message.role === "user" ? "用户" : "TVstar"}

+

{message.content}

+
)} +
+ {shared.plan ?

创作规划

{shared.plan.summary}

: null} + {shared.shots?.length ?

分镜

    {shared.shots.map((shot, index) =>
  1. {shot.shot} · {shot.duration}

    {shot.sceneDescription}

    2. )}
: null} +

此页面是分享时的对话快照，无法编辑画布；对话正文中的文字和链接会原样显示。

+
+
; +} diff --git a/components/agents/AgentAiTemplateDialog.tsx b/components/agents/AgentAiTemplateDialog.tsx new file mode 100644 index 000000000..649fe82c3 --- /dev/null +++ b/components/agents/AgentAiTemplateDialog.tsx @@ -0,0 +1,315 @@ +"use client"; + +import { useEffect, useId, useRef, useState } from "react"; +import { LayoutTemplate, Loader2, Upload } from "lucide-react"; +import { toast } from "sonner"; +import { AgentPreviewImage } from "@/components/agents/AgentPreviewImage"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Textarea } from "@/components/ui/textarea"; +import { fetchUploadWithRetry } from "@/lib/client/upload-with-retry"; +import { imageGenerationService } from "@/services/canvas"; + +const POLL_INTERVAL_MS = 2500; +const POLL_LIMIT = 240; + +export type AgentAiTemplateResult = { + imageUrl: string; + sourceImageUrl: string; + referenceImageUrl: string; +}; + +type AgentAiTemplateDialogProps = { + open: boolean; + sourceImageUrl?: string | null; + model?: string | null; + imageSize?: string | null; + projectId: string; + onOpenChange: (open: boolean) => void; + onProcessingChange?: (processing: boolean, sourceImageUrl: string) => void; + onResultCreated: (result: AgentAiTemplateResult) => void | Promise; +}; + +function isHttpImageUrl(value?: string | null) { + return /^https?:\/\//i.test(String(value || "").trim()); +} + +function sleep(ms: number) { + return new Promise((resolve) => window.setTimeout(resolve, ms)); +} + +async function pollAiTemplateTask(taskId: string) { + for (let attempt = 0; attempt < POLL_LIMIT; attempt += 1) { + await sleep(POLL_INTERVAL_MS); + const taskResult = await imageGenerationService.getImageTask(taskId); + const task = taskResult.data?.task; + if (!taskResult.success || !task) { + continue; + } + + const status = String(task.status || "").toLowerCase(); + if (status === "succeeded" || status === "completed") { + const image = task.result?.images?.[0]; + const imageUrl = String(image?.originalUrl || image?.thumbnailUrl || "").trim(); + if (!isHttpImageUrl(imageUrl)) { + throw new Error("AI套模版已完成但未返回结果图片"); + } + return imageUrl; + } + + if (["failed", "cancelled", "expired", "banned"].includes(status)) { + throw new Error(task.error || "AI套模版生成失败"); + } + } + + throw new Error("AI套模版任务等待超时，请稍后在结果区查看"); +} + +export function AgentAiTemplateDialog({ + open, + sourceImageUrl, + model, + imageSize, + projectId, + onOpenChange, + onProcessingChange, + onResultCreated, +}: AgentAiTemplateDialogProps) { + const referenceUploadId = useId(); + const [referenceImageUrl, setReferenceImageUrl] = useState(""); + const [supplementalPrompt, setSupplementalPrompt] = useState(""); + const [isUploading, setIsUploading] = useState(false); + const [isSubmitting, setIsSubmitting] = useState(false); + const normalizedSourceUrl = String(sourceImageUrl || "").trim(); + const onProcessingChangeRef = useRef(onProcessingChange); + const onResultCreatedRef = useRef(onResultCreated); + + useEffect(() => { + onProcessingChangeRef.current = onProcessingChange; + onResultCreatedRef.current = onResultCreated; + }, [onProcessingChange, onResultCreated]); + + useEffect(() => { + if (!open || isSubmitting) return; + setReferenceImageUrl(""); + setSupplementalPrompt(""); + setIsUploading(false); + }, [isSubmitting, open, normalizedSourceUrl]); + + const handleReferenceUpload = async (file?: File) => { + if (!file) return; + if (!file.type.startsWith("image/")) { + toast.error("请上传图片文件"); + return; + } + + setIsUploading(true); + try { + const formData = new FormData(); + formData.append("file", file); + const response = await fetchUploadWithRetry(formData); + const payload = await response.json().catch(() => null); + const uploadedUrl = String(payload?.url || "").trim(); + if (!response.ok || !isHttpImageUrl(uploadedUrl)) { + throw new Error(payload?.error || "参考图上传失败"); + } + setReferenceImageUrl(uploadedUrl); + toast.success("版式参考图已上传"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "参考图上传失败"); + } finally { + setIsUploading(false); + } + }; + + const handleSubmit = async () => { + const normalizedReferenceUrl = referenceImageUrl.trim(); + if (!isHttpImageUrl(normalizedSourceUrl)) { + toast.error("当前主图结果地址不可用"); + return; + } + if (!isHttpImageUrl(normalizedReferenceUrl)) { + toast.error("请先上传或填写版式参考图"); + return; + } + if (isSubmitting) return; + + setIsSubmitting(true); + onProcessingChangeRef.current?.(true, normalizedSourceUrl); + try { + const submitResult = await imageGenerationService.layoutReplace({ + sourceImageUrl: normalizedSourceUrl, + referenceImageUrl: normalizedReferenceUrl, + supplementalPrompt: supplementalPrompt.trim() || undefined, + model: model && model !== "auto" ? model : undefined, + imageSize: imageSize || "2K", + agentProjectId: projectId, + referenceLabel: "风格化主图 AI 套模版参考图", + debugMeta: { + operation: "agent-main-ai-template", + source: "agent-main-result-card", + }, + }); + const taskId = String(submitResult.data?.taskId || "").trim(); + if (!submitResult.success || !taskId) { + throw new Error(submitResult.error || "AI套模版提交失败"); + } + + onOpenChange(false); + toast.success("AI套模版已进入生图队列"); + const resultImageUrl = await pollAiTemplateTask(taskId); + await onResultCreatedRef.current({ + imageUrl: resultImageUrl, + sourceImageUrl: normalizedSourceUrl, + referenceImageUrl: normalizedReferenceUrl, + }); + toast.success("AI套模版结果已加入主图结果区"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "AI套模版生成失败"); + } finally { + setIsSubmitting(false); + onProcessingChangeRef.current?.(false, normalizedSourceUrl); + } + }; + + return ( + { + if (!isUploading) { + onOpenChange(nextOpen); + } + }} + > + + + + + AI套模版 + + + 保留参考图的版式、文字和图标，仅将产品主体替换为当前主图。 + + + +
+
+
+ +
+ {isHttpImageUrl(normalizedSourceUrl) ? ( + + ) : null} +
+
+ +
+ +
+ setReferenceImageUrl(event.currentTarget.value)} + placeholder="填写图片链接或上传" + value={referenceImageUrl} + /> + + { + const file = event.currentTarget.files?.[0]; + event.currentTarget.value = ""; + void handleReferenceUpload(file); + }} + type="file" + /> +
+
+ {isUploading ? ( + + ) : isHttpImageUrl(referenceImageUrl) ? ( + + ) : ( + + )} +
+
+
+ +
+ +