feat(agent): support dedicated coding model id and detail pipeline hardening
This commit is contained in:
parent
9c4e51d3fd
commit
ac76f31878
@ -158,6 +158,7 @@ type AiModel = {
|
||||
isDefault: boolean;
|
||||
sortOrder: number;
|
||||
isAgentModel?: boolean;
|
||||
agentCodingModelId?: string;
|
||||
isMultimodal?: boolean;
|
||||
imageGenConfig?: ImageGenConfig; // 🆕 图片生成特性配置
|
||||
};
|
||||
@ -1072,6 +1073,7 @@ export function AdminSettingsClient() {
|
||||
isHidden: false,
|
||||
isDefault: false,
|
||||
isAgentModel: false,
|
||||
agentCodingModelId: "",
|
||||
isMultimodal: false,
|
||||
sortOrder: 0,
|
||||
fallbackModelId1: "", // 备用模型 ID 1
|
||||
@ -2007,6 +2009,7 @@ export function AdminSettingsClient() {
|
||||
isHidden: false,
|
||||
isDefault: false,
|
||||
isAgentModel: false,
|
||||
agentCodingModelId: "",
|
||||
isMultimodal: false,
|
||||
sortOrder: 0,
|
||||
fallbackModelId1: "",
|
||||
@ -6138,6 +6141,24 @@ export function AdminSettingsClient() {
|
||||
/>
|
||||
<Label>Agent 自动模型(前端隐藏,仅后台 Agent 使用)</Label>
|
||||
</div>
|
||||
{modelForm.isAgentModel && (
|
||||
<div className="space-y-2">
|
||||
<Label className="text-sm">编程模型 ID(可选)</Label>
|
||||
<Input
|
||||
value={modelForm.agentCodingModelId}
|
||||
onChange={(e) =>
|
||||
setModelForm({
|
||||
...modelForm,
|
||||
agentCodingModelId: e.target.value,
|
||||
})
|
||||
}
|
||||
placeholder="例如: doubao-seed-2-0-code-preview-260215"
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Agent 需要结构化规划/编程输出时优先使用该模型 ID,留空则使用当前 Agent 模型。
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
<div className="flex items-center gap-2">
|
||||
<Switch
|
||||
checked={modelForm.isMultimodal || false}
|
||||
@ -6355,6 +6376,7 @@ export function AdminSettingsClient() {
|
||||
isHidden: model.isHidden || false,
|
||||
isDefault: model.isDefault,
|
||||
isAgentModel: model.isAgentModel || false,
|
||||
agentCodingModelId: model.agentCodingModelId || "",
|
||||
isMultimodal: model.isMultimodal || false,
|
||||
sortOrder: model.sortOrder || 0,
|
||||
fallbackModelId1: (model as any).fallbackModelId1 || "",
|
||||
|
||||
@ -10,7 +10,11 @@ import {
|
||||
getAiProviderById,
|
||||
} from "@/lib/db/queries";
|
||||
import { ChatSDKError } from "@/lib/errors";
|
||||
import { isAiModelAgentAuto, isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||||
import {
|
||||
getAiModelAgentCodingModelId,
|
||||
isAiModelAgentAuto,
|
||||
isAiModelMultimodal,
|
||||
} from "@/lib/db/ai-model-flags";
|
||||
|
||||
// 🆕 图片生成模型特性配置 Schema
|
||||
const imageGenConfigSchema = z.object({
|
||||
@ -41,6 +45,7 @@ const createModelSchema = z.object({
|
||||
fallbackModelId1: z.string().max(128).optional(),
|
||||
fallbackModelId2: z.string().max(128).optional(),
|
||||
isAgentModel: z.boolean().optional(),
|
||||
agentCodingModelId: z.string().max(128).optional(),
|
||||
isMultimodal: z.boolean().optional(),
|
||||
imageGenConfig: imageGenConfigSchema, // 🆕 图片生成特性配置
|
||||
});
|
||||
@ -61,6 +66,7 @@ const updateModelSchema = z.object({
|
||||
fallbackModelId1: z.string().max(128).optional(),
|
||||
fallbackModelId2: z.string().max(128).optional(),
|
||||
isAgentModel: z.boolean().optional(),
|
||||
agentCodingModelId: z.string().max(128).optional(),
|
||||
isMultimodal: z.boolean().optional(),
|
||||
imageGenConfig: imageGenConfigSchema, // 🆕 图片生成特性配置
|
||||
});
|
||||
@ -69,6 +75,7 @@ function normalizeModel(model: any) {
|
||||
return {
|
||||
...model,
|
||||
isAgentModel: isAiModelAgentAuto(model),
|
||||
agentCodingModelId: getAiModelAgentCodingModelId(model) || "",
|
||||
isMultimodal: isAiModelMultimodal(model),
|
||||
};
|
||||
}
|
||||
|
||||
@ -5,8 +5,12 @@ import {
|
||||
getAgentAiModelWithProvider,
|
||||
getEnabledAiModelsWithProvider,
|
||||
} from "@/lib/db/queries";
|
||||
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||||
import {
|
||||
getAiModelAgentCodingModelId,
|
||||
isAiModelMultimodal,
|
||||
} from "@/lib/db/ai-model-flags";
|
||||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||||
import { inlineVisionImages } from "@/lib/ai/vision-inline";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 180;
|
||||
@ -1598,6 +1602,7 @@ export async function POST(request: Request) {
|
||||
id: plannerConfig.model.id,
|
||||
displayName: plannerConfig.model.displayName,
|
||||
modelId: plannerConfig.model.modelId,
|
||||
codingModelId: getAiModelAgentCodingModelId(plannerConfig.model),
|
||||
providerType: plannerConfig.provider.type,
|
||||
supportsVision: plannerConfig.model.supportsVision,
|
||||
multimodalFlag: isAiModelMultimodal({
|
||||
@ -1607,7 +1612,11 @@ export async function POST(request: Request) {
|
||||
hasReferenceImages: referenceImages.length > 0,
|
||||
});
|
||||
|
||||
const model = await getLanguageModelById(plannerConfig.model.id);
|
||||
const codingModelId = getAiModelAgentCodingModelId(plannerConfig.model);
|
||||
const model = await getLanguageModelById(
|
||||
plannerConfig.model.id,
|
||||
codingModelId
|
||||
);
|
||||
if (!model) {
|
||||
return NextResponse.json({ success: true, data: fallback });
|
||||
}
|
||||
@ -1638,7 +1647,15 @@ export async function POST(request: Request) {
|
||||
|
||||
const imageInputs =
|
||||
supportsVisionPlanning(plannerConfig.model) && referenceImages.length > 0
|
||||
? referenceImages.map((url) => ({ type: "image" as const, image: url }))
|
||||
? (
|
||||
await inlineVisionImages(referenceImages, {
|
||||
maxImages: 6,
|
||||
maxSide: 900,
|
||||
timeoutMs: 12000,
|
||||
retries: 2,
|
||||
loggerPrefix: "[Canvas Detail Plan]",
|
||||
})
|
||||
).inlineDataUrls.map((dataUrl) => ({ type: "image" as const, image: dataUrl }))
|
||||
: [];
|
||||
|
||||
let text = "";
|
||||
|
||||
@ -5,8 +5,12 @@ import {
|
||||
getAgentAiModelWithProvider,
|
||||
getEnabledAiModelsWithProvider,
|
||||
} from "@/lib/db/queries";
|
||||
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||||
import {
|
||||
getAiModelAgentCodingModelId,
|
||||
isAiModelMultimodal,
|
||||
} from "@/lib/db/ai-model-flags";
|
||||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||||
import { inlineVisionImages } from "@/lib/ai/vision-inline";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 60;
|
||||
@ -186,7 +190,11 @@ export async function POST(request: Request) {
|
||||
});
|
||||
}
|
||||
|
||||
const model = await getLanguageModelById(reviewerConfig.model.id);
|
||||
const codingModelId = getAiModelAgentCodingModelId(reviewerConfig.model);
|
||||
const model = await getLanguageModelById(
|
||||
reviewerConfig.model.id,
|
||||
codingModelId
|
||||
);
|
||||
if (!model) {
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
@ -238,10 +246,18 @@ export async function POST(request: Request) {
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
|
||||
const { inlineDataUrls } = await inlineVisionImages([imageUrl, ...referenceImages], {
|
||||
maxImages: 6,
|
||||
maxSide: 900,
|
||||
timeoutMs: 12000,
|
||||
retries: 2,
|
||||
loggerPrefix: "[Canvas Detail Review]",
|
||||
});
|
||||
const effectiveReviewImages = inlineDataUrls.length > 0 ? inlineDataUrls : [imageUrl, ...referenceImages];
|
||||
|
||||
const content: Array<{ type: "text"; text: string } | { type: "image"; image: string }> = [
|
||||
{ type: "text", text: promptParts },
|
||||
{ type: "image", image: imageUrl },
|
||||
...referenceImages.map((url) => ({ type: "image" as const, image: url })),
|
||||
...effectiveReviewImages.map((image) => ({ type: "image" as const, image })),
|
||||
];
|
||||
|
||||
const { text } = await generateText({
|
||||
|
||||
@ -5397,10 +5397,10 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
}, [activeSessionTitle, projectTitle]);
|
||||
|
||||
return (
|
||||
<div className="relative flex h-full min-h-0 flex-col bg-background">
|
||||
<div className="relative flex h-full min-h-0 flex-col bg-background/95 backdrop-blur-sm">
|
||||
{/* 全局遮罩提示 - 当处于画布选图模式时显示 */}
|
||||
{detailSlotPickerIndex !== null && (
|
||||
<div className="absolute inset-0 z-50 flex flex-col items-center justify-center bg-background/80 backdrop-blur-sm animate-in fade-in duration-200">
|
||||
<div className="absolute inset-0 z-50 flex flex-col items-center justify-center bg-background/90 backdrop-blur-sm animate-in fade-in duration-200">
|
||||
<div className="flex flex-col items-center gap-4 p-6 text-center">
|
||||
<div className="flex h-12 w-12 items-center justify-center rounded-full bg-primary/10 text-primary ring-4 ring-primary/5 animate-pulse">
|
||||
<ImagePlus className="h-6 w-6" />
|
||||
@ -5463,7 +5463,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
<History className="h-4 w-4" />
|
||||
</Button>
|
||||
{isSessionMenuOpen && (
|
||||
<div className="absolute right-0 top-10 w-64 rounded-xl border bg-background/90 backdrop-blur-md shadow-xl p-2">
|
||||
<div className="absolute right-0 top-10 w-64 rounded-xl border bg-background/95 backdrop-blur-sm shadow-xl p-2">
|
||||
<div className="px-2 py-1 text-xs text-muted-foreground">聊天窗口记录</div>
|
||||
<div className="max-h-56 overflow-y-auto space-y-1">
|
||||
{chatSessions.length === 0 ? (
|
||||
@ -6052,7 +6052,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
)}
|
||||
|
||||
{/* 底部输入区域 - 混合输入(图片标签+文字可自由混合) */}
|
||||
<div className="p-4 bg-background border-t">
|
||||
<div className="p-4 bg-background/95 backdrop-blur-sm border-t">
|
||||
<div
|
||||
className="relative flex flex-col bg-card rounded-3xl border transition-all duration-200 focus-within:border-primary/50 focus-within:bg-background"
|
||||
data-canvas-chat-input-anchor="true"
|
||||
|
||||
@ -99,7 +99,7 @@ export function CanvasFileList({ open, onOpenChange, currentId }: CanvasFileList
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="max-w-md max-h-[80vh] flex flex-col p-0 gap-0 overflow-hidden bg-background">
|
||||
<DialogContent className="max-w-md max-h-[80vh] flex flex-col p-0 gap-0 overflow-hidden bg-background/95 backdrop-blur-sm">
|
||||
<DialogHeader className="p-4 border-b">
|
||||
<div className="flex items-center justify-between">
|
||||
<DialogTitle>已生成文件列表</DialogTitle>
|
||||
|
||||
@ -956,7 +956,7 @@ export function RealtimeCanvasDialog({
|
||||
</DialogHeader>
|
||||
{/* Top Header - Minimal */}
|
||||
<div className="absolute left-0 right-0 top-0 z-50 flex h-14 items-center justify-start px-6 bg-gradient-to-b from-black/20 to-transparent pointer-events-none">
|
||||
<div className="pointer-events-auto flex items-center gap-3 rounded-full bg-background px-4 py-1.5 border border-border/50 shadow-sm">
|
||||
<div className="pointer-events-auto flex items-center gap-3 rounded-full bg-background/95 backdrop-blur-sm px-4 py-1.5 border border-border/50 shadow-sm">
|
||||
<span className="font-semibold text-sm">Realtime Canvas</span>
|
||||
<div className="h-4 w-[1px] bg-border" />
|
||||
<span className={cn("flex items-center gap-1.5 text-xs font-medium", statusBadge.className, "bg-transparent p-0")}>
|
||||
|
||||
523
docs/AI详情生成-AGENT可编程渲染开发文档.md
Normal file
523
docs/AI详情生成-AGENT可编程渲染开发文档.md
Normal file
@ -0,0 +1,523 @@
|
||||
# AI详情生成(AGENT + 生图模型 + 可编程渲染)开发文档
|
||||
|
||||
## 1. 文档目的
|
||||
|
||||
本文档用于落地一套稳定的 AI 详情生成系统:
|
||||
|
||||
- 保留现有传统生图模式(快速产图路线)
|
||||
- 保留生图模型能力(画面创作、质感、场景)
|
||||
- 用 AGENT 负责策划、文案、结构和审稿
|
||||
- 用编程渲染保证版式统一和可编辑
|
||||
- 支持用户切换两种模式(传统生图 / AGENT排版)
|
||||
- 最终结果直接进入无限画布,支持手动改字、拖拽图标、单屏重生
|
||||
|
||||
目标是解决当前主要问题:生图模型理解偏差、全套风格不统一、失败不可控、返工成本高。
|
||||
|
||||
---
|
||||
|
||||
## 2. 核心结论与原则
|
||||
|
||||
### 2.1 核心结论
|
||||
|
||||
不能只靠生图模型“整页一次生成”。推荐架构是:
|
||||
|
||||
`AGENT 策划 -> 分屏生成 -> 程序化合成 -> 画布可编辑 -> 审核回路`
|
||||
|
||||
同时保留现有传统生图链路,采用双路线并行:
|
||||
|
||||
- 路线 A:`传统生图模式`(保持现状,快速出图)
|
||||
- 路线 B:`AGENT排版模式`(生图只负责画面,文字与排版交给程序)
|
||||
|
||||
### 2.2 设计原则
|
||||
|
||||
1. **模型出画面,代码定版式**
|
||||
2. **背景锁定,前景可生**(避免模型重画结构)
|
||||
3. **每屏可独立重试**(不整套重跑)
|
||||
4. **所有输入输出可追溯**(项目、版本、日志、成本)
|
||||
5. **画布内可编辑优先**(文字和图标非烤图)
|
||||
6. **双模式可切换且互不影响**(老链路稳定,新链路增量上线)
|
||||
|
||||
---
|
||||
|
||||
## 3. 业务范围
|
||||
|
||||
## 3.1 本期范围
|
||||
|
||||
- AI详情模式固定 14 屏流程(可配置)
|
||||
- 每屏固定尺寸 `790x1300`
|
||||
- 用户上传:
|
||||
- 5 张产品图(必填)
|
||||
- 1 张模特图(可用于卖家视角场景)
|
||||
- 3 张搭配图(可选,触发关联销售屏)
|
||||
- 输入业务信息:人群、场景、核心需求、SPU 等
|
||||
- 自动生成详情文案与视觉版式
|
||||
- 结果进入画布后可修改文字、拖拽图标、替换素材
|
||||
- 自动存档、历史恢复、继续生成
|
||||
|
||||
## 3.2 非本期范围
|
||||
|
||||
- 移动端编辑体验优化
|
||||
- 多人协同编辑冲突合并
|
||||
- 自动电商平台一键上架
|
||||
|
||||
## 3.3 模式策略(新增)
|
||||
|
||||
- `传统生图模式(classic)`
|
||||
- 使用现有生图流程
|
||||
- 保留当前用户习惯与响应节奏
|
||||
- 适合快速尝试和低编辑需求场景
|
||||
- `AGENT排版模式(agent_layout)`
|
||||
- 生图模型只产出画面素材层,不产出文字排版
|
||||
- AGENT + 程序完成文案结构、版式和 UI 统一
|
||||
- 适合稳定量产、可编辑和可复用场景
|
||||
- 前端必须提供模式切换,并把模式写入任务上下文,后端按模式分流 pipeline
|
||||
|
||||
---
|
||||
|
||||
## 4. 技术架构
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
A["用户输入(图片+业务信息)"] --> B["AGENT 规划器"]
|
||||
B --> C["ScreenSpec 14屏结构化脚本"]
|
||||
C --> D["生图编排器(逐屏调用模型)"]
|
||||
D --> E["程序化合成器(背景/文案/图标)"]
|
||||
E --> F["双总监审稿(设计+营销)"]
|
||||
F -->|通过| G["写入S3/MinIO + DB版本"]
|
||||
F -->|不通过| D
|
||||
G --> H["画布落地(垂直排版)"]
|
||||
H --> I["手动编辑(改字/拖拽/重生单屏)"]
|
||||
```
|
||||
|
||||
双路线分流(逻辑):
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
U["用户选择模式"] --> M{"generation_mode"}
|
||||
M -->|classic| C1["传统生图Pipeline"]
|
||||
M -->|agent_layout| C2["AGENT排版Pipeline"]
|
||||
C1 --> R["统一写入画布与历史记录"]
|
||||
C2 --> R
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. 推荐开源技术选型
|
||||
|
||||
## 5.1 画布编辑层
|
||||
|
||||
- `react-konva + konva`:用于画布图层编辑、拖拽、缩放、导出
|
||||
- `Moveable` 或 `interact.js`:吸附对齐、参考线、间距控制
|
||||
|
||||
## 5.2 渲染层
|
||||
|
||||
- 主方案:HTML/CSS 模板 + Playwright 截图(最接近设计稿控制)
|
||||
- 备选:`Satori + resvg-js`(高一致性服务端矢量渲染)
|
||||
|
||||
## 5.3 模型编排层
|
||||
|
||||
- 继续使用当前生图供应商体系(支持路由和故障迁移)
|
||||
- AGENT 负责:
|
||||
- 详情策略规划
|
||||
- Prompt 生成与优化
|
||||
- 结果审稿和重试建议
|
||||
|
||||
## 5.4 可选参考(不直接引入)
|
||||
|
||||
- Excalidraw:交互/白板体验参考
|
||||
- Penpot:组件化设计体系参考
|
||||
- tldraw:能力强,但许可证需谨慎评估
|
||||
|
||||
---
|
||||
|
||||
## 6. 数据模型设计(数据库)
|
||||
|
||||
建议新增或扩展以下表(命名可按现有规范调整):
|
||||
|
||||
## 6.1 `detail_projects`
|
||||
|
||||
- `id`
|
||||
- `user_id`
|
||||
- `canvas_id`
|
||||
- `spu`
|
||||
- `demand_type`
|
||||
- `project_name`(默认 `SPU+需求类型`)
|
||||
- `status`(draft/planning/generating/reviewing/completed/failed)
|
||||
- `generation_mode`(classic/agent_layout,默认 classic)
|
||||
- `created_at` / `updated_at`
|
||||
|
||||
## 6.2 `detail_project_inputs`
|
||||
|
||||
- `id`
|
||||
- `project_id`
|
||||
- `core_audience`
|
||||
- `usage_scenarios`
|
||||
- `core_needs`
|
||||
- `selling_points`
|
||||
- `tone_style`
|
||||
- `extra_requirements`
|
||||
- `raw_form_json`
|
||||
|
||||
## 6.3 `detail_assets`
|
||||
|
||||
- `id`
|
||||
- `project_id`
|
||||
- `asset_type`(product/model/combo/reference)
|
||||
- `source_url`
|
||||
- `storage_url`
|
||||
- `width` / `height`
|
||||
- `meta_json`
|
||||
|
||||
## 6.4 `detail_screen_specs`
|
||||
|
||||
- `id`
|
||||
- `project_id`
|
||||
- `screen_index`(1..14)
|
||||
- `screen_type`
|
||||
- `spec_json`(可编辑图层结构)
|
||||
- `prompt_json`
|
||||
- `status`
|
||||
|
||||
## 6.5 `detail_screen_renders`
|
||||
|
||||
- `id`
|
||||
- `project_id`
|
||||
- `screen_index`
|
||||
- `version_no`
|
||||
- `image_url`
|
||||
- `thumb_url`
|
||||
- `model_provider`
|
||||
- `model_name`
|
||||
- `cost_credits`
|
||||
- `latency_ms`
|
||||
- `review_score_design`
|
||||
- `review_score_marketing`
|
||||
- `review_notes`
|
||||
|
||||
## 6.6 `detail_generation_jobs`
|
||||
|
||||
- `id`
|
||||
- `project_id`
|
||||
- `job_type`(plan/generate/review/retry)
|
||||
- `generation_mode`(classic/agent_layout)
|
||||
- `progress`(0~100)
|
||||
- `stage`(规划中/生成中/审稿中/完成)
|
||||
- `error_code`
|
||||
- `error_message`
|
||||
- `started_at` / `finished_at`
|
||||
|
||||
## 6.7 现有聊天/消息表扩展(建议)
|
||||
|
||||
- 在会话或任务表增加 `generation_mode`
|
||||
- 在消息元数据增加 `mode_snapshot`(防止切换模式导致历史重放错乱)
|
||||
- 在生成记录增加 `resolved_pipeline`(最终实际执行链路)
|
||||
|
||||
---
|
||||
|
||||
## 7. ScreenSpec(核心协议)
|
||||
|
||||
AGENT 和渲染器之间只通过结构化协议通信,不直接拼字符串 UI。
|
||||
|
||||
```json
|
||||
{
|
||||
"projectId": "dp_xxx",
|
||||
"canvasId": "cv_xxx",
|
||||
"size": { "width": 790, "height": 1300 },
|
||||
"globalStyle": {
|
||||
"fontFamily": "PingFang SC",
|
||||
"titleSize": 68,
|
||||
"subtitleSize": 36,
|
||||
"bodySize": 28,
|
||||
"lineHeight": 1.35,
|
||||
"palette": { "primary": "#5B4DFF", "text": "#111111" }
|
||||
},
|
||||
"screen": {
|
||||
"index": 1,
|
||||
"type": "hero_poster",
|
||||
"copy": {
|
||||
"name": "安敏裤",
|
||||
"title": "安敏裤 | 敏宝整夜安睡不红屁屁",
|
||||
"subtitle": "过敏体质宝宝专研,夜用安心不反渗"
|
||||
},
|
||||
"layers": [
|
||||
{ "id": "bg", "kind": "backgroundImage", "editable": false, "src": "https://..." },
|
||||
{ "id": "title", "kind": "text", "editable": true, "x": 56, "y": 120, "w": 678, "h": 180 },
|
||||
{ "id": "badge", "kind": "icon", "editable": true, "x": 56, "y": 340, "w": 160, "h": 52 }
|
||||
],
|
||||
"gen": {
|
||||
"needModelImage": true,
|
||||
"prompt": "只生成产品前景和光效,不生成文字,不改背景结构",
|
||||
"negativePrompt": "杂乱文字, 水印, logo"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. 14屏生产流程(标准)
|
||||
|
||||
1. 首屏海报(命名+核心需求一句话)
|
||||
2. 用户痛点
|
||||
3. 核心卖点机制
|
||||
4. 卖点证据/参数
|
||||
5. 场景 1(日常)
|
||||
6. 场景 2(高频痛点)
|
||||
7. 细节特写
|
||||
8. 对比页
|
||||
9. 使用方法
|
||||
10. 搭配销售(用户上传搭配图时启用)
|
||||
11. 买家评价 1(自动生成评价+3图)
|
||||
12. 买家评价 2(可再次生成新2条)
|
||||
13. 品牌背书/服务承诺
|
||||
14. 收口 CTA
|
||||
|
||||
约束:必须完整到第 14 屏才能标记 `completed`。
|
||||
|
||||
---
|
||||
|
||||
## 9. AGENT 职责拆分
|
||||
|
||||
## 9.1 Planner Agent(规划师)
|
||||
|
||||
- 解析输入信息和图片素材
|
||||
- 生成 14 屏文案骨架与屏幕目标
|
||||
- 生成命名(大白话、可记忆、带卖点)
|
||||
|
||||
## 9.2 Prompt Agent(生图导演)
|
||||
|
||||
- 将每屏目标转换为生图 prompt
|
||||
- 强约束“仅出前景元素,不写文案”
|
||||
- 设定风格一致性关键词
|
||||
|
||||
## 9.3 Review Agent(双总监审稿)
|
||||
|
||||
- 设计总监评分:版式层级、视觉统一、可读性
|
||||
- 营销总监评分:卖点清晰、转化逻辑、购买驱动
|
||||
- 低于阈值自动重生单屏
|
||||
|
||||
---
|
||||
|
||||
## 10. 画布集成要求
|
||||
|
||||
## 10.1 渲染落位
|
||||
|
||||
- 14 屏按顺序垂直插入画布
|
||||
- 固定屏间距(建议 80px)
|
||||
- 自动居中对齐
|
||||
- 生成时先放占位卡(含进度)
|
||||
|
||||
## 10.2 编辑能力
|
||||
|
||||
- 文本层可直接双击编辑
|
||||
- 图标层可拖拽/缩放
|
||||
- 支持吸附对齐线(含最小间距,避免粘连)
|
||||
- 单屏“重生”按钮(保留版式,仅替换画面)
|
||||
|
||||
## 10.3 存档与恢复
|
||||
|
||||
- 自动保存当前编辑状态(去抖)
|
||||
- 恢复项目时还原全部输入、素材、屏幕版本和图层状态
|
||||
- 支持“另存为新版本”
|
||||
|
||||
---
|
||||
|
||||
## 11. API 设计(建议)
|
||||
|
||||
## 11.1 项目接口
|
||||
|
||||
- `POST /api/canvas/[id]/detail-projects`:创建项目
|
||||
- `GET /api/canvas/[id]/detail-projects`:项目列表
|
||||
- `GET /api/canvas/[id]/detail-projects/:projectId`:项目详情
|
||||
- `DELETE /api/canvas/[id]/detail-projects/:projectId`:删除存档
|
||||
|
||||
## 11.2 生成接口
|
||||
|
||||
- `POST /api/canvas/[id]/detail-projects/:projectId/plan`
|
||||
- `POST /api/canvas/[id]/detail-projects/:projectId/generate`
|
||||
- `POST /api/canvas/[id]/detail-projects/:projectId/retry-screen`
|
||||
- `POST /api/canvas/[id]/detail-projects/:projectId/review`
|
||||
- 请求体需带 `generation_mode`,或由项目默认模式继承
|
||||
|
||||
## 11.3 进度接口
|
||||
|
||||
- `GET /api/canvas/[id]/detail-projects/:projectId/progress`(SSE 推荐)
|
||||
- 前端展示阶段:规划中 -> 生成中 -> 审稿中 -> 完成/失败
|
||||
|
||||
## 11.4 模式切换接口(新增)
|
||||
|
||||
- `POST /api/canvas/[id]/generation-mode`
|
||||
- 入参:`conversationId`、`generation_mode`
|
||||
- 作用:更新当前会话默认模式
|
||||
- 模式切换只影响新任务,不回写旧任务执行链路
|
||||
|
||||
---
|
||||
|
||||
## 12. 失败与兜底策略
|
||||
|
||||
1. 单屏失败优先重试,不整套失败
|
||||
2. 主模型失败自动故障迁移
|
||||
3. 审稿失败回到该屏 prompt 优化重生
|
||||
4. 图片 URL 拉取失败时切换缩略图/中转 URL
|
||||
5. 超时任务可恢复继续跑,不丢进度
|
||||
|
||||
---
|
||||
|
||||
## 13. 性能与成本控制
|
||||
|
||||
- 生图并发建议:2~3(避免过载和限流)
|
||||
- 文字层和图标层走本地渲染,减少模型调用
|
||||
- 单屏缓存:输入未变则复用结果
|
||||
- 记录每屏耗时与积分,输出成本报表
|
||||
|
||||
---
|
||||
|
||||
## 13.1 排版遮挡与可读性自动质检(新增)
|
||||
|
||||
为避免“文字压住产品主体、文案不可读、版式失衡”,在每屏渲染后新增 `Layout QA` 阶段。
|
||||
|
||||
### A. 质检输入
|
||||
|
||||
- 渲染结果图(790x1300)
|
||||
- `ScreenSpec` 图层信息(文本框、图标框、背景层)
|
||||
- 产品主体掩码 `product_mask`(由分割模型产出,建议 SAM2 或 YOLO Seg)
|
||||
- 图像显著区域 `saliency_map`(可选)
|
||||
|
||||
### B. 核心判定指标
|
||||
|
||||
- `occlusion_ratio = area(text_box ∩ product_mask) / area(product_mask)`
|
||||
- `text_intrusion = area(text_box ∩ product_mask) / area(text_box)`
|
||||
- 文本区域对比度(背景亮度与文字颜色对比)
|
||||
- 安全区越界(顶部标题区、边距、栅格)
|
||||
|
||||
### C. 建议阈值(首版)
|
||||
|
||||
- `occlusion_ratio <= 0.03`
|
||||
- `text_intrusion <= 0.10`
|
||||
- 主标题与产品主体最小距离 >= 24px
|
||||
- 正文对比度 >= 4.5:1;大标题对比度 >= 3.0:1
|
||||
- 文本不得覆盖显著区域 Top 20%
|
||||
|
||||
### D. 自动修复策略
|
||||
|
||||
1. 先移动文本框(上 -> 下 -> 侧)
|
||||
2. 再缩短文案(调用 AGENT copy shortener)
|
||||
3. 再调整字号和行高(保持全局层级)
|
||||
4. 最后加底板/描边增强可读性
|
||||
5. 超过重排次数仍失败 -> 标记人工介入
|
||||
|
||||
### E. 结果落库
|
||||
|
||||
- 每屏记录 `qa_passed`、`qa_score`、`qa_issues`、`qa_fix_actions`
|
||||
- 支持在画布中打开“质检报告”,可一键触发该屏重排/重生
|
||||
|
||||
---
|
||||
|
||||
## 14. 安全与合规
|
||||
|
||||
- 所有 API Key 仅服务端读取(数据库加密存储)
|
||||
- 前端不暴露供应商密钥
|
||||
- 生成内容日志脱敏(URL、用户隐私)
|
||||
- 存储地址统一走可控域名,避免跨域拉图波动
|
||||
|
||||
---
|
||||
|
||||
## 15. 里程碑计划
|
||||
|
||||
## 阶段 A(1 周):可跑通 MVP
|
||||
|
||||
- 前端模式切换(传统生图 / AGENT排版)
|
||||
- 后端按 `generation_mode` 分流
|
||||
- ScreenSpec 协议
|
||||
- 14 屏生成状态机
|
||||
- 画布垂直落位 + 占位卡
|
||||
- 单屏重试
|
||||
|
||||
## 阶段 B(1~2 周):可编辑与存档
|
||||
|
||||
- 文本层编辑、图标拖拽、吸附线
|
||||
- 项目存档、恢复、版本化
|
||||
- 失败恢复和续跑
|
||||
|
||||
## 阶段 C(1 周):质量增强
|
||||
|
||||
- 双总监审稿自动回路
|
||||
- 屏幕质量评分
|
||||
- 成本和速度优化
|
||||
|
||||
---
|
||||
|
||||
## 16. 验收标准(必须满足)
|
||||
|
||||
1. 14 屏完整生成成功率 >= 95%(可重试后)
|
||||
2. 每屏版式和字体风格一致,无明显错位
|
||||
3. 画布中可改字、可拖拽图标、可单屏重生
|
||||
4. 刷新页面后项目状态和结果不丢失
|
||||
5. 失败有明确原因并可一键重试
|
||||
|
||||
---
|
||||
|
||||
## 17. 风险与应对
|
||||
|
||||
- **风险:模型仍偏题** -> 强化 prompt 约束 + 审稿回路
|
||||
- **风险:图片链路不稳定** -> 同域代理/中转 + 缩略图兜底
|
||||
- **风险:编辑体验复杂** -> 分层上线,先文本编辑再高级控件
|
||||
- **风险:成本高** -> 单屏缓存、只重跑失败屏、限制并发
|
||||
|
||||
---
|
||||
|
||||
## 18. 开发清单(执行版)
|
||||
|
||||
- [ ] 新增详情项目数据表与迁移脚本
|
||||
- [ ] 实现 ScreenSpec 生成器(AGENT Planner)
|
||||
- [ ] 实现每屏生图编排器
|
||||
- [ ] 实现程序化合成器(固定 790x1300)
|
||||
- [ ] 实现 Layout QA(遮挡检测、可读性检测、自动重排)
|
||||
- [ ] 实现双总监审稿器和重试策略
|
||||
- [ ] 接入画布垂直落位与占位进度
|
||||
- [ ] 接入文本编辑/图标拖拽/吸附线
|
||||
- [ ] 接入存档、恢复、删除、另存版本
|
||||
- [ ] 接入监控日志与成本统计
|
||||
|
||||
---
|
||||
|
||||
## 19. 编程生成PPT技术路线参考(新增)
|
||||
|
||||
本项目主渲染仍以“画布图层渲染”为核心,PPT 相关能力建议定位为“导出能力”。
|
||||
|
||||
### 20.1 推荐路线(与当前栈匹配)
|
||||
|
||||
- `PptxGenJS`(Node/TS 生态)
|
||||
- 优点:与现有前后端栈一致、可编程布局能力强、便于把 `ScreenSpec` 映射为幻灯片对象
|
||||
- 适用:把 14 屏详情导出为可编辑 PPTX 交付给运营团队
|
||||
|
||||
### 20.2 可选路线
|
||||
|
||||
- `python-pptx`
|
||||
- 优点:批处理成熟,适合离线报告任务
|
||||
- 缺点:需要额外 Python 服务,增加运维与调用链复杂度
|
||||
- `Open XML SDK`(.NET)/ `Apache POI`(Java)
|
||||
- 适合企业系统深度集成,不作为当前首选
|
||||
|
||||
### 20.3 参考价值判断
|
||||
|
||||
- 有价值点:
|
||||
- `ScreenSpec -> PPTX` 的结构化映射
|
||||
- 模板化母版(Master)和统一字号体系
|
||||
- 批量导出与版本归档
|
||||
- 无法替代点:
|
||||
- PPT 方案本身不解决“文字遮挡主体”问题,仍需 Layout QA
|
||||
- 不适合替代当前画布内精细编辑与逐屏重生流程
|
||||
|
||||
### 20.4 落地建议
|
||||
|
||||
1. 先完成画布内稳定生产(ScreenSpec + Layout QA + 自动重排)
|
||||
2. 再增加 `Export to PPTX`,作为交付层功能
|
||||
3. 导出时保留标题/副标题/正文可编辑,背景图锁定,避免误改布局
|
||||
|
||||
---
|
||||
|
||||
## 20. 结语
|
||||
|
||||
这套方案不是“取消生图模型”,而是把生图模型放到最擅长的位置(视觉生成),把结构一致性、文案逻辑和生产稳定性交给 AGENT + 编程系统。
|
||||
对你当前业务目标(稳定出高转化详情、可持续迭代)是更可控、可扩展、可运营的路径。
|
||||
@ -3,7 +3,10 @@ import { generateText } from "ai";
|
||||
import sharp from "sharp";
|
||||
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
|
||||
import type { RouteResult } from "@/lib/ai/model-router";
|
||||
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||||
import {
|
||||
getAiModelAgentCodingModelId,
|
||||
isAiModelMultimodal,
|
||||
} from "@/lib/db/ai-model-flags";
|
||||
|
||||
interface HistoryMessage {
|
||||
role: "user" | "assistant";
|
||||
@ -688,6 +691,26 @@ function parseJsonBlock<T = Record<string, unknown>>(text: string): T | null {
|
||||
}
|
||||
}
|
||||
|
||||
function resolveAgentCodingModelOverride(
|
||||
selectedModelId: string,
|
||||
configuredAgentModel:
|
||||
| {
|
||||
model?: {
|
||||
id?: string | null;
|
||||
defaultParams?: unknown;
|
||||
capabilities?: unknown;
|
||||
} | null;
|
||||
}
|
||||
| null
|
||||
| undefined,
|
||||
): string | null {
|
||||
const configuredId = configuredAgentModel?.model?.id || null;
|
||||
if (!configuredId || configuredId !== selectedModelId) {
|
||||
return null;
|
||||
}
|
||||
return getAiModelAgentCodingModelId(configuredAgentModel?.model || {}) || null;
|
||||
}
|
||||
|
||||
async function generateAgentConversation(
|
||||
analysis: AnalysisResult,
|
||||
state: AutoAgentStateType
|
||||
@ -714,7 +737,10 @@ async function generateAgentConversation(
|
||||
if (!reasoningModelConfig) {
|
||||
return analysis.conversationalReply || null;
|
||||
}
|
||||
const reasoningModel = await getLanguageModelById(reasoningModelConfig.model.id);
|
||||
const reasoningModel = await getLanguageModelById(
|
||||
reasoningModelConfig.model.id,
|
||||
resolveAgentCodingModelOverride(reasoningModelConfig.model.id, agentModel),
|
||||
);
|
||||
if (!reasoningModel) {
|
||||
return analysis.conversationalReply || null;
|
||||
}
|
||||
@ -858,7 +884,10 @@ async function recommendImageParamsWithAgent(
|
||||
if (!plannerModelConfig) {
|
||||
return heuristicImageParams(state.text, analysis);
|
||||
}
|
||||
const plannerModel = await getLanguageModelById(plannerModelConfig.model.id);
|
||||
const plannerModel = await getLanguageModelById(
|
||||
plannerModelConfig.model.id,
|
||||
resolveAgentCodingModelOverride(plannerModelConfig.model.id, agentModel),
|
||||
);
|
||||
if (!plannerModel) {
|
||||
return heuristicImageParams(state.text, analysis);
|
||||
}
|
||||
@ -933,7 +962,10 @@ async function recommendVideoPlanWithAgent(
|
||||
if (!plannerModelConfig) {
|
||||
return heuristicVideoPlan(state);
|
||||
}
|
||||
const plannerModel = await getLanguageModelById(plannerModelConfig.model.id);
|
||||
const plannerModel = await getLanguageModelById(
|
||||
plannerModelConfig.model.id,
|
||||
resolveAgentCodingModelOverride(plannerModelConfig.model.id, agentModel),
|
||||
);
|
||||
if (!plannerModel) {
|
||||
return heuristicVideoPlan(state);
|
||||
}
|
||||
@ -1041,7 +1073,10 @@ async function refineRouteWithAgent(
|
||||
});
|
||||
const plannerModelConfig = pickAgentReasoningModel(primaryAgentModel, allModelsRaw);
|
||||
if (!plannerModelConfig) return baseRoute;
|
||||
const plannerModel = await getLanguageModelById(plannerModelConfig.model.id);
|
||||
const plannerModel = await getLanguageModelById(
|
||||
plannerModelConfig.model.id,
|
||||
resolveAgentCodingModelOverride(plannerModelConfig.model.id, agentModel),
|
||||
);
|
||||
if (!plannerModel) return baseRoute;
|
||||
|
||||
const candidatesText = imageCandidates
|
||||
|
||||
@ -228,7 +228,8 @@ function getOrCreateProvider(provider: Pick<AiProvider, "id" | "type" | "baseUrl
|
||||
|
||||
// 获取动态配置的语言模型
|
||||
export async function getDynamicModel(
|
||||
modelId: string
|
||||
modelId: string,
|
||||
overrideModelId?: string | null
|
||||
): Promise<LanguageModel | null> {
|
||||
try {
|
||||
const model = await getAiModelById({ id: modelId });
|
||||
@ -256,8 +257,10 @@ export async function getDynamicModel(
|
||||
return null;
|
||||
}
|
||||
|
||||
const resolvedModelId = String(overrideModelId || "").trim() || model.modelId;
|
||||
|
||||
// 创建基础模型
|
||||
console.log(`[getDynamicModel] Creating model: ${model.modelId} (${model.displayName}) using provider: ${provider.name} (${provider.type})`);
|
||||
console.log(`[getDynamicModel] Creating model: ${resolvedModelId} (${model.displayName}) using provider: ${provider.name} (${provider.type})`);
|
||||
|
||||
// 根据 provider 类型选择正确的方法和模型ID
|
||||
// customProvider (gemini, local) 使用 .languageModel()
|
||||
@ -266,56 +269,56 @@ export async function getDynamicModel(
|
||||
if (provider.type === "gemini") {
|
||||
// Gemini 图片生成使用 chat() 方法(与 local-qwen 相同的调用方式)
|
||||
console.log("[getDynamicModel] Getting Gemini chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(model.modelId);
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Gemini model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
console.log("[getDynamicModel] Gemini model type:", typeof baseModel); // 调试
|
||||
console.log("[getDynamicModel] Gemini model keys:", baseModel ? Object.keys(baseModel).join(', ') : 'N/A'); // 调试
|
||||
} else if (provider.type === "local") {
|
||||
// 本地模型使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting local chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(model.modelId);
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Local model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "qianwen-video") {
|
||||
// 千问视频生成使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Qianwen Video chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(model.modelId);
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Qianwen Video model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "fal-video") {
|
||||
// FAL 视频生成使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Fal Video chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(model.modelId);
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Fal Video model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "volcano-vision") {
|
||||
// 火山引擎视觉生成使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Volcano Vision chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(model.modelId);
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Volcano Vision model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "alibaba-image") {
|
||||
// 阿里云千问图片生成使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Alibaba Image chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(model.modelId);
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Alibaba Image model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "ruxa-image") {
|
||||
// RUXA 图片编辑生成使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Ruxa Image chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(model.modelId);
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Ruxa Image model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "fal-image") {
|
||||
// FAL 图片编辑生成使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Fal Image chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(model.modelId);
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Fal Image model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "huanke-gemini") {
|
||||
// 环科 Gemini 原生格式使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Huanke Gemini chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(model.modelId);
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Huanke Gemini model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
console.log("[getDynamicModel] Huanke Gemini model type:", typeof baseModel); // 调试
|
||||
console.log("[getDynamicModel] Huanke Gemini model keys:", baseModel ? Object.keys(baseModel).join(', ') : 'N/A'); // 调试
|
||||
} else {
|
||||
// OpenAI-compatible providers
|
||||
console.log("[getDynamicModel] Getting OpenAI-compatible model..."); // 调试日志
|
||||
baseModel = (providerInstance as any).chat(model.modelId);
|
||||
baseModel = (providerInstance as any).chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] OpenAI model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
}
|
||||
|
||||
@ -340,9 +343,10 @@ export async function getDynamicModel(
|
||||
|
||||
// 通过模型的数据库 ID 获取语言模型实例
|
||||
export async function getLanguageModelById(
|
||||
modelId: string
|
||||
modelId: string,
|
||||
overrideModelId?: string | null
|
||||
): Promise<LanguageModel | null> {
|
||||
return getDynamicModel(modelId);
|
||||
return getDynamicModel(modelId, overrideModelId);
|
||||
}
|
||||
|
||||
// 获取所有可用的模型列表(用于前端展示)
|
||||
|
||||
@ -544,6 +544,7 @@ export async function optimizePromptWithLLM(params: {
|
||||
getEnabledAiModelsWithProvider,
|
||||
getAgentAiModelWithProvider,
|
||||
} = await import("@/lib/db/queries");
|
||||
const { getAiModelAgentCodingModelId } = await import("@/lib/db/ai-model-flags");
|
||||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||||
|
||||
// 提示词优化优先使用 Agent 模型(建议智普),失败时回退
|
||||
@ -562,13 +563,22 @@ export async function optimizePromptWithLLM(params: {
|
||||
(m) => m.provider.type === "alibaba" && m.model.supportsVision
|
||||
);
|
||||
const selectedOptimizerModel = usableAgentModel || zhipuModel || qwenModel;
|
||||
const agentCodingModelId =
|
||||
selectedOptimizerModel &&
|
||||
usableAgentModel &&
|
||||
selectedOptimizerModel.model.id === usableAgentModel.model.id
|
||||
? getAiModelAgentCodingModelId(usableAgentModel.model)
|
||||
: null;
|
||||
|
||||
if (!selectedOptimizerModel) {
|
||||
console.log("[Prompt Optimizer LLM] No optimizer model available, falling back to template");
|
||||
return optimizePrompt({ originalPrompt, intent, subIntent });
|
||||
}
|
||||
|
||||
const model = await getLanguageModelById(selectedOptimizerModel.model.id);
|
||||
const model = await getLanguageModelById(
|
||||
selectedOptimizerModel.model.id,
|
||||
agentCodingModelId
|
||||
);
|
||||
if (!model) {
|
||||
console.log("[Prompt Optimizer LLM] Failed to get model, falling back to template");
|
||||
return optimizePrompt({ originalPrompt, intent, subIntent });
|
||||
@ -598,7 +608,8 @@ export async function optimizePromptWithLLM(params: {
|
||||
console.log(
|
||||
"[Prompt Optimizer LLM] Calling optimizer model:",
|
||||
selectedOptimizerModel.model.displayName,
|
||||
`(${selectedOptimizerModel.provider.type})`
|
||||
`(${selectedOptimizerModel.provider.type})`,
|
||||
agentCodingModelId ? `override=${agentCodingModelId}` : ""
|
||||
);
|
||||
|
||||
const { text: responseText } = await generateText({
|
||||
|
||||
117
lib/ai/vision-inline.ts
Normal file
117
lib/ai/vision-inline.ts
Normal file
@ -0,0 +1,117 @@
|
||||
import sharp from "sharp";
|
||||
|
||||
type InlineVisionOptions = {
|
||||
maxImages?: number;
|
||||
maxSide?: number;
|
||||
timeoutMs?: number;
|
||||
retries?: number;
|
||||
loggerPrefix?: string;
|
||||
jpegQuality?: number;
|
||||
};
|
||||
|
||||
type InlineVisionResult = {
|
||||
inlineDataUrls: string[];
|
||||
skipped: Array<{ url: string; reason: string }>;
|
||||
};
|
||||
|
||||
const DEFAULT_MAX_IMAGES = 6;
|
||||
const DEFAULT_MAX_SIDE = 900;
|
||||
const DEFAULT_TIMEOUT_MS = 12000;
|
||||
const DEFAULT_RETRIES = 2;
|
||||
const DEFAULT_JPEG_QUALITY = 82;
|
||||
|
||||
async function fetchImageWithRetry(url: string, timeoutMs: number, retries: number) {
|
||||
let lastError: unknown;
|
||||
|
||||
for (let attempt = 1; attempt <= retries; attempt++) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
return Buffer.from(arrayBuffer);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (attempt < retries) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 250 * attempt));
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError instanceof Error ? lastError : new Error("图片拉取失败");
|
||||
}
|
||||
|
||||
async function downscaleForVision(
|
||||
inputBuffer: Buffer,
|
||||
maxSide: number,
|
||||
jpegQuality: number,
|
||||
) {
|
||||
const image = sharp(inputBuffer, { failOn: "none" });
|
||||
const metadata = await image.metadata();
|
||||
const hasAlpha = Boolean(metadata.hasAlpha);
|
||||
|
||||
const resized = image
|
||||
.rotate()
|
||||
.resize({
|
||||
width: maxSide,
|
||||
height: maxSide,
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
});
|
||||
|
||||
if (hasAlpha) {
|
||||
const output = await resized.png({ compressionLevel: 9 }).toBuffer();
|
||||
return { mime: "image/png", buffer: output };
|
||||
}
|
||||
|
||||
const output = await resized.jpeg({ quality: jpegQuality, mozjpeg: true }).toBuffer();
|
||||
return { mime: "image/jpeg", buffer: output };
|
||||
}
|
||||
|
||||
export async function inlineVisionImages(
|
||||
urls: string[],
|
||||
options: InlineVisionOptions = {},
|
||||
): Promise<InlineVisionResult> {
|
||||
const maxImages = Math.max(1, options.maxImages ?? DEFAULT_MAX_IMAGES);
|
||||
const maxSide = Math.max(320, options.maxSide ?? DEFAULT_MAX_SIDE);
|
||||
const timeoutMs = Math.max(3000, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||
const retries = Math.max(1, options.retries ?? DEFAULT_RETRIES);
|
||||
const jpegQuality = Math.max(55, Math.min(95, options.jpegQuality ?? DEFAULT_JPEG_QUALITY));
|
||||
const loggerPrefix = options.loggerPrefix ?? "[Vision Inline]";
|
||||
|
||||
const inlineDataUrls: string[] = [];
|
||||
const skipped: Array<{ url: string; reason: string }> = [];
|
||||
|
||||
const candidates = urls
|
||||
.map((item) => String(item || "").trim())
|
||||
.filter((url) => /^https?:\/\//i.test(url))
|
||||
.slice(0, maxImages);
|
||||
|
||||
for (const url of candidates) {
|
||||
try {
|
||||
const buffer = await fetchImageWithRetry(url, timeoutMs, retries);
|
||||
const processed = await downscaleForVision(buffer, maxSide, jpegQuality);
|
||||
const dataUrl = `data:${processed.mime};base64,${processed.buffer.toString("base64")}`;
|
||||
inlineDataUrls.push(dataUrl);
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : "unknown_error";
|
||||
skipped.push({ url, reason });
|
||||
console.warn(`${loggerPrefix} skip reference image:`, {
|
||||
url,
|
||||
reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { inlineDataUrls, skipped };
|
||||
}
|
||||
@ -1,5 +1,6 @@
|
||||
const AGENT_AUTO_MODEL_KEY = "agentAutoModel";
|
||||
const MULTIMODAL_MODEL_KEY = "multimodalModel";
|
||||
const AGENT_CODING_MODEL_ID_KEY = "agentCodingModelId";
|
||||
|
||||
function toRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
@ -62,6 +63,43 @@ export function withAgentAutoModelFlag(
|
||||
return Object.keys(next).length > 0 ? next : null;
|
||||
}
|
||||
|
||||
export function getAiModelAgentCodingModelId(input: {
|
||||
defaultParams?: unknown;
|
||||
capabilities?: unknown;
|
||||
}): string | null {
|
||||
const defaultParams = toRecord(input.defaultParams);
|
||||
const defaultValue = defaultParams?.[AGENT_CODING_MODEL_ID_KEY];
|
||||
if (typeof defaultValue === "string") {
|
||||
const normalized = defaultValue.trim();
|
||||
if (normalized) return normalized;
|
||||
}
|
||||
|
||||
const capabilities = toRecord(input.capabilities);
|
||||
const capabilityValue = capabilities?.[AGENT_CODING_MODEL_ID_KEY];
|
||||
if (typeof capabilityValue === "string") {
|
||||
const normalized = capabilityValue.trim();
|
||||
if (normalized) return normalized;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function withAgentCodingModelId(
|
||||
defaultParams: unknown,
|
||||
codingModelId?: string | null
|
||||
): Record<string, unknown> | null {
|
||||
const next = { ...(toRecord(defaultParams) || {}) };
|
||||
const normalized = String(codingModelId || "").trim();
|
||||
|
||||
if (normalized) {
|
||||
next[AGENT_CODING_MODEL_ID_KEY] = normalized;
|
||||
} else {
|
||||
delete next[AGENT_CODING_MODEL_ID_KEY];
|
||||
}
|
||||
|
||||
return Object.keys(next).length > 0 ? next : null;
|
||||
}
|
||||
|
||||
export function isAiModelMultimodal(input: {
|
||||
capabilities?: unknown;
|
||||
defaultParams?: unknown;
|
||||
|
||||
@ -57,6 +57,7 @@ import { generateHashedPassword } from "./utils";
|
||||
import {
|
||||
isAiModelAgentAuto,
|
||||
withAgentAutoModelFlag,
|
||||
withAgentCodingModelId,
|
||||
withMultimodalModelFlag,
|
||||
} from "./ai-model-flags";
|
||||
import {
|
||||
@ -1440,6 +1441,7 @@ export async function createAiModel({
|
||||
imageGenConfig,
|
||||
isAgentModel,
|
||||
isMultimodal,
|
||||
agentCodingModelId,
|
||||
}: {
|
||||
providerId: string;
|
||||
modelId: string;
|
||||
@ -1457,6 +1459,7 @@ export async function createAiModel({
|
||||
imageGenConfig?: ImageGenConfig;
|
||||
isAgentModel?: boolean;
|
||||
isMultimodal?: boolean;
|
||||
agentCodingModelId?: string;
|
||||
}): Promise<AiModel[]> {
|
||||
try {
|
||||
// 如果设置为默认,先取消其他默认
|
||||
@ -1467,6 +1470,16 @@ export async function createAiModel({
|
||||
.where(eq(aiModel.isDefault, true));
|
||||
}
|
||||
|
||||
const defaultParams =
|
||||
agentCodingModelId !== undefined
|
||||
? withAgentCodingModelId(
|
||||
isAgentModel === undefined ? null : withAgentAutoModelFlag(null, isAgentModel),
|
||||
agentCodingModelId
|
||||
)
|
||||
: isAgentModel === undefined
|
||||
? null
|
||||
: withAgentAutoModelFlag(null, isAgentModel);
|
||||
|
||||
return await db
|
||||
.insert(aiModel)
|
||||
.values({
|
||||
@ -1484,10 +1497,7 @@ export async function createAiModel({
|
||||
fallbackModelId1: fallbackModelId1 || null,
|
||||
fallbackModelId2: fallbackModelId2 || null,
|
||||
imageGenConfig: imageGenConfig || null,
|
||||
defaultParams:
|
||||
isAgentModel === undefined
|
||||
? null
|
||||
: withAgentAutoModelFlag(null, isAgentModel),
|
||||
defaultParams,
|
||||
capabilities:
|
||||
isMultimodal === undefined
|
||||
? null
|
||||
@ -1520,6 +1530,7 @@ export async function updateAiModel({
|
||||
imageGenConfig,
|
||||
isAgentModel,
|
||||
isMultimodal,
|
||||
agentCodingModelId,
|
||||
}: {
|
||||
id: string;
|
||||
providerId?: string;
|
||||
@ -1538,6 +1549,7 @@ export async function updateAiModel({
|
||||
imageGenConfig?: ImageGenConfig;
|
||||
isAgentModel?: boolean;
|
||||
isMultimodal?: boolean;
|
||||
agentCodingModelId?: string;
|
||||
}): Promise<AiModel[]> {
|
||||
try {
|
||||
// 如果设置为默认,先取消其他默认
|
||||
@ -1566,7 +1578,11 @@ export async function updateAiModel({
|
||||
if (fallbackModelId2 !== undefined) updateData.fallbackModelId2 = fallbackModelId2 || null;
|
||||
// 🆕 图片生成配置
|
||||
if (imageGenConfig !== undefined) updateData.imageGenConfig = imageGenConfig || null;
|
||||
if (isAgentModel !== undefined || isMultimodal !== undefined) {
|
||||
if (
|
||||
isAgentModel !== undefined ||
|
||||
isMultimodal !== undefined ||
|
||||
agentCodingModelId !== undefined
|
||||
) {
|
||||
const [existingModel] = await db
|
||||
.select({
|
||||
defaultParams: aiModel.defaultParams,
|
||||
@ -1580,12 +1596,22 @@ export async function updateAiModel({
|
||||
return [];
|
||||
}
|
||||
|
||||
let nextDefaultParams = existingModel.defaultParams;
|
||||
if (isAgentModel !== undefined) {
|
||||
updateData.defaultParams = withAgentAutoModelFlag(
|
||||
existingModel.defaultParams,
|
||||
nextDefaultParams = withAgentAutoModelFlag(
|
||||
nextDefaultParams,
|
||||
isAgentModel
|
||||
);
|
||||
}
|
||||
if (agentCodingModelId !== undefined) {
|
||||
nextDefaultParams = withAgentCodingModelId(
|
||||
nextDefaultParams,
|
||||
agentCodingModelId
|
||||
);
|
||||
}
|
||||
if (isAgentModel !== undefined || agentCodingModelId !== undefined) {
|
||||
updateData.defaultParams = nextDefaultParams;
|
||||
}
|
||||
|
||||
if (isMultimodal !== undefined) {
|
||||
updateData.capabilities = withMultimodalModelFlag(
|
||||
|
||||
@ -77,6 +77,9 @@ export const rateLimiters = {
|
||||
// 通用API限制:每分钟100次请求
|
||||
api: new RateLimiter(60000, 100),
|
||||
|
||||
// 高频只读接口(首页/画布轮询):每分钟1200次请求
|
||||
uiPolling: new RateLimiter(60000, 1200),
|
||||
|
||||
// 画布保存接口:每分钟300次请求(含自动保存)
|
||||
canvas: new RateLimiter(60000, 300),
|
||||
|
||||
|
||||
@ -4,6 +4,17 @@ import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
|
||||
import { put } from "@vercel/blob";
|
||||
import { getDefaultStorageConfig } from "@/lib/db/queries";
|
||||
|
||||
type S3UploadConfig = {
|
||||
endpoint: string;
|
||||
region: string;
|
||||
bucket: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
publicUrl?: string;
|
||||
};
|
||||
|
||||
let lastKnownGoodDbS3Config: S3UploadConfig | null = null;
|
||||
|
||||
// 获取文件的 Content-Type
|
||||
function getContentType(filename: string): string {
|
||||
const ext = filename.toLowerCase().split(".").pop();
|
||||
@ -28,14 +39,7 @@ function getContentType(filename: string): string {
|
||||
async function uploadToS3(
|
||||
filename: string,
|
||||
buffer: ArrayBuffer,
|
||||
config: {
|
||||
endpoint: string;
|
||||
region: string;
|
||||
bucket: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
publicUrl?: string;
|
||||
}
|
||||
config: S3UploadConfig
|
||||
) {
|
||||
const { endpoint, region, bucket, accessKeyId, secretAccessKey, publicUrl } = config;
|
||||
|
||||
@ -101,6 +105,49 @@ async function uploadToS3(
|
||||
}
|
||||
}
|
||||
|
||||
function toDbS3UploadConfig(raw: unknown): S3UploadConfig | null {
|
||||
const value = raw as Record<string, unknown> | null | undefined;
|
||||
if (
|
||||
!value ||
|
||||
value.type !== "s3" ||
|
||||
!value.isEnabled ||
|
||||
!value.endpoint ||
|
||||
!value.bucket ||
|
||||
!value.accessKeyId ||
|
||||
!value.secretAccessKey
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
endpoint: String(value.endpoint),
|
||||
region: String(value.region || "auto"),
|
||||
bucket: String(value.bucket),
|
||||
accessKeyId: String(value.accessKeyId),
|
||||
secretAccessKey: String(value.secretAccessKey),
|
||||
publicUrl: value.publicUrl ? String(value.publicUrl) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function buildEnvS3Config(): S3UploadConfig | null {
|
||||
if (
|
||||
process.env.S3_ENDPOINT &&
|
||||
process.env.S3_BUCKET &&
|
||||
process.env.S3_ACCESS_KEY_ID &&
|
||||
process.env.S3_SECRET_ACCESS_KEY
|
||||
) {
|
||||
return {
|
||||
endpoint: process.env.S3_ENDPOINT,
|
||||
region: process.env.S3_REGION ?? "auto",
|
||||
bucket: process.env.S3_BUCKET,
|
||||
accessKeyId: process.env.S3_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
|
||||
publicUrl: process.env.S3_PUBLIC_URL,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// 上传文件到存储服务
|
||||
export async function uploadFile(
|
||||
filename: string,
|
||||
@ -108,55 +155,41 @@ export async function uploadFile(
|
||||
): Promise<{ url: string; pathname: string; contentType: string }> {
|
||||
console.log(`[Upload] Starting upload for file: ${filename}`);
|
||||
|
||||
let dbS3Config: S3UploadConfig | null = null;
|
||||
try {
|
||||
// 尝试从数据库获取存储配置
|
||||
const storageConfig = await getDefaultStorageConfig();
|
||||
|
||||
if (
|
||||
storageConfig &&
|
||||
storageConfig.type === "s3" &&
|
||||
storageConfig.endpoint &&
|
||||
storageConfig.bucket &&
|
||||
storageConfig.accessKeyId &&
|
||||
storageConfig.secretAccessKey &&
|
||||
storageConfig.isEnabled
|
||||
) {
|
||||
console.log(`[Upload] Using database S3 config: ${storageConfig.name}`);
|
||||
console.log(`[Upload] Endpoint: ${storageConfig.endpoint}`);
|
||||
console.log(`[Upload] Bucket: ${storageConfig.bucket}`);
|
||||
|
||||
// 使用数据库配置的 S3/MinIO 存储
|
||||
return await uploadToS3(filename, buffer, {
|
||||
endpoint: storageConfig.endpoint,
|
||||
region: storageConfig.region ?? "auto",
|
||||
bucket: storageConfig.bucket,
|
||||
accessKeyId: storageConfig.accessKeyId,
|
||||
secretAccessKey: storageConfig.secretAccessKey,
|
||||
publicUrl: storageConfig.publicUrl ?? undefined,
|
||||
});
|
||||
} else {
|
||||
console.warn("[Upload] No valid S3 config in database");
|
||||
dbS3Config = toDbS3UploadConfig(storageConfig);
|
||||
if (dbS3Config) {
|
||||
console.log(`[Upload] Using database S3 config: ${storageConfig?.name}`);
|
||||
console.log(`[Upload] Endpoint: ${dbS3Config.endpoint}`);
|
||||
console.log(`[Upload] Bucket: ${dbS3Config.bucket}`);
|
||||
const uploaded = await uploadToS3(filename, buffer, dbS3Config);
|
||||
lastKnownGoodDbS3Config = dbS3Config;
|
||||
return uploaded;
|
||||
}
|
||||
console.warn("[Upload] No valid S3 config in database");
|
||||
} catch (error) {
|
||||
console.error("[Upload] Failed to get storage config from database:", error);
|
||||
console.error("[Upload] Database storage path failed:", error);
|
||||
if (lastKnownGoodDbS3Config) {
|
||||
try {
|
||||
console.log("[Upload] Retrying with last known good database S3 config");
|
||||
return await uploadToS3(filename, buffer, lastKnownGoodDbS3Config);
|
||||
} catch (retryError) {
|
||||
console.error("[Upload] Last known S3 config upload failed:", retryError);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 回退到环境变量配置的 S3
|
||||
if (
|
||||
process.env.S3_ENDPOINT &&
|
||||
process.env.S3_BUCKET &&
|
||||
process.env.S3_ACCESS_KEY_ID &&
|
||||
process.env.S3_SECRET_ACCESS_KEY
|
||||
) {
|
||||
const envS3Config = buildEnvS3Config();
|
||||
if (envS3Config) {
|
||||
console.log("[Upload] Using environment variable S3 config");
|
||||
return await uploadToS3(filename, buffer, {
|
||||
endpoint: process.env.S3_ENDPOINT,
|
||||
region: process.env.S3_REGION ?? "auto",
|
||||
bucket: process.env.S3_BUCKET,
|
||||
accessKeyId: process.env.S3_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
|
||||
publicUrl: process.env.S3_PUBLIC_URL,
|
||||
});
|
||||
return await uploadToS3(filename, buffer, envS3Config);
|
||||
}
|
||||
|
||||
if (!process.env.BLOB_READ_WRITE_TOKEN) {
|
||||
throw new Error(
|
||||
"No available storage backend: database S3 failed, env S3 missing, and BLOB_READ_WRITE_TOKEN is not configured."
|
||||
);
|
||||
}
|
||||
|
||||
// 最终回退到 Vercel Blob
|
||||
|
||||
33
proxy.ts
33
proxy.ts
@ -4,6 +4,24 @@ import { isBlockedIP, getClientIP } from "@/lib/security/ip-blocklist";
|
||||
import { checkRateLimit, rateLimiters } from "@/lib/security/rate-limiter";
|
||||
import { isSuspiciousRequest, logSuspiciousRequest } from "@/lib/security/request-monitor";
|
||||
|
||||
function isHighFrequencyReadApi(pathname: string, method: string) {
|
||||
if (method !== "GET") return false;
|
||||
if (/^\/api\/canvas\/[^/]+\/detail-archives$/.test(pathname)) return true;
|
||||
|
||||
return [
|
||||
"/api/models",
|
||||
"/api/canvas",
|
||||
"/api/nav-links",
|
||||
"/api/site-settings",
|
||||
"/api/messages",
|
||||
"/api/history",
|
||||
"/api/video-tasks",
|
||||
"/api/tutorial-videos",
|
||||
"/api/user/membership-summary",
|
||||
"/api/user/account-center",
|
||||
].includes(pathname);
|
||||
}
|
||||
|
||||
export async function proxy(request: NextRequest) {
|
||||
const { pathname } = request.nextUrl;
|
||||
|
||||
@ -67,6 +85,21 @@ export async function proxy(request: NextRequest) {
|
||||
{ status: 429 }
|
||||
);
|
||||
}
|
||||
} else if (isHighFrequencyReadApi(pathname, request.method)) {
|
||||
const pollingRateLimit = checkRateLimit(request, rateLimiters.uiPolling);
|
||||
if (!pollingRateLimit.allowed) {
|
||||
console.warn(`[Security] UI polling rate limit exceeded for IP: ${clientIP}`, {
|
||||
pathname,
|
||||
resetTime: new Date(pollingRateLimit.resetTime).toISOString(),
|
||||
});
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "Too many polling requests",
|
||||
retryAfter: Math.ceil((pollingRateLimit.resetTime - Date.now()) / 1000),
|
||||
},
|
||||
{ status: 429 }
|
||||
);
|
||||
}
|
||||
} else if (pathname.startsWith("/api/")) {
|
||||
const rateLimit = checkRateLimit(request, rateLimiters.api);
|
||||
if (!rateLimit.allowed) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user