diff --git a/app/api/agent-workflow/material-search/route.ts b/app/api/agent-workflow/material-search/route.ts index 238737a10..1317cabe1 100644 --- a/app/api/agent-workflow/material-search/route.ts +++ b/app/api/agent-workflow/material-search/route.ts @@ -11,6 +11,7 @@ import { AGENT_WORKFLOW_MATERIAL_API_CONFIG_KEY, normalizeAgentWorkflowMaterialApiConfig, } from "@/lib/agents/workflow-material-config"; +import { analyzeWorkflowMaterialAppearanceFromImage } from "@/lib/agents/workflow-material-appearance"; import { AGENT_WORKFLOW_GOODS_INFO_API_CONFIG_KEY, normalizeAgentWorkflowGoodsInfoApiConfig, @@ -883,6 +884,20 @@ export async function GET(request: NextRequest) { requiredCount, }) : []; + const firstImageAppearanceSummary = await analyzeWorkflowMaterialAppearanceFromImage( + importedImages[0]?.imageUrl || "" + ).catch((error) => { + console.warn("[Agent Workflow Material Search] Appearance analysis failed:", error); + return null; + }); + const annotatedImages = importedImages.map((image, index) => + index === 0 && firstImageAppearanceSummary + ? { + ...image, + appearanceSummary: firstImageAppearanceSummary, + } + : image + ); return NextResponse.json({ success: true, @@ -894,10 +909,10 @@ export async function GET(request: NextRequest) { materialId: String(upstreamResult.data.materialId || "").trim() || null, productModel, zipUrl: String(upstreamResult.data.downloadUrls?.zip || "").trim() || null, - fixedCount: importedImages.filter((item) => item.group === "fixed").length, - otherCount: importedImages.filter((item) => item.group === "other").length, - totalCount: importedImages.length, - images: importedImages, + fixedCount: annotatedImages.filter((item) => item.group === "fixed").length, + otherCount: annotatedImages.filter((item) => item.group === "other").length, + totalCount: annotatedImages.length, + images: annotatedImages, recommendedImageIds, skuOptions: [], goodsInfo: goodsInfo @@ -908,8 +923,13 @@ export async function GET(request: NextRequest) { designer: goodsInfo.designer, buyer: goodsInfo.buyer, boardType: goodsInfo.boardType, + appearanceSummary: firstImageAppearanceSummary, } - : null, + : firstImageAppearanceSummary + ? { + appearanceSummary: firstImageAppearanceSummary, + } + : null, }, }); } catch (error) { diff --git a/app/api/agent-workflow/material-upload/route.ts b/app/api/agent-workflow/material-upload/route.ts index 5de23516e..13afa4f02 100644 --- a/app/api/agent-workflow/material-upload/route.ts +++ b/app/api/agent-workflow/material-upload/route.ts @@ -3,6 +3,7 @@ import { NextRequest, NextResponse } from "next/server"; import { z } from "zod"; import { auth } from "@/app/(auth)/auth"; import { executeRmbgJob, RmbgServiceError } from "@/lib/ai/rmbg/service"; +import { analyzeWorkflowMaterialAppearanceFromImage } from "@/lib/agents/workflow-material-appearance"; import { upsertMediaAsset } from "@/lib/db/queries"; export const runtime = "nodejs"; @@ -109,6 +110,12 @@ export async function POST(request: NextRequest) { const originalFileName = String(file.name || "").trim() || `workflow-upload-${Date.now()}.png`; const originalStem = sanitizeFilenameStem(originalFileName) || `workflow-upload-${Date.now()}`; const displayName = `${originalStem}-rmbg.png`; + const appearanceSummary = await analyzeWorkflowMaterialAppearanceFromImage( + result.original_url + ).catch((error) => { + console.warn("[Agent Workflow Material Upload] Appearance analysis failed:", error); + return null; + }); const asset = await upsertMediaAsset({ userId: session.user.id, mediaType: "image", @@ -147,6 +154,7 @@ export async function POST(request: NextRequest) { imageUrl: result.original_url, assetId: asset?.id ?? null, mimeType: "image/png", + appearanceSummary, sourceType: "upload", group: "other", sortOrder: 0, diff --git a/components/agents/AgentWorkflowMaterialPicker.tsx b/components/agents/AgentWorkflowMaterialPicker.tsx index a67be75fd..9d5cbf580 100644 --- a/components/agents/AgentWorkflowMaterialPicker.tsx +++ b/components/agents/AgentWorkflowMaterialPicker.tsx @@ -135,6 +135,16 @@ export function AgentWorkflowMaterialPicker({ label: "板材类型", value: result?.goodsInfo?.boardType || null, }, + { + label: "外观识别", + value: + result?.images + ?.slice() + .sort((left, right) => left.sortOrder - right.sortOrder)[0]?.appearanceSummary || + result?.goodsInfo?.appearanceSummary || + null, + fullWidth: true, + }, ].filter((item) => Boolean(item.value)); const remainingSelectableCount = resolvedSelectionLimit === null ? null : Math.max(resolvedSelectionLimit - selectedCount, 0); diff --git a/lib/agents/workflow-composer.ts b/lib/agents/workflow-composer.ts index aa9d79f9d..da32b8439 100644 --- a/lib/agents/workflow-composer.ts +++ b/lib/agents/workflow-composer.ts @@ -64,6 +64,24 @@ function buildGoodsInfoSizeConstraintLine(params: { return `${params.label}尺寸比例约束：尺寸信息为 ${sizeInfo}。生成场景时必须依据该尺寸信息判断产品在空间中的真实体量、长宽高关系和与周边家具/人物/通道的比例，不得随意放大、缩小或改变产品结构比例。`; } +function buildGoodsInfoAppearanceConstraintLine(params: { + label: string; + result?: AgentWorkflowMaterialSelections["primary"]["result"] | null; +}) { + const appearanceSummary = String( + params.result?.images?.slice().sort((left, right) => left.sortOrder - right.sortOrder)[0] + ?.appearanceSummary || + params.result?.goodsInfo?.appearanceSummary || + "" + ).trim(); + + if (!appearanceSummary) { + return ""; + } + + return `${params.label}外观结构与颜色约束：首张SKU图识别结果为“${appearanceSummary}”。后续生成必须严格保持该产品的外观轮廓、主体结构、层板/格口/门板/抽屉/支撑件等数量关系和主副色，不得擅自改款、增减结构件、改变开合方式或替换主体颜色。`; +} + function buildGoodsInfoSummaryLine(params: { label: string; result?: AgentWorkflowMaterialSelections["primary"]["result"] | null; @@ -307,6 +325,13 @@ export function buildAgentWorkflowComposerContext(params: { if (primaryGoodsInfoSizeConstraint) { promptLines.push(primaryGoodsInfoSizeConstraint); } + const primaryGoodsInfoAppearanceConstraint = buildGoodsInfoAppearanceConstraintLine({ + label: "主SKU", + result: params.materials?.primary?.result || null, + }); + if (primaryGoodsInfoAppearanceConstraint) { + promptLines.push(primaryGoodsInfoAppearanceConstraint); + } const secondaryGoodsInfoPrompt = buildGoodsInfoPromptLine({ label: "搭配SKU商品信息", result: params.materials?.secondary?.result || null, @@ -321,6 +346,13 @@ export function buildAgentWorkflowComposerContext(params: { if (secondaryGoodsInfoSizeConstraint) { promptLines.push(secondaryGoodsInfoSizeConstraint); } + const secondaryGoodsInfoAppearanceConstraint = buildGoodsInfoAppearanceConstraintLine({ + label: "搭配SKU", + result: params.materials?.secondary?.result || null, + }); + if (secondaryGoodsInfoAppearanceConstraint) { + promptLines.push(secondaryGoodsInfoAppearanceConstraint); + } if (referenceImages.some((item) => item.source === "primary")) { const labels = referenceImages .filter((item) => item.source === "primary") diff --git a/lib/agents/workflow-material-appearance.ts b/lib/agents/workflow-material-appearance.ts new file mode 100644 index 000000000..d7919b43f --- /dev/null +++ b/lib/agents/workflow-material-appearance.ts @@ -0,0 +1,173 @@ +import "server-only"; + +import { generateText } from "ai"; +import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; +import { callQwen3Vl } from "@/lib/ai/qwen3-vl"; +import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; + +const QWEN_VL_MODEL_MATCHER = + /(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i; + +function sanitizeAppearanceSummary(value: unknown, maxLength = 220) { + return String(value || "").replace(/\s+/g, " ").trim().slice(0, maxLength); +} + +function extractJsonObjectText(text: string): string | null { + const normalized = String(text || "").trim(); + if (!normalized) { + return null; + } + + const fencedMatch = normalized.match(/```(?:json)?\s*([\s\S]*?)```/i); + if (fencedMatch?.[1]) { + const fencedText = fencedMatch[1].trim(); + if (fencedText.startsWith("{") && fencedText.endsWith("}")) { + return fencedText; + } + } + + if (normalized.startsWith("{") && normalized.endsWith("}")) { + return normalized; + } + + const firstBraceIndex = normalized.indexOf("{"); + const lastBraceIndex = normalized.lastIndexOf("}"); + if (firstBraceIndex >= 0 && lastBraceIndex > firstBraceIndex) { + return normalized.slice(firstBraceIndex, lastBraceIndex + 1); + } + + return null; +} + +function parseAppearanceSummary(text: string) { + const jsonText = extractJsonObjectText(text); + if (jsonText) { + try { + const parsed = JSON.parse(jsonText) as Record; + const summary = sanitizeAppearanceSummary( + parsed.appearanceSummary || parsed.summary || parsed.description + ); + if (summary) { + return summary; + } + } catch (error) { + console.warn( + "[Workflow Material Appearance] Failed to parse JSON response:", + error + ); + } + } + + return sanitizeAppearanceSummary( + String(text || "") + .replace(/```(?:json)?/gi, "") + .replace(/```/g, "") + ); +} + +function buildAppearancePrompt() { + return [ + "你是电商商品白底图识别器。", + "请只基于商品本体，识别并描述产品的外观结构和颜色，用于后续生图时保持产品结构不要改变。", + "描述重点：1. 产品品类与整体轮廓；2. 关键结构件及数量关系，例如层板、格口、门板、抽屉、腿、扶手、靠背、边框、台面、开孔、开口方向；3. 主色、副色，以及木纹、金属、布艺等肉眼可见材质或纹理。", + "不要描述背景、光影、拍摄角度、水印、文案，也不要输出营销形容词。", + '只输出 JSON：{"appearanceSummary":"..."}', + "appearanceSummary 必须是中文，长度 60 到 120 字，具体、克制、可直接作为生图约束。", + ].join("\n"); +} + +async function analyzeWithConfiguredQwenVisionModel(params: { + imageUrl: string; + prompt: string; +}) { + const configuredModels = await getEnabledAiModelsWithProvider().catch((error) => { + console.warn( + "[Workflow Material Appearance] Failed to load configured models:", + error + ); + return []; + }); + const configuredQwenVisionModel = configuredModels.find( + ({ model }) => + model.supportsVision && + !model.isSubModel && + QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`) + ); + + if (!configuredQwenVisionModel) { + return null; + } + + try { + const languageModel = await getLanguageModelById( + configuredQwenVisionModel.model.id + ); + if (!languageModel) { + return null; + } + + const { text } = await generateText({ + model: languageModel, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: params.prompt, + }, + { + type: "image", + image: params.imageUrl, + }, + ], + }, + ], + temperature: 0, + maxTokens: 220, + abortSignal: AbortSignal.timeout(25_000), + } as any); + + return parseAppearanceSummary(text); + } catch (error) { + console.warn( + "[Workflow Material Appearance] Configured Qwen-VL analysis failed:", + error + ); + return null; + } +} + +export async function analyzeWorkflowMaterialAppearanceFromImage(imageUrl: string) { + const normalizedUrl = String(imageUrl || "").trim(); + if (!/^https?:\/\//i.test(normalizedUrl)) { + return null; + } + + const prompt = buildAppearancePrompt(); + const configuredSummary = await analyzeWithConfiguredQwenVisionModel({ + imageUrl: normalizedUrl, + prompt, + }); + if (configuredSummary) { + return configuredSummary; + } + + try { + const result = await callQwen3Vl({ + imageUrl: normalizedUrl, + prompt, + temperature: 0, + maxTokens: 220, + timeoutMs: 25_000, + retries: 1, + }); + return parseAppearanceSummary(result.text); + } catch (error) { + console.warn( + "[Workflow Material Appearance] Fallback Qwen3-VL analysis failed:", + error + ); + return null; + } +} diff --git a/lib/agents/workflow-material-config.ts b/lib/agents/workflow-material-config.ts index a50351e03..b231f85d7 100644 --- a/lib/agents/workflow-material-config.ts +++ b/lib/agents/workflow-material-config.ts @@ -42,6 +42,7 @@ function normalizeMaterialImage( imageUrl, assetId: sanitizeText(raw.assetId, 128) || null, mimeType: sanitizeText(raw.mimeType, 128) || null, + appearanceSummary: sanitizeText(raw.appearanceSummary, 255) || null, sourceType: raw.sourceType === "upload" ? "upload" : "search", group, sortOrder: Number.isFinite(Number(raw.sortOrder)) @@ -131,6 +132,9 @@ function normalizeMaterialSearchResult(value: unknown): AgentWorkflowMaterialSea designer: sanitizeText((raw.goodsInfo as Record).designer, 128) || null, buyer: sanitizeText((raw.goodsInfo as Record).buyer, 128) || null, boardType: sanitizeText((raw.goodsInfo as Record).boardType, 128) || null, + appearanceSummary: + sanitizeText((raw.goodsInfo as Record).appearanceSummary, 255) || + null, } : null, }; diff --git a/types/agent.ts b/types/agent.ts index a66ece22d..958e00486 100644 --- a/types/agent.ts +++ b/types/agent.ts @@ -208,6 +208,7 @@ export interface AgentWorkflowMaterialImage { imageUrl: string; assetId?: string | null; mimeType?: string | null; + appearanceSummary?: string | null; sourceType?: "search" | "upload"; group: "fixed" | "other"; sortOrder: number; @@ -243,6 +244,7 @@ export interface AgentWorkflowMaterialSearchResult { designer?: string | null; buyer?: string | null; boardType?: string | null; + appearanceSummary?: string | null; } | null; }