feat: add SKU appearance constraints via Qwen VL

app/api/agent-workflow/material-search/route.ts

@@ -11,6 +11,7 @@ import {
   AGENT_WORKFLOW_MATERIAL_API_CONFIG_KEY,
   normalizeAgentWorkflowMaterialApiConfig,
 } from "@/lib/agents/workflow-material-config";
+import { analyzeWorkflowMaterialAppearanceFromImage } from "@/lib/agents/workflow-material-appearance";
 import {
   AGENT_WORKFLOW_GOODS_INFO_API_CONFIG_KEY,
   normalizeAgentWorkflowGoodsInfoApiConfig,
@@ -883,6 +884,20 @@ export async function GET(request: NextRequest) {
           requiredCount,
         })
       : [];
+    const firstImageAppearanceSummary = await analyzeWorkflowMaterialAppearanceFromImage(
+      importedImages[0]?.imageUrl || ""
+    ).catch((error) => {
+      console.warn("[Agent Workflow Material Search] Appearance analysis failed:", error);
+      return null;
+    });
+    const annotatedImages = importedImages.map((image, index) =>
+      index === 0 && firstImageAppearanceSummary
+        ? {
+            ...image,
+            appearanceSummary: firstImageAppearanceSummary,
+          }
+        : image
+    );
 
     return NextResponse.json({
       success: true,
@@ -894,10 +909,10 @@ export async function GET(request: NextRequest) {
         materialId: String(upstreamResult.data.materialId || "").trim() || null,
         productModel,
         zipUrl: String(upstreamResult.data.downloadUrls?.zip || "").trim() || null,
-        fixedCount: importedImages.filter((item) => item.group === "fixed").length,
-        otherCount: importedImages.filter((item) => item.group === "other").length,
-        totalCount: importedImages.length,
-        images: importedImages,
+        fixedCount: annotatedImages.filter((item) => item.group === "fixed").length,
+        otherCount: annotatedImages.filter((item) => item.group === "other").length,
+        totalCount: annotatedImages.length,
+        images: annotatedImages,
         recommendedImageIds,
         skuOptions: [],
         goodsInfo: goodsInfo
@@ -908,8 +923,13 @@ export async function GET(request: NextRequest) {
               designer: goodsInfo.designer,
               buyer: goodsInfo.buyer,
               boardType: goodsInfo.boardType,
+              appearanceSummary: firstImageAppearanceSummary,
             }
-          : null,
+          : firstImageAppearanceSummary
+            ? {
+                appearanceSummary: firstImageAppearanceSummary,
+              }
+            : null,
       },
     });
   } catch (error) {

app/api/agent-workflow/material-upload/route.ts

@@ -3,6 +3,7 @@ import { NextRequest, NextResponse } from "next/server";
 import { z } from "zod";
 import { auth } from "@/app/(auth)/auth";
 import { executeRmbgJob, RmbgServiceError } from "@/lib/ai/rmbg/service";
+import { analyzeWorkflowMaterialAppearanceFromImage } from "@/lib/agents/workflow-material-appearance";
 import { upsertMediaAsset } from "@/lib/db/queries";
 
 export const runtime = "nodejs";
@@ -109,6 +110,12 @@ export async function POST(request: NextRequest) {
     const originalFileName = String(file.name || "").trim() || `workflow-upload-${Date.now()}.png`;
     const originalStem = sanitizeFilenameStem(originalFileName) || `workflow-upload-${Date.now()}`;
     const displayName = `${originalStem}-rmbg.png`;
+    const appearanceSummary = await analyzeWorkflowMaterialAppearanceFromImage(
+      result.original_url
+    ).catch((error) => {
+      console.warn("[Agent Workflow Material Upload] Appearance analysis failed:", error);
+      return null;
+    });
     const asset = await upsertMediaAsset({
       userId: session.user.id,
       mediaType: "image",
@@ -147,6 +154,7 @@ export async function POST(request: NextRequest) {
         imageUrl: result.original_url,
         assetId: asset?.id ?? null,
         mimeType: "image/png",
+        appearanceSummary,
         sourceType: "upload",
         group: "other",
         sortOrder: 0,

components/agents/AgentWorkflowMaterialPicker.tsx

@@ -135,6 +135,16 @@ export function AgentWorkflowMaterialPicker({
       label: "板材类型",
       value: result?.goodsInfo?.boardType || null,
     },
+    {
+      label: "外观识别",
+      value:
+        result?.images
+          ?.slice()
+          .sort((left, right) => left.sortOrder - right.sortOrder)[0]?.appearanceSummary ||
+        result?.goodsInfo?.appearanceSummary ||
+        null,
+      fullWidth: true,
+    },
   ].filter((item) => Boolean(item.value));
   const remainingSelectableCount =
     resolvedSelectionLimit === null ? null : Math.max(resolvedSelectionLimit - selectedCount, 0);

lib/agents/workflow-composer.ts

@@ -64,6 +64,24 @@ function buildGoodsInfoSizeConstraintLine(params: {
   return `${params.label}尺寸比例约束：尺寸信息为 ${sizeInfo}。生成场景时必须依据该尺寸信息判断产品在空间中的真实体量、长宽高关系和与周边家具/人物/通道的比例，不得随意放大、缩小或改变产品结构比例。`;
 }
 
+function buildGoodsInfoAppearanceConstraintLine(params: {
+  label: string;
+  result?: AgentWorkflowMaterialSelections["primary"]["result"] | null;
+}) {
+  const appearanceSummary = String(
+    params.result?.images?.slice().sort((left, right) => left.sortOrder - right.sortOrder)[0]
+      ?.appearanceSummary ||
+      params.result?.goodsInfo?.appearanceSummary ||
+      ""
+  ).trim();
+
+  if (!appearanceSummary) {
+    return "";
+  }
+
+  return `${params.label}外观结构与颜色约束：首张SKU图识别结果为“${appearanceSummary}”。后续生成必须严格保持该产品的外观轮廓、主体结构、层板/格口/门板/抽屉/支撑件等数量关系和主副色，不得擅自改款、增减结构件、改变开合方式或替换主体颜色。`;
+}
+
 function buildGoodsInfoSummaryLine(params: {
   label: string;
   result?: AgentWorkflowMaterialSelections["primary"]["result"] | null;
@@ -307,6 +325,13 @@ export function buildAgentWorkflowComposerContext(params: {
   if (primaryGoodsInfoSizeConstraint) {
     promptLines.push(primaryGoodsInfoSizeConstraint);
   }
+  const primaryGoodsInfoAppearanceConstraint = buildGoodsInfoAppearanceConstraintLine({
+    label: "主SKU",
+    result: params.materials?.primary?.result || null,
+  });
+  if (primaryGoodsInfoAppearanceConstraint) {
+    promptLines.push(primaryGoodsInfoAppearanceConstraint);
+  }
   const secondaryGoodsInfoPrompt = buildGoodsInfoPromptLine({
     label: "搭配SKU商品信息",
     result: params.materials?.secondary?.result || null,
@@ -321,6 +346,13 @@ export function buildAgentWorkflowComposerContext(params: {
   if (secondaryGoodsInfoSizeConstraint) {
     promptLines.push(secondaryGoodsInfoSizeConstraint);
   }
+  const secondaryGoodsInfoAppearanceConstraint = buildGoodsInfoAppearanceConstraintLine({
+    label: "搭配SKU",
+    result: params.materials?.secondary?.result || null,
+  });
+  if (secondaryGoodsInfoAppearanceConstraint) {
+    promptLines.push(secondaryGoodsInfoAppearanceConstraint);
+  }
   if (referenceImages.some((item) => item.source === "primary")) {
     const labels = referenceImages
       .filter((item) => item.source === "primary")

lib/agents/workflow-material-appearance.ts

@@ -0,0 +1,173 @@
+import "server-only";
+
+import { generateText } from "ai";
+import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
+import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
+import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
+
+const QWEN_VL_MODEL_MATCHER =
+  /(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i;
+
+function sanitizeAppearanceSummary(value: unknown, maxLength = 220) {
+  return String(value || "").replace(/\s+/g, " ").trim().slice(0, maxLength);
+}
+
+function extractJsonObjectText(text: string): string | null {
+  const normalized = String(text || "").trim();
+  if (!normalized) {
+    return null;
+  }
+
+  const fencedMatch = normalized.match(/```(?:json)?\s*([\s\S]*?)```/i);
+  if (fencedMatch?.[1]) {
+    const fencedText = fencedMatch[1].trim();
+    if (fencedText.startsWith("{") && fencedText.endsWith("}")) {
+      return fencedText;
+    }
+  }
+
+  if (normalized.startsWith("{") && normalized.endsWith("}")) {
+    return normalized;
+  }
+
+  const firstBraceIndex = normalized.indexOf("{");
+  const lastBraceIndex = normalized.lastIndexOf("}");
+  if (firstBraceIndex >= 0 && lastBraceIndex > firstBraceIndex) {
+    return normalized.slice(firstBraceIndex, lastBraceIndex + 1);
+  }
+
+  return null;
+}
+
+function parseAppearanceSummary(text: string) {
+  const jsonText = extractJsonObjectText(text);
+  if (jsonText) {
+    try {
+      const parsed = JSON.parse(jsonText) as Record<string, unknown>;
+      const summary = sanitizeAppearanceSummary(
+        parsed.appearanceSummary || parsed.summary || parsed.description
+      );
+      if (summary) {
+        return summary;
+      }
+    } catch (error) {
+      console.warn(
+        "[Workflow Material Appearance] Failed to parse JSON response:",
+        error
+      );
+    }
+  }
+
+  return sanitizeAppearanceSummary(
+    String(text || "")
+      .replace(/```(?:json)?/gi, "")
+      .replace(/```/g, "")
+  );
+}
+
+function buildAppearancePrompt() {
+  return [
+    "你是电商商品白底图识别器。",
+    "请只基于商品本体，识别并描述产品的外观结构和颜色，用于后续生图时保持产品结构不要改变。",
+    "描述重点：1. 产品品类与整体轮廓；2. 关键结构件及数量关系，例如层板、格口、门板、抽屉、腿、扶手、靠背、边框、台面、开孔、开口方向；3. 主色、副色，以及木纹、金属、布艺等肉眼可见材质或纹理。",
+    "不要描述背景、光影、拍摄角度、水印、文案，也不要输出营销形容词。",
+    '只输出 JSON：{"appearanceSummary":"..."}',
+    "appearanceSummary 必须是中文，长度 60 到 120 字，具体、克制、可直接作为生图约束。",
+  ].join("\n");
+}
+
+async function analyzeWithConfiguredQwenVisionModel(params: {
+  imageUrl: string;
+  prompt: string;
+}) {
+  const configuredModels = await getEnabledAiModelsWithProvider().catch((error) => {
+    console.warn(
+      "[Workflow Material Appearance] Failed to load configured models:",
+      error
+    );
+    return [];
+  });
+  const configuredQwenVisionModel = configuredModels.find(
+    ({ model }) =>
+      model.supportsVision &&
+      !model.isSubModel &&
+      QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`)
+  );
+
+  if (!configuredQwenVisionModel) {
+    return null;
+  }
+
+  try {
+    const languageModel = await getLanguageModelById(
+      configuredQwenVisionModel.model.id
+    );
+    if (!languageModel) {
+      return null;
+    }
+
+    const { text } = await generateText({
+      model: languageModel,
+      messages: [
+        {
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: params.prompt,
+            },
+            {
+              type: "image",
+              image: params.imageUrl,
+            },
+          ],
+        },
+      ],
+      temperature: 0,
+      maxTokens: 220,
+      abortSignal: AbortSignal.timeout(25_000),
+    } as any);
+
+    return parseAppearanceSummary(text);
+  } catch (error) {
+    console.warn(
+      "[Workflow Material Appearance] Configured Qwen-VL analysis failed:",
+      error
+    );
+    return null;
+  }
+}
+
+export async function analyzeWorkflowMaterialAppearanceFromImage(imageUrl: string) {
+  const normalizedUrl = String(imageUrl || "").trim();
+  if (!/^https?:\/\//i.test(normalizedUrl)) {
+    return null;
+  }
+
+  const prompt = buildAppearancePrompt();
+  const configuredSummary = await analyzeWithConfiguredQwenVisionModel({
+    imageUrl: normalizedUrl,
+    prompt,
+  });
+  if (configuredSummary) {
+    return configuredSummary;
+  }
+
+  try {
+    const result = await callQwen3Vl({
+      imageUrl: normalizedUrl,
+      prompt,
+      temperature: 0,
+      maxTokens: 220,
+      timeoutMs: 25_000,
+      retries: 1,
+    });
+    return parseAppearanceSummary(result.text);
+  } catch (error) {
+    console.warn(
+      "[Workflow Material Appearance] Fallback Qwen3-VL analysis failed:",
+      error
+    );
+    return null;
+  }
+}

lib/agents/workflow-material-config.ts

@@ -42,6 +42,7 @@ function normalizeMaterialImage(
     imageUrl,
     assetId: sanitizeText(raw.assetId, 128) || null,
     mimeType: sanitizeText(raw.mimeType, 128) || null,
+    appearanceSummary: sanitizeText(raw.appearanceSummary, 255) || null,
     sourceType: raw.sourceType === "upload" ? "upload" : "search",
     group,
     sortOrder: Number.isFinite(Number(raw.sortOrder))
@@ -131,6 +132,9 @@ function normalizeMaterialSearchResult(value: unknown): AgentWorkflowMaterialSea
             designer: sanitizeText((raw.goodsInfo as Record<string, unknown>).designer, 128) || null,
             buyer: sanitizeText((raw.goodsInfo as Record<string, unknown>).buyer, 128) || null,
             boardType: sanitizeText((raw.goodsInfo as Record<string, unknown>).boardType, 128) || null,
+            appearanceSummary:
+              sanitizeText((raw.goodsInfo as Record<string, unknown>).appearanceSummary, 255) ||
+              null,
           }
         : null,
   };

types/agent.ts

@@ -208,6 +208,7 @@ export interface AgentWorkflowMaterialImage {
   imageUrl: string;
   assetId?: string | null;
   mimeType?: string | null;
+  appearanceSummary?: string | null;
   sourceType?: "search" | "upload";
   group: "fixed" | "other";
   sortOrder: number;
@@ -243,6 +244,7 @@ export interface AgentWorkflowMaterialSearchResult {
     designer?: string | null;
     buyer?: string | null;
     boardType?: string | null;
+    appearanceSummary?: string | null;
   } | null;
 }