WuYongJie/furniture-3d-system-Wu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(intake): 优化AI白底图多视角生成

Browse Source
This commit is contained in:
WuYongJie 2026-06-18 11:08:29 +08:00
parent 62eaf50800
commit 853d22d044
5 changed files with 413 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
furniture-3d-system/src/app/intake/non-integrated/workspaces/ai-white-background/NonIntegratedAiWhiteBackgroundWorkspace.test.tsx
View File

@ -1,7 +1,9 @@
import { fireEvent, render, screen, waitFor } from "@testing-library/react";
import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
import {
AI_WHITE_BACKGROUND_SOURCE_SLOTS,
AI_WHITE_BACKGROUND_VIEW_KEY_TO_SOURCE_SLOT,
type AiWhiteBackgroundSourceSlot,
type AiWhiteBackgroundViewKey,
} from "./nonIntegratedAiWhiteBackground";
import type {
NonIntegratedProductRecord,
@ -54,6 +56,31 @@ const mockedGenerateSession = jest.mocked(
generateSharedNonIntegratedAiWhiteBackgroundSession,
);
function createGenerateResponse(viewKey: AiWhiteBackgroundViewKey) {
return {
session: {
id: "session-1",
styleKey: "K01",
targetBarcode: "BAR-002",
sourceBarcode: "BAR-001",
barcode: "BAR-002",
skuBatchNo: 1,
status: "generated" as const,
sourceFileIdsBySlot: {
[AI_WHITE_BACKGROUND_VIEW_KEY_TO_SOURCE_SLOT[viewKey]]: `file-${viewKey}`,
},
generatedFileIdsByView: { [viewKey]: `generated-${viewKey}` },
createdAt: "2026-06-16T10:00:00.000Z",
updatedAt: "2026-06-16T10:01:00.000Z",
},
product: {
id: "product-1",
uploadFiles: [],
metadata: undefined,
} as NonIntegratedProductRecord,
};
}
function createSkuItem(
overrides: Partial<NonIntegratedSkuItem> = {},
): NonIntegratedSkuItem {
@ -355,4 +382,105 @@ describe("NonIntegratedAiWhiteBackgroundWorkspace", () => {
}),
);
});
it("runs bulk generation with limited concurrent view requests", async () => {
const pendingRequests: Array<{
viewKey: AiWhiteBackgroundViewKey;
resolve: (value: ReturnType<typeof createGenerateResponse>) => void;
}> = [];
mockedGenerateSession.mockImplementation((_productId, payload) => {
const viewKey = payload.requestedViewKeys?.[0] as AiWhiteBackgroundViewKey;
return new Promise((resolve) => {
pendingRequests.push({ viewKey, resolve });
});
});
render(
<NonIntegratedAiWhiteBackgroundWorkspace
productId="product-1"
isReadOnly={false}
canUseAiWhiteBackground
currentProductPhotoStyleKey="K01"
currentProductPhotoStyleLabel="娆惧紡涓€"
currentStyleBarcodeEntries={[
createBarcodeEntry({
skuItemId: "sku-source-1",
barcode: "BAR-001",
label: "BAR-001 / 鎵规1",
secondaryLabel: "鑳℃鏈ㄨ壊",
hasOriginalImages: true,
aiSourceSlotFiles: createAiSourceSlotFiles({
positive: createUploadFile("positive", { name: "front-source.png" }),
back: createUploadFile("back", { name: "back-source.png" }),
left: createUploadFile("left", { name: "left-source.png" }),
right: createUploadFile("right", { name: "right-source.png" }),
}),
availableAiSourceSlotCount: 4,
}),
createBarcodeEntry({
skuItemId: "sku-target-1",
barcode: "BAR-002",
label: "BAR-002 / 鎵规1",
secondaryLabel: "娴呮鏈ㄨ壊",
hasOriginalImages: false,
aiSourceSlotFiles: createAiSourceSlotFiles({}),
availableAiSourceSlotCount: 0,
}),
]}
currentTargetBarcodeEntry={createBarcodeEntry({
skuItemId: "sku-target-1",
barcode: "BAR-002",
label: "BAR-002 / 鎵规1",
secondaryLabel: "娴呮鏈ㄨ壊",
hasOriginalImages: false,
aiSourceSlotFiles: createAiSourceSlotFiles({}),
availableAiSourceSlotCount: 0,
})}
selectedTargetBarcode="BAR-002"
displayedSkuItems={[createSkuItem({ barcode: "BAR-001" })]}
skuItems={[createSkuItem({ barcode: "BAR-001" })]}
materialLibraryFinishes={[]}
uploadFiles={[]}
productMetadata={undefined}
onProductSynced={() => {}}
onNotify={() => {}}
/>,
);
fireEvent.click(screen.getByRole("button", { name: "select-source" }));
await waitFor(() => {
expect(latestPanelProps?.generateDisabledReason).toBeNull();
});
fireEvent.click(screen.getByRole("button", { name: "generate-bulk" }));
await waitFor(() => {
expect(mockedGenerateSession).toHaveBeenCalledTimes(3);
});
expect(pendingRequests.map((request) => request.viewKey)).toEqual([
"front",
"back",
"left",
]);
await act(async () => {
pendingRequests[0]!.resolve(createGenerateResponse("front"));
});
await waitFor(() => {
expect(mockedGenerateSession).toHaveBeenCalledTimes(4);
});
expect(pendingRequests.map((request) => request.viewKey)).toEqual([
"front",
"back",
"left",
"right",
]);
await act(async () => {
pendingRequests.slice(1).forEach((request) => {
request.resolve(createGenerateResponse(request.viewKey));
});
});
});
});

65
furniture-3d-system/src/app/intake/non-integrated/workspaces/ai-white-background/NonIntegratedAiWhiteBackgroundWorkspace.tsx
View File

@ -71,6 +71,8 @@ type AiWhiteBackgroundBulkGenerateProgress = {
currentViewKey: AiWhiteBackgroundViewKey | null;
};
const AI_WHITE_BACKGROUND_BULK_GENERATE_CONCURRENCY = 3;
function buildClientAiWhiteBackgroundSessionId() {
const suffix =
typeof crypto !== "undefined" && typeof crypto.randomUUID === "function"
@ -644,8 +646,13 @@ export function NonIntegratedAiWhiteBackgroundWorkspace({
setPendingSessionId(sessionId);
}
const runSingleView = async (viewKey: AiWhiteBackgroundViewKey) => {
setGeneratingViewKeys([viewKey]);
const runSingleView = async (
viewKey: AiWhiteBackgroundViewKey,
options: { manageGeneratingState?: boolean } = {},
) => {
if (options.manageGeneratingState !== false) {
setGeneratingViewKeys([viewKey]);
}
const result = await generateSharedNonIntegratedAiWhiteBackgroundSession(
productId,
{
@ -688,22 +695,48 @@ export function NonIntegratedAiWhiteBackgroundWorkspace({
const failedViews: string[] = [];
let successCount = 0;
let completedCount = 0;
let nextIndex = 0;
const activeViewKeys = new Set<AiWhiteBackgroundViewKey>();
const workerCount = Math.min(
AI_WHITE_BACKGROUND_BULK_GENERATE_CONCURRENCY,
nextViewKeys.length,
);
try {
for (let index = 0; index < nextViewKeys.length; index += 1) {
const viewKey = nextViewKeys[index]!;
setBulkGenerateProgress({
total: nextViewKeys.length,
completed: index,
currentViewKey: viewKey,
});
try {
await runSingleView(viewKey);
successCount += 1;
} catch {
failedViews.push(viewKey);
}
}
await Promise.all(
Array.from({ length: workerCount }, async () => {
while (nextIndex < nextViewKeys.length) {
const viewKey = nextViewKeys[nextIndex]!;
nextIndex += 1;
activeViewKeys.add(viewKey);
setGeneratingViewKeys([...activeViewKeys]);
setBulkGenerateProgress({
total: nextViewKeys.length,
completed: completedCount,
currentViewKey: viewKey,
});
try {
await runSingleView(viewKey, { manageGeneratingState: false });
successCount += 1;
} catch {
failedViews.push(viewKey);
} finally {
completedCount += 1;
activeViewKeys.delete(viewKey);
const nextActiveViewKey =
activeViewKeys.values().next().value ?? null;
setGeneratingViewKeys([...activeViewKeys]);
setBulkGenerateProgress({
total: nextViewKeys.length,
completed: completedCount,
currentViewKey: nextActiveViewKey,
});
}
}
})
);
if (successCount > 0 && failedViews.length === 0) {
onNotify(`已生成 ${successCount} 个视角白底图。`, "success");

25
furniture-3d-system/src/app/intake/non-integrated/workspaces/ai-white-background/nonIntegratedAiWhiteBackground.ts
View File

@ -67,6 +67,7 @@ export type AiWhiteBackgroundSession = {
status: AiWhiteBackgroundSessionStatus;
sourceFileIdsBySlot: Partial<Record<AiWhiteBackgroundSourceSlot, string>>;
generatedFileIdsByView: Partial<Record<AiWhiteBackgroundViewKey, string>>;
structureReferenceFileIdsByView?: Partial<Record<AiWhiteBackgroundViewKey, string[]>>;
lastManifestId?: string;
createdAt: string;
updatedAt: string;
@ -329,6 +330,27 @@ function normalizeGeneratedFileIdsByView(value: unknown) {
) as Partial<Record<AiWhiteBackgroundViewKey, string>>;
}
function normalizeStructureReferenceFileIdsByView(value: unknown) {
if (!isRecord(value)) {
return {};
}
return Object.fromEntries(
Object.entries(value).flatMap(([viewKey, fileIds]) => {
if (!isAiWhiteBackgroundViewKey(viewKey) || !Array.isArray(fileIds)) {
return [];
}
const normalizedFileIds = fileIds
.filter((fileId): fileId is string => typeof fileId === "string")
.map((fileId) => fileId.trim())
.filter(Boolean);
return normalizedFileIds.length > 0
? [[viewKey, normalizedFileIds] as [AiWhiteBackgroundViewKey, string[]]]
: [];
}),
) as Partial<Record<AiWhiteBackgroundViewKey, string[]>>;
}
function normalizeSessionStatus(value: unknown): AiWhiteBackgroundSessionStatus {
return value === "generated" ||
value === "uploaded" ||
@ -414,6 +436,9 @@ function normalizeSession(value: unknown): AiWhiteBackgroundSession | null {
generatedFileIdsByView: normalizeGeneratedFileIdsByView(
value.generatedFileIdsByView,
),
structureReferenceFileIdsByView: normalizeStructureReferenceFileIdsByView(
value.structureReferenceFileIdsByView,
),
lastManifestId: normalizeString(value.lastManifestId) || undefined,
createdAt:
normalizeString(value.createdAt) || new Date().toISOString(),

113
furniture-3d-system/src/server/services/__tests__/non-integrated-ai-white-background.service.test.ts
View File

@ -1102,6 +1102,119 @@ describe("nonIntegratedAiWhiteBackgroundService", () => {
).toBe("BAR-002");
});
it("uses adjacent source views as structure references for a generated view", async () => {
mockedGetActiveProviderConfig.mockResolvedValue({
baseUrl: "https://ai.example.com",
apiKey: "ai-key",
} as never);
mockedFetch.mockImplementation(async (input) => {
const url = typeof input === "string" ? input : String(input);
if (
url.includes("/api/intake/non-integrated/assets/") ||
url.includes("non-integrated-assets/")
) {
return {
ok: true,
arrayBuffer: async () =>
png1x1Buffer.buffer.slice(
png1x1Buffer.byteOffset,
png1x1Buffer.byteOffset + png1x1Buffer.byteLength,
),
headers: {
get: () => "image/png",
},
} as never;
}
if (url === "https://ai.example.com/api/external/image/generate") {
return {
ok: true,
json: async () => ({
data: {
requestId: "ai-request-structure-1",
images: [{ url: "https://cdn.example.com/generated-front.png" }],
},
}),
} as never;
}
if (url === "https://cdn.example.com/generated-front.png") {
return {
ok: true,
arrayBuffer: async () => generatedImageBuffer,
headers: {
get: () => "image/png",
},
} as never;
}
throw new Error(`Unexpected fetch call: ${url}`);
});
const sourceCategory = createGeneratedFile("session-category", "front").category;
const product = createProduct({
uploadFiles: [
{
id: "source-positive",
name: "K01-positive.png",
category: sourceCategory,
dataUrl: "/api/intake/non-integrated/assets/source-positive.png",
mimeType: "image/png",
photoGroup: "positive",
productPhotoStyleKey: "K01",
productPhotoBarcode: "BAR-001",
},
{
id: "source-reverse",
name: "K01-reverse.png",
category: sourceCategory,
dataUrl: "/api/intake/non-integrated/assets/source-reverse.png",
mimeType: "image/png",
photoGroup: "reverse",
productPhotoStyleKey: "K01",
productPhotoBarcode: "BAR-001",
},
{
id: "source-back",
name: "K01-back.png",
category: sourceCategory,
dataUrl: "/api/intake/non-integrated/assets/source-back.png",
mimeType: "image/png",
photoGroup: "back",
productPhotoStyleKey: "K01",
productPhotoBarcode: "BAR-001",
},
],
});
mockedGetProductById.mockResolvedValue(product as never);
const result = await nonIntegratedAiWhiteBackgroundService.generateSession(
"product-1",
{
styleKey: "K01",
barcode: "BAR-001",
requestedViewKeys: ["front"],
},
authUser,
);
const generateCall = mockedFetch.mock.calls.find(
([url]) => String(url) === "https://ai.example.com/api/external/image/generate",
);
const generateRequest = generateCall?.[1] as RequestInit | undefined;
const generateFormData = generateRequest?.body as FormData | undefined;
expect(generateFormData?.getAll("images")).toHaveLength(3);
expect(String(generateFormData?.get("prompt"))).toContain(
"The primary reference image is the front view",
);
expect(String(generateFormData?.get("prompt"))).toContain(
"additional structure reference images",
);
expect(String(generateFormData?.get("prompt"))).toContain(
"The final output must still be the front view",
);
expect(result.session.structureReferenceFileIdsByView).toEqual({
front: ["source-reverse", "source-back"],
});
});
it("rejects generation when the target barcode has no color text", async () => {
mockedGetActiveProviderConfig.mockResolvedValue({
baseUrl: "https://ai.example.com",

101
furniture-3d-system/src/server/services/non-integrated-ai-white-background.service.ts
View File

@ -66,6 +66,7 @@ const AI_WHITE_BACKGROUND_UPLOAD_FILE_GROUP = "ai-white-background";
const AI_WHITE_BACKGROUND_IMAGE_SIZE = "2K";
const AI_WHITE_BACKGROUND_ASPECT_RATIO = "1:1";
const AI_WHITE_BACKGROUND_SAVE_RETRY_LIMIT = 5;
const AI_WHITE_BACKGROUND_STRUCTURE_REFERENCE_OFFSET = 1;
const AI_WHITE_BACKGROUND_RESULT_FETCH_TIMEOUT_MS = 45_000;
const AI_WHITE_BACKGROUND_RESULT_CURL_CONNECT_TIMEOUT_SECONDS = 10;
const AI_WHITE_BACKGROUND_RESULT_CURL_MAX_TIME_SECONDS = 45;
@ -263,6 +264,7 @@ function buildAiWhiteBackgroundPrompt(
viewKey: AiWhiteBackgroundViewKey,
targetColor: string | null,
materialReferences: AiWhiteBackgroundResolvedMaterialReference[],
structureReferenceViewLabels: string[] = [],
) {
const englishViewLabelByKey: Record<AiWhiteBackgroundViewKey, string> = {
front: "front view",
@ -288,9 +290,16 @@ function buildAiWhiteBackgroundPrompt(
}
return [];
});
const normalizedStructureReferenceLabels = structureReferenceViewLabels
.map((label) => label.trim())
.filter(Boolean);
return [
"Create a realistic studio product render from this furniture product photo.",
`The primary reference image is the ${englishViewLabelByKey[viewKey]}; lock the final camera angle, framing, orientation, and visible side relationships to this primary image.`,
normalizedStructureReferenceLabels.length > 0
? `Use the additional structure reference images (${normalizedStructureReferenceLabels.join(", ")}) only to verify product details that are unclear in the primary image, such as shelves, wheels, back panels, side panels, holes, hardware, seams, and thickness. The final output must still be the ${englishViewLabelByKey[viewKey]}, not any auxiliary reference view.`
: "",
targetColor
? `The target product finish theme is ${targetColor}.`
: "Keep the original overall finish family unless a part instruction says otherwise.",
@ -299,7 +308,7 @@ function buildAiWhiteBackgroundPrompt(
...materialInstructions,
"Use a clean white background, centered composition, and product-photography lighting.",
`Keep the current view as ${englishViewLabelByKey[viewKey]}.`,
].join(" ");
].filter(Boolean).join(" ");
}
function buildAiWhiteBackgroundNegativePrompt(targetColor: string | null) {
@ -1339,6 +1348,65 @@ function buildSourceFileMap(
);
}
function getAiWhiteBackgroundStructureReferenceViewKeys(
viewKey: AiWhiteBackgroundViewKey,
) {
const currentIndex = AI_WHITE_BACKGROUND_VIEW_KEYS.indexOf(viewKey);
if (currentIndex < 0) {
return [];
}
const total = AI_WHITE_BACKGROUND_VIEW_KEYS.length;
const previousIndex =
(currentIndex - AI_WHITE_BACKGROUND_STRUCTURE_REFERENCE_OFFSET + total) %
total;
const nextIndex =
(currentIndex + AI_WHITE_BACKGROUND_STRUCTURE_REFERENCE_OFFSET) % total;
return [previousIndex, nextIndex]
.map((index) => AI_WHITE_BACKGROUND_VIEW_KEYS[index])
.filter((item): item is AiWhiteBackgroundViewKey => Boolean(item));
}
async function buildAiWhiteBackgroundStructureReferencesForView(
viewKey: AiWhiteBackgroundViewKey,
sourceFileMap: Record<AiWhiteBackgroundSourceSlot, NonIntegratedUploadFile | null>,
) {
const referenceImages: AiWhiteBackgroundReferenceImage[] = [];
const fileIds: string[] = [];
const viewLabels: string[] = [];
for (const referenceViewKey of getAiWhiteBackgroundStructureReferenceViewKeys(
viewKey,
)) {
const sourceSlot = AI_WHITE_BACKGROUND_VIEW_KEY_TO_SOURCE_SLOT[referenceViewKey];
const sourceFile = sourceFileMap[sourceSlot];
if (!sourceFile) {
continue;
}
try {
const sourceBuffer = await readAssetBufferByUrl(sourceFile.dataUrl);
const normalizedBuffer = await normalizeReferenceImageForAi(sourceBuffer);
referenceImages.push({
buffer: normalizedBuffer,
contentType: "image/jpeg",
fileName: `structure-reference-${referenceImages.length + 1}-${referenceViewKey}.jpg`,
});
fileIds.push(sourceFile.id);
viewLabels.push(AI_WHITE_BACKGROUND_VIEW_LABELS[referenceViewKey]);
} catch {
// Optional structure references should improve accuracy without blocking the main view.
}
}
return {
referenceImages,
fileIds,
viewLabels,
};
}
function buildGeneratedFileMap(
product: NonIntegratedProductRecord,
session: AiWhiteBackgroundSession,
@ -1478,7 +1546,7 @@ function createAiWhiteBackgroundSession(params: {
confirmedPartGroupingSource: AiWhiteBackgroundPartGroupingSource | null;
now: string;
baseSession?: AiWhiteBackgroundSession | null;
}) {
}): AiWhiteBackgroundSession {
const materialRecipeSnapshot = createAiWhiteBackgroundMaterialRecipeSnapshot(
params.partGroups,
);
@ -1624,6 +1692,7 @@ async function saveGeneratedSessionWithRetry(params: {
confirmedPartGroupingSource: AiWhiteBackgroundPartGroupingSource | null;
sourceFileIdsBySlot: Partial<Record<AiWhiteBackgroundSourceSlot, string>>;
generatedFileIdsByView: Partial<Record<AiWhiteBackgroundViewKey, string>>;
structureReferenceFileIdsByView: Partial<Record<AiWhiteBackgroundViewKey, string[]>>;
generatedFiles: Array<NonIntegratedUploadFile & { storageKey?: string; size?: number }>;
}) {
for (let attempt = 0; attempt < AI_WHITE_BACKGROUND_SAVE_RETRY_LIMIT; attempt += 1) {
@ -1661,6 +1730,10 @@ async function saveGeneratedSessionWithRetry(params: {
...session.generatedFileIdsByView,
...params.generatedFileIdsByView,
},
structureReferenceFileIdsByView: {
...(session.structureReferenceFileIdsByView ?? {}),
...params.structureReferenceFileIdsByView,
},
updatedAt: now,
};
const nextProduct = applyAiWhiteBackgroundPartGroupTemplates({
@ -1940,7 +2013,7 @@ export class NonIntegratedAiWhiteBackgroundService {
});
const materialReferences =
await resolveAiWhiteBackgroundMaterialReferences(partGroups);
const additionalReferenceImages = materialReferences
const materialReferenceImages = materialReferences
.map((reference) => reference.referenceImage ?? null)
.filter(
(reference): reference is AiWhiteBackgroundReferenceImage =>
@ -1952,6 +2025,9 @@ export class NonIntegratedAiWhiteBackgroundService {
> = [];
const nextSourceFileIdsBySlot = { ...session.sourceFileIdsBySlot };
const nextGeneratedFileIdsByView = { ...session.generatedFileIdsByView };
const nextStructureReferenceFileIdsByView = {
...(session.structureReferenceFileIdsByView ?? {}),
};
for (const viewKey of effectiveRequestedViewKeys) {
const sourceSlot = AI_WHITE_BACKGROUND_VIEW_KEY_TO_SOURCE_SLOT[viewKey];
@ -1966,6 +2042,11 @@ export class NonIntegratedAiWhiteBackgroundService {
const sourceBuffer = await readAssetBufferByUrl(sourceFile.dataUrl);
const normalizedReferenceBuffer =
await normalizeReferenceImageForAi(sourceBuffer);
const structureReferences =
await buildAiWhiteBackgroundStructureReferencesForView(
viewKey,
sourceFileMap,
);
let generatedResult: Awaited<
ReturnType<typeof generateAiWhiteBackgroundImage>
>;
@ -1973,11 +2054,15 @@ export class NonIntegratedAiWhiteBackgroundService {
generatedResult = await generateAiWhiteBackgroundImage({
referenceBuffer: normalizedReferenceBuffer,
referenceContentType: "image/jpeg",
additionalReferenceImages,
additionalReferenceImages: [
...materialReferenceImages,
...structureReferences.referenceImages,
],
prompt: buildAiWhiteBackgroundPrompt(
viewKey,
targetColor,
materialReferences,
structureReferences.viewLabels,
),
negativePrompt: buildAiWhiteBackgroundNegativePrompt(targetColor),
});
@ -2026,6 +2111,13 @@ export class NonIntegratedAiWhiteBackgroundService {
nextSourceFileIdsBySlot[sourceSlot] = sourceFile.id;
nextGeneratedFileIdsByView[viewKey] = fileId;
if (structureReferences.fileIds.length > 0) {
nextStructureReferenceFileIdsByView[viewKey] = [
...structureReferences.fileIds,
];
} else {
delete nextStructureReferenceFileIdsByView[viewKey];
}
nextGeneratedFiles.push({
id: fileId,
name: path.basename(uploadTarget.storedFileName),
@ -2057,6 +2149,7 @@ export class NonIntegratedAiWhiteBackgroundService {
confirmedPartGroupingSource,
sourceFileIdsBySlot: nextSourceFileIdsBySlot,
generatedFileIdsByView: nextGeneratedFileIdsByView,
structureReferenceFileIdsByView: nextStructureReferenceFileIdsByView,
generatedFiles: nextGeneratedFiles,
});
} catch (error) {