feat: improve image generation concurrency
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-05-08 21:23:44 +08:00
parent e157480ec2
commit 1991e5c4cc
11 changed files with 371 additions and 89 deletions

View File

@ -245,6 +245,8 @@ import { generateTitleFromUserMessage } from "../../actions";
import { type PostRequestBody, postRequestBodySchema } from "./schema";
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
const AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT = 4;
/**
* 将图片 URL 转换为 base64 格式
* 用于将 S3 URL 的图片转换为 base64 发送给 Gemini API
@ -744,6 +746,38 @@ function startStreamKeepalive(writer: StreamWriter, label: string, intervalMs =
return () => clearInterval(timer);
}
async function runBoundedParallel<T>(
items: T[],
workerCount: number,
worker: (item: T, index: number) => Promise<void>
) {
const maxWorkers = Math.max(1, Math.min(workerCount, items.length));
let cursor = 0;
let firstError: unknown = null;
const workers = Array.from({ length: maxWorkers }, async () => {
while (cursor < items.length) {
const index = cursor;
cursor += 1;
if (firstError) {
return;
}
try {
await worker(items[index]!, index);
} catch (error) {
firstError = firstError || error;
}
}
});
await Promise.all(workers);
if (firstError) {
throw firstError;
}
}
type VideoModeType = "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
function isVideoProviderType(
@ -7098,6 +7132,7 @@ export async function POST(request: Request) {
String(params.imageSizeLabel || "").trim() || resolvedImageSize;
const assistantText = prependImageModelBadge(
[
`<!-- AGENT_REPLACEMENT_CARD_ID:${String(params.reference.id || "").trim()} -->`,
`### ${params.reference.label}`,
String(params.bodyText || "").trim(),
buildAgentToolImagesMarkdown([
@ -7175,7 +7210,10 @@ export async function POST(request: Request) {
};
try {
for (const [referenceIndex, reference] of replacementReferences.entries()) {
await runBoundedParallel(
Array.from(replacementReferences.entries()),
AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT,
async ([referenceIndex, reference]) => {
if (reference.passthrough === true) {
const referenceSourceImageUrls = Array.from(
new Set([
@ -7196,7 +7234,7 @@ export async function POST(request: Request) {
streamLabel: "agent-mode:image:detail-replacement:item-passthrough",
});
successCount += 1;
continue;
return;
}
const replacementExecutionPlan = await analyzeReplacementExecutionPlan({
@ -7230,7 +7268,7 @@ export async function POST(request: Request) {
streamLabel: "agent-mode:image:detail-replacement:item",
});
successCount += 1;
continue;
return;
}
const referenceGeometry = await resolveReplacementReferenceGeometry(reference.imageUrl);
@ -7272,6 +7310,7 @@ export async function POST(request: Request) {
prompt: replacementPrompt,
negativePrompt: effectiveImageParams?.negativePrompt,
model: resolvedModelRef,
ownerKey: session.user.id,
aspectRatio: generationAspectRatio,
imageSize: resolvedImageSize,
imageCount: 1,
@ -7357,7 +7396,7 @@ export async function POST(request: Request) {
});
successCount += 1;
}
}
});
} finally {
stopKeepalive();
}
@ -7725,6 +7764,7 @@ export async function POST(request: Request) {
String(params.imageSizeLabel || "").trim() || resolvedImageSize;
const assistantText = prependImageModelBadge(
[
`<!-- AGENT_REPLACEMENT_CARD_ID:${String(params.reference.id || "").trim()} -->`,
`### ${params.reference.label}`,
String(params.bodyText || "").trim(),
buildAgentToolImagesMarkdown([
@ -7810,7 +7850,10 @@ export async function POST(request: Request) {
};
try {
for (const [referenceIndex, reference] of replacementReferences.entries()) {
await runBoundedParallel(
Array.from(replacementReferences.entries()),
AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT,
async ([referenceIndex, reference]) => {
if (reference.passthrough === true) {
const replacementSourceType = isDetailReplacementPreset
? "detail_replacement"
@ -7837,7 +7880,7 @@ export async function POST(request: Request) {
: "agent-preset:image:sku-replacement:item-passthrough",
});
successCount += 1;
continue;
return;
}
const pairedProductIndex =
@ -7932,7 +7975,7 @@ export async function POST(request: Request) {
: "agent-preset:image:sku-replacement:item",
});
successCount += 1;
continue;
return;
}
const referenceGeometry = await resolveReplacementReferenceGeometry(reference.imageUrl);
@ -8013,6 +8056,7 @@ export async function POST(request: Request) {
? mergedImageParams.negativePrompt
: undefined,
model: resolvedModelRef,
ownerKey: session.user.id,
aspectRatio: generationAspectRatio,
imageSize: resolvedImageSize,
imageCount: 1,
@ -8112,7 +8156,7 @@ export async function POST(request: Request) {
});
successCount += 1;
}
}
});
} finally {
stopKeepalive();
}
@ -8293,6 +8337,7 @@ export async function POST(request: Request) {
? mergedImageParams.negativePrompt
: undefined,
model: resolvedModelRef,
ownerKey: session.user.id,
aspectRatio:
typeof mergedImageParams.aspectRatio === "string"
? mergedImageParams.aspectRatio
@ -8506,6 +8551,7 @@ export async function POST(request: Request) {
? mergedImageParams.negativePrompt
: undefined,
model: presetModelId,
ownerKey: session.user.id,
aspectRatio:
typeof mergedImageParams.aspectRatio === "string"
? mergedImageParams.aspectRatio
@ -9022,6 +9068,7 @@ export async function POST(request: Request) {
const directImageResult = await executeDirectAgentImageGeneration({
requestTextContent,
optimizedPrompt: smartAutoOptimizedPrompt,
userId: session.user.id,
finalModelId,
imageParams: effectiveImageParams,
defaultImageQualityParams,

View File

@ -114,7 +114,10 @@ export async function POST(request: Request) {
}
const body: GenerateImageRequest = await request.json();
const payload = toGenerateImagePayload(body);
const payload: CanvasGenerateImageRequest = {
...toGenerateImagePayload(body),
ownerKey: session.user.id,
};
if (!payload.prompt) {
return NextResponse.json(
{ success: false, error: "请输入提示词" },

View File

@ -31,6 +31,7 @@ import {
normalizeSceneFusionProviderType,
type SceneFusionImageGenConfig,
} from "@/lib/scene-fusion/output-capabilities";
import { acquireCanvasImageModelSlot } from "@/lib/canvas/image-concurrency";
// 请求体 Schema
const sceneFusionRequestSchema = z.object({
@ -56,8 +57,8 @@ const SCENE_FUSION_MODEL_COOLDOWN_MS = 2 * 60 * 1000;
const SCENE_FUSION_MAX_ITEMS = 10;
const SCENE_FUSION_REQUEST_STAGGER_MS = 350;
const SCENE_FUSION_AGENT_TIMEOUT_MS = 20000;
const SCENE_FUSION_AGENT_MAX_WORKERS = 1;
const SCENE_FUSION_PROCESS_MAX_WORKERS = 1;
const SCENE_FUSION_AGENT_MAX_WORKERS = 3;
const SCENE_FUSION_PROCESS_MAX_WORKERS = 4;
const SCENE_FUSION_PRE_RMBG_TIMEOUT_MS = 60_000;
const SCENE_FUSION_ECHO_HASH_TIMEOUT_MS = 20_000;
const SCENE_FUSION_ECHO_HASH_THRESHOLD = 8;
@ -66,8 +67,8 @@ const SCENE_FUSION_STRICT_PROMPT_TAIL =
"硬性要求:严禁直接返回第一张参考图(产品图)原图,必须保留第二张图的场景与构图,仅替换目标主体;结果中不得同时完整保留原目标主体与新产品;若识别不确定,请优先保持第二张图背景不变并完成主体替换。";
const SCENE_FUSION_RANDOM_BG_PROMPT_TAIL =
"硬性要求:背景必须与原图明显不同(颜色、结构、元素至少两项显著变化),禁止保留原背景纹理或直接回传原图;仅保留产品主体并保证主体边缘完整自然。";
const sceneFusionModelCooldown = new Map<string, number>();
let sceneFusionRoundRobinCursor = 0;
type SceneFusionModelCooldown = Map<string, number>;
interface SceneFusionModelCandidate {
model: {
@ -100,33 +101,44 @@ interface SceneFusionAgentMeta {
}>;
}
function pruneSceneFusionCooldown(now: number = Date.now()) {
for (const [key, expireAt] of sceneFusionModelCooldown.entries()) {
function pruneSceneFusionCooldown(
cooldown: SceneFusionModelCooldown,
now: number = Date.now()
) {
for (const [key, expireAt] of cooldown.entries()) {
if (expireAt <= now) {
sceneFusionModelCooldown.delete(key);
cooldown.delete(key);
}
}
}
function isSceneFusionCandidateCoolingDown(
candidate: SceneFusionModelCandidate,
cooldown: SceneFusionModelCooldown,
now: number = Date.now()
): boolean {
const expireAt = sceneFusionModelCooldown.get(candidate.model.id);
const expireAt = cooldown.get(candidate.model.id);
if (!expireAt) return false;
if (expireAt <= now) {
sceneFusionModelCooldown.delete(candidate.model.id);
cooldown.delete(candidate.model.id);
return false;
}
return true;
}
function markSceneFusionCandidateFailed(candidate: SceneFusionModelCandidate, now: number = Date.now()) {
sceneFusionModelCooldown.set(candidate.model.id, now + SCENE_FUSION_MODEL_COOLDOWN_MS);
function markSceneFusionCandidateFailed(
candidate: SceneFusionModelCandidate,
cooldown: SceneFusionModelCooldown,
now: number = Date.now()
) {
cooldown.set(candidate.model.id, now + SCENE_FUSION_MODEL_COOLDOWN_MS);
}
function markSceneFusionCandidateHealthy(candidate: SceneFusionModelCandidate) {
sceneFusionModelCooldown.delete(candidate.model.id);
function markSceneFusionCandidateHealthy(
candidate: SceneFusionModelCandidate,
cooldown: SceneFusionModelCooldown
) {
cooldown.delete(candidate.model.id);
}
function rotateCandidates<T>(items: T[], startOffset: number): T[] {
@ -1797,6 +1809,7 @@ async function processSceneWithDiversity(
requestId: string;
candidates: SceneFusionModelCandidate[];
imageHashCache: Map<string, string>;
modelCooldown: SceneFusionModelCooldown;
}
): Promise<{
result: { thumbnail: string; original: string; prompt: string };
@ -1814,10 +1827,10 @@ async function processSceneWithDiversity(
supportedCandidates,
params.requestOffset + params.sceneIndex
);
pruneSceneFusionCooldown();
pruneSceneFusionCooldown(params.modelCooldown);
const healthyCandidates = rotatedCandidates.filter(
(candidate) => !isSceneFusionCandidateCoolingDown(candidate)
(candidate) => !isSceneFusionCandidateCoolingDown(candidate, params.modelCooldown)
);
const orderedHealthyCandidates = [...healthyCandidates].sort((left, right) => {
const scoreDiff =
@ -1858,25 +1871,41 @@ async function processSceneWithDiversity(
);
}
const result = await processSceneWithProvider(
params.productImageUrl,
params.sceneImageUrl,
params.productName,
params.targetName,
params.positionHint,
params.replaceCue,
params.customPrompt,
params.defaultPrompt,
params.sceneMode,
{
type: candidate.provider.type,
baseUrl: candidate.provider.baseUrl,
apiKey: candidate.provider.apiKey,
const modelSlot = await acquireCanvasImageModelSlot({
modelKey: candidate.model.modelId || candidate.model.id,
debugLabel: "scene-fusion",
debugMeta: {
requestId: params.requestId,
sceneIndex: params.sceneIndex,
modelId: candidate.model.modelId,
modelRef: candidate.model.id,
providerType: candidate.provider.type,
},
candidate.model.modelId,
normalizedOutputParams.aspectRatio,
normalizedOutputParams.imageSize
);
});
let result: Awaited<ReturnType<typeof processSceneWithProvider>>;
try {
result = await processSceneWithProvider(
params.productImageUrl,
params.sceneImageUrl,
params.productName,
params.targetName,
params.positionHint,
params.replaceCue,
params.customPrompt,
params.defaultPrompt,
params.sceneMode,
{
type: candidate.provider.type,
baseUrl: candidate.provider.baseUrl,
apiKey: candidate.provider.apiKey,
},
candidate.model.modelId,
normalizedOutputParams.aspectRatio,
normalizedOutputParams.imageSize
);
} finally {
await modelSlot.release();
}
const echoCheck = await detectEchoedProductResult({
requestId: params.requestId,
@ -1913,7 +1942,7 @@ async function processSceneWithDiversity(
);
}
markSceneFusionCandidateHealthy(candidate);
markSceneFusionCandidateHealthy(candidate, params.modelCooldown);
return {
result,
candidate,
@ -1922,7 +1951,7 @@ async function processSceneWithDiversity(
} catch (error) {
const errorMessage = error instanceof Error ? error.message : "未知错误";
failureReasons.push(`${candidate.model.displayName}: ${errorMessage}`);
markSceneFusionCandidateFailed(candidate);
markSceneFusionCandidateFailed(candidate, params.modelCooldown);
console.error("[SceneFusion] Candidate failed, try next:", {
requestId: params.requestId,
sceneIndex: params.sceneIndex,
@ -2093,6 +2122,7 @@ export async function POST(request: Request) {
}
const resolvedSceneMap = new Map(resolvedScenes.map((scene) => [scene.id, scene]));
const imageHashCache = new Map<string, string>();
const modelCooldown: SceneFusionModelCooldown = new Map();
console.log(
`[SceneFusion][${requestId}] Agent labels: product=${resolvedProductName || "(none)"}; targets=${resolvedScenes
@ -2155,7 +2185,6 @@ export async function POST(request: Request) {
1,
Math.min(
SCENE_FUSION_PROCESS_MAX_WORKERS,
candidateModels.length,
processedScenes.length
)
);
@ -2218,6 +2247,7 @@ export async function POST(request: Request) {
requestId,
candidates: candidateModels,
imageHashCache,
modelCooldown,
});
const dbImageSize = imageSize || "2K";

View File

@ -246,6 +246,7 @@ interface GeneratedImageGroup {
messageId: string;
imageUrls: string[];
text: string;
replacementCardId?: string | null;
}
interface AgentModeReferenceSearchResponse {
@ -507,10 +508,16 @@ function normalizeAssistantText(value: string) {
.replace(/\n*<!-- GENERATING_IMAGE -->\n*/g, "")
.replace(/\n*<!-- IMAGE_GENERATION_START -->\n*/g, "")
.replace(/\n*<!-- IMAGE_GENERATION_END -->\n*/g, "")
.replace(/\n*<!-- AGENT_REPLACEMENT_CARD_ID:[^>]*-->\n*/g, "")
.replace(/!\[[^\]]*\]\((https?:\/\/[^\s)]+)\)/g, "")
.trim();
}
function extractReplacementCardIdFromAssistantText(value: string) {
const match = String(value || "").match(/<!--\s*AGENT_REPLACEMENT_CARD_ID:([^>]+?)\s*-->/);
return match?.[1]?.trim() || null;
}
function hasLikelyGenerationErrorText(value: string | null | undefined) {
const normalized = normalizeAssistantText(String(value || ""));
if (!normalized) {
@ -577,11 +584,52 @@ type DirectDetailReviewResult = {
const AGENT_IMAGE_TASK_POLL_INTERVAL_MS = 1600;
const AGENT_IMAGE_TASK_TIMEOUT_MS = 4 * 60 * 1000;
const AGENT_DETAIL_IMAGE_PARALLEL_WORKER_COUNT = 4;
function sleep(ms: number) {
return new Promise((resolve) => window.setTimeout(resolve, ms));
}
async function runAgentBoundedParallel<T>(
items: T[],
workerCount: number,
worker: (item: T, index: number) => Promise<void>
) {
const maxWorkers = Math.max(1, Math.min(workerCount, items.length));
let cursor = 0;
let firstError: unknown = null;
const workers = Array.from({ length: maxWorkers }, async () => {
while (cursor < items.length) {
const index = cursor;
cursor += 1;
if (firstError) {
return;
}
try {
await worker(items[index]!, index);
} catch (error) {
firstError = firstError || error;
}
}
});
await Promise.all(workers);
if (firstError) {
throw firstError;
}
}
function compactDirectDetailGenerationImages(
images: Array<DirectDetailGenerationImage | null | undefined>
) {
return images.filter(
(item): item is DirectDetailGenerationImage => Boolean(item?.originalUrl || item?.thumbnailUrl)
);
}
async function requestCanvasImageGeneration(
payload: {
prompt: string;
@ -2307,13 +2355,15 @@ export function AgentGenerationWorkspaceDarkV2({
const textParts = (message.parts || [])
.filter((part: any) => part.type === "text" && part.text)
.map((part: any) => String(part.text));
const rawText = textParts.join("\n\n");
const imageUrls = Array.from(
new Set(textParts.flatMap((text) => extractImageUrlsFromText(text)))
);
return {
messageId: message.id,
imageUrls,
text: normalizeAssistantText(textParts.join("\n\n")),
text: normalizeAssistantText(rawText),
replacementCardId: extractReplacementCardIdFromAssistantText(rawText),
};
})
.filter((group) => group.imageUrls.length > 0);
@ -2415,6 +2465,7 @@ export function AgentGenerationWorkspaceDarkV2({
replacementGeneratedGroupsAfterLastUser.map((group) => ({
url: group.imageUrls[0] || null,
text: group.text || "",
replacementCardId: group.replacementCardId || null,
})),
[replacementGeneratedGroupsAfterLastUser]
);
@ -2499,7 +2550,7 @@ export function AgentGenerationWorkspaceDarkV2({
let changed = false;
latestReplacementResults.forEach((result, index) => {
const cardId = cardIds[index];
const cardId = result.replacementCardId || cardIds[index];
if (!cardId) {
return;
}
@ -4351,6 +4402,8 @@ export function AgentGenerationWorkspaceDarkV2({
timestamp: Date.now(),
});
let completedDetailGenerationCount = 0;
try {
const planResponse = await fetch("/api/canvas/detail-plan", {
method: "POST",
@ -4406,8 +4459,16 @@ export function AgentGenerationWorkspaceDarkV2({
finishedAt: null,
});
const generatedImages: DirectDetailGenerationImage[] = [];
for (let index = 0; index < prompts.length; index += 1) {
const generatedImages: Array<DirectDetailGenerationImage | null> = Array.from(
{ length: prompts.length },
() => null
);
const publishGeneratedImages = () => {
const compactImages = compactDirectDetailGenerationImages(generatedImages);
completedDetailGenerationCount = compactImages.length;
setDetailGenerationRunResults(compactImages);
};
const generateDetailScreen = async (index: number) => {
if (detailGenerationCancelRef.current) {
throw new Error("已停止当前 AI详情生成任务");
}
@ -4598,26 +4659,39 @@ export function AgentGenerationWorkspaceDarkV2({
subtitle: String(screen?.subtitle || "").trim(),
copy: String(screen?.copy || "").trim(),
};
generatedImages.push(nextImage);
setDetailGenerationRunResults([...generatedImages]);
generatedImages[index] = nextImage;
publishGeneratedImages();
};
await generateDetailScreen(0);
const remainingPromptIndexes = prompts.slice(1).map((_, index) => index + 1);
if (remainingPromptIndexes.length > 0) {
await runAgentBoundedParallel(
remainingPromptIndexes,
AGENT_DETAIL_IMAGE_PARALLEL_WORKER_COUNT,
async (promptIndex) => {
await generateDetailScreen(promptIndex);
}
);
}
const finalGeneratedImages = compactDirectDetailGenerationImages(generatedImages);
updateDetailGenerationProgressState({
status: "done",
stage: `AI详情生成完成,共 ${generatedImages.length}/${prompts.length} 屏`,
stage: `AI详情生成完成,共 ${finalGeneratedImages.length}/${prompts.length} 屏`,
current: Math.max(1, prompts.length * 2 + 1),
total: Math.max(1, prompts.length * 2 + 1),
startedAt,
finishedAt: new Date().toISOString(),
});
toast.success(`AI详情生成完成,已输出 ${generatedImages.length} 屏详情图`);
toast.success(`AI详情生成完成,已输出 ${finalGeneratedImages.length} 屏详情图`);
} catch (error) {
const message =
error instanceof Error ? error.message : "AI详情生成失败";
updateDetailGenerationProgressState({
status: /已停止/.test(message) ? "idle" : "error",
stage: message,
current: detailGenerationRunResults.length,
current: completedDetailGenerationCount,
total: pendingGeneration?.expectedCount || detailCount,
startedAt,
finishedAt: new Date().toISOString(),

View File

@ -51,7 +51,7 @@ import { cn } from "@/lib/utils";
const fetcher = (url: string) => fetch(url).then((res) => res.json());
const MAX_BATCH_ITEMS = 10;
const BATCH_REQUEST_STAGGER_MS = 700;
const BATCH_SERIAL_WORKER_COUNT = 1;
const BATCH_PARALLEL_WORKER_COUNT = 4;
const MULTI_PRODUCT_FAILOVER_MAX_ROUNDS = 5;
const MULTI_PRODUCT_FAILOVER_RETRY_DELAY_MS = 1500;
@ -1158,7 +1158,7 @@ export function SceneFusionView({
const queue = [...itemsToProcess];
const workerCount = Math.max(
1,
Math.min(BATCH_SERIAL_WORKER_COUNT, queue.length)
Math.min(BATCH_PARALLEL_WORKER_COUNT, queue.length)
);
let successCount = 0;
let failedCount = 0;
@ -1332,7 +1332,7 @@ export function SceneFusionView({
const queue = [...itemsToProcess];
const workerCount = Math.max(
1,
Math.min(BATCH_SERIAL_WORKER_COUNT, queue.length)
Math.min(BATCH_PARALLEL_WORKER_COUNT, queue.length)
);
let successCount = 0;
let failedCount = 0;

View File

@ -209,6 +209,7 @@ export async function executeDirectAgentImageEditTool(
prompt: directEditPrompt,
negativePrompt: params.imageParams?.negativePrompt,
model: params.finalModelId,
ownerKey: params.userId,
aspectRatio,
imageSize,
imageCount: 1,

View File

@ -52,6 +52,7 @@ export interface DirectAgentImageGenerationResult {
export interface ExecuteDirectAgentImageGenerationParams {
requestTextContent: string;
optimizedPrompt?: string;
userId?: string;
finalModelId: string;
imageParams?: DirectAgentImageParamsSnapshot | null;
defaultImageQualityParams?: DirectAgentImageParamsSnapshot | null;
@ -172,6 +173,7 @@ export async function executeDirectAgentImageGeneration(
prompt,
negativePrompt: params.imageParams?.negativePrompt,
model: params.finalModelId,
ownerKey: params.userId,
aspectRatio:
params.imageParams?.aspectRatio || params.defaultImageQualityParams?.aspectRatio,
imageSize,

View File

@ -28,6 +28,7 @@ export interface CanvasGenerateImageRequest {
referenceImage?: string;
referenceImages?: string[];
billingFeature?: "multi-view" | "repaint";
ownerKey?: string;
debugLabel?: string;
debugMeta?: Record<string, unknown>;
}
@ -152,6 +153,10 @@ export async function generateCanvasImage(
: 1;
const promptExtend =
typeof request.promptExtend === "boolean" ? request.promptExtend : undefined;
const ownerKey =
typeof request.ownerKey === "string" && request.ownerKey.trim()
? request.ownerKey.trim()
: undefined;
const referenceImage =
typeof request.referenceImage === "string" ? request.referenceImage : undefined;
const referenceImages = Array.isArray(request.referenceImages)
@ -454,6 +459,7 @@ export async function generateCanvasImage(
);
const slot = await acquireCanvasImageModelSlot({
modelKey: resolvedModelKey,
ownerKey,
debugLabel,
debugMeta: {
selectedModelId: selectedModel.model.modelId,

View File

@ -14,11 +14,12 @@ const concurrencyStateKey = "__canvas_image_model_concurrency__";
interface ModelWaiter {
ticketId: string;
ownerKey: string;
resolve: () => void;
}
interface ModelConcurrencyState {
running: Set<string>;
running: Map<string, string>;
waiters: ModelWaiter[];
dispatchTimer: ReturnType<typeof setTimeout> | null;
delayedUntil: number | null;
@ -57,7 +58,7 @@ function getOrCreateModelState(modelKey: string) {
}
const created: ModelConcurrencyState = {
running: new Set<string>(),
running: new Map<string, string>(),
waiters: [],
dispatchTimer: null,
delayedUntil: null,
@ -97,6 +98,32 @@ export function normalizeCanvasImageModelKey(value: unknown) {
return normalized || "__default__";
}
function normalizeCanvasImageOwnerKey(value: unknown) {
if (typeof value !== "string") {
return "__anonymous__";
}
const normalized = value.trim();
return normalized || "__anonymous__";
}
function getMaxConcurrentPerOwner(maxConcurrent: number) {
return Math.max(1, Math.ceil(Math.max(1, maxConcurrent) / 2));
}
function getRunningCountForOwner(
modelState: ModelConcurrencyState,
ownerKey: string
) {
let count = 0;
for (const runningOwnerKey of modelState.running.values()) {
if (runningOwnerKey === ownerKey) {
count += 1;
}
}
return count;
}
export async function getCanvasImageTaskQueueConfig(): Promise<CanvasImageTaskQueueConfig> {
const now = Date.now();
if (cachedConfig && cachedConfig.expiresAt > now) {
@ -189,11 +216,20 @@ async function drainModelWaiters(
clearModelDispatchTimer(modelState);
while (modelState.running.size < maxConcurrent && modelState.waiters.length > 0) {
const waiter = modelState.waiters.shift();
const maxConcurrentPerOwner = getMaxConcurrentPerOwner(maxConcurrent);
const waiterIndex = modelState.waiters.findIndex(
(item) =>
getRunningCountForOwner(modelState, item.ownerKey) < maxConcurrentPerOwner
);
if (waiterIndex < 0) {
break;
}
const [waiter] = modelState.waiters.splice(waiterIndex, 1);
if (!waiter) {
break;
}
modelState.running.add(waiter.ticketId);
modelState.running.set(waiter.ticketId, waiter.ownerKey);
waiter.resolve();
}
@ -202,10 +238,12 @@ async function drainModelWaiters(
export async function acquireCanvasImageModelSlot(params: {
modelKey: string;
ownerKey?: string;
debugLabel?: string;
debugMeta?: Record<string, unknown>;
}) {
const normalizedModelKey = normalizeCanvasImageModelKey(params.modelKey);
const normalizedOwnerKey = normalizeCanvasImageOwnerKey(params.ownerKey);
const {
concurrency: maxConcurrent,
batchDispatchIntervalMs,
@ -215,13 +253,15 @@ export async function acquireCanvasImageModelSlot(params: {
const waitingSince = Date.now();
const hasActiveDelay =
modelState.delayedUntil !== null && modelState.delayedUntil > waitingSince;
const maxConcurrentPerOwner = getMaxConcurrentPerOwner(maxConcurrent);
if (
modelState.waiters.length === 0 &&
!hasActiveDelay &&
modelState.running.size < maxConcurrent
modelState.running.size < maxConcurrent &&
getRunningCountForOwner(modelState, normalizedOwnerKey) < maxConcurrentPerOwner
) {
modelState.running.add(ticketId);
modelState.running.set(ticketId, normalizedOwnerKey);
return {
release: async () => {
const state = getGlobalConcurrencyState().models.get(normalizedModelKey);
@ -238,6 +278,7 @@ export async function acquireCanvasImageModelSlot(params: {
await new Promise<void>((resolve) => {
modelState.waiters.push({
ticketId,
ownerKey: normalizedOwnerKey,
resolve,
});
void drainModelWaiters(normalizedModelKey, "arrival");

View File

@ -19,13 +19,20 @@ type CanvasImageTaskStatus = "pending" | "running" | "succeeded" | "failed";
interface PendingTaskEntry {
taskId: string;
modelKey: string;
userId: string;
}
interface RunningTaskEntry {
modelKey: string;
userId: string;
}
interface QueueState {
pending: PendingTaskEntry[];
pendingSet: Set<string>;
running: Map<string, string>;
running: Map<string, RunningTaskEntry>;
runningByModel: Map<string, Set<string>>;
runningByModelUser: Map<string, Set<string>>;
nextBatchEligibleAt: Map<string, number>;
nextBatchTimers: Map<string, ReturnType<typeof setTimeout>>;
awaitingNextBatch: Set<string>;
@ -44,8 +51,9 @@ function getQueueState(): QueueState {
globalObj[queueStateKey] = {
pending: [],
pendingSet: new Set<string>(),
running: new Map<string, string>(),
running: new Map<string, RunningTaskEntry>(),
runningByModel: new Map<string, Set<string>>(),
runningByModelUser: new Map<string, Set<string>>(),
nextBatchEligibleAt: new Map<string, number>(),
nextBatchTimers: new Map<string, ReturnType<typeof setTimeout>>(),
awaitingNextBatch: new Set<string>(),
@ -64,14 +72,29 @@ function normalizeTaskModelKey(request: unknown) {
return normalizeCanvasImageModelKey(rawModel);
}
async function getTaskModelKey(taskId: string) {
function getModelUserKey(modelKey: string, userId: string) {
return `${modelKey}::${userId || "__anonymous__"}`;
}
function getMaxConcurrentPerUser(maxConcurrentPerModel: number) {
return Math.max(1, Math.ceil(Math.max(1, maxConcurrentPerModel) / 2));
}
async function getTaskQueueMeta(taskId: string) {
const rows = await db
.select({ request: canvasImageTask.request })
.select({
request: canvasImageTask.request,
userId: canvasImageTask.userId,
})
.from(canvasImageTask)
.where(eq(canvasImageTask.id, taskId))
.limit(1);
return normalizeTaskModelKey(rows[0]?.request);
const row = rows[0];
return {
modelKey: normalizeTaskModelKey(row?.request),
userId: String(row?.userId || "__anonymous__"),
};
}
function getRunningCountForModel(modelKey: string) {
@ -79,30 +102,48 @@ function getRunningCountForModel(modelKey: string) {
return state.runningByModel.get(modelKey)?.size || 0;
}
function markRunningTask(taskId: string, modelKey: string) {
function getRunningCountForModelUser(modelKey: string, userId: string) {
const state = getQueueState();
state.running.set(taskId, modelKey);
return state.runningByModelUser.get(getModelUserKey(modelKey, userId))?.size || 0;
}
function markRunningTask(taskId: string, modelKey: string, userId: string) {
const state = getQueueState();
state.running.set(taskId, { modelKey, userId });
const tasks = state.runningByModel.get(modelKey) || new Set<string>();
tasks.add(taskId);
state.runningByModel.set(modelKey, tasks);
const modelUserKey = getModelUserKey(modelKey, userId);
const userTasks = state.runningByModelUser.get(modelUserKey) || new Set<string>();
userTasks.add(taskId);
state.runningByModelUser.set(modelUserKey, userTasks);
}
function releaseRunningTask(taskId: string) {
const state = getQueueState();
const modelKey = state.running.get(taskId);
if (!modelKey) {
const entry = state.running.get(taskId);
if (!entry) {
return null;
}
state.running.delete(taskId);
const { modelKey, userId } = entry;
const tasks = state.runningByModel.get(modelKey);
if (!tasks) {
return;
if (tasks) {
tasks.delete(taskId);
if (tasks.size === 0) {
state.runningByModel.delete(modelKey);
}
}
tasks.delete(taskId);
if (tasks.size === 0) {
state.runningByModel.delete(modelKey);
const modelUserKey = getModelUserKey(modelKey, userId);
const userTasks = state.runningByModelUser.get(modelUserKey);
if (userTasks) {
userTasks.delete(taskId);
if (userTasks.size === 0) {
state.runningByModelUser.delete(modelUserKey);
}
}
return modelKey;
@ -155,28 +196,52 @@ function scheduleModelNextBatch(modelKey: string, delayMs: number) {
return true;
}
async function enqueueLocalTask(taskId: string, modelKey?: string) {
async function enqueueLocalTask(
taskId: string,
meta?: { modelKey?: string; userId?: string } | string
) {
const state = getQueueState();
if (!taskId || state.running.has(taskId) || state.pendingSet.has(taskId)) {
return;
}
const resolvedModelKey = modelKey || (await getTaskModelKey(taskId));
const resolvedMeta =
typeof meta === "string"
? { modelKey: meta, userId: undefined }
: meta || {};
const dbMeta =
resolvedMeta.modelKey && resolvedMeta.userId
? null
: await getTaskQueueMeta(taskId);
const resolvedModelKey = resolvedMeta.modelKey || dbMeta?.modelKey || "__default__";
const resolvedUserId = resolvedMeta.userId || dbMeta?.userId || "__anonymous__";
state.pending.push({
taskId,
modelKey: resolvedModelKey,
userId: resolvedUserId,
});
state.pendingSet.add(taskId);
const maxConcurrentPerModel = await getCanvasImageSingleModelConcurrency();
if (getRunningCountForModel(resolvedModelKey) >= maxConcurrentPerModel) {
if (
getRunningCountForModel(resolvedModelKey) >= maxConcurrentPerModel ||
getRunningCountForModelUser(resolvedModelKey, resolvedUserId) >=
getMaxConcurrentPerUser(maxConcurrentPerModel)
) {
markModelAwaitingNextBatch(resolvedModelKey);
}
}
function hasModelCapacity(modelKey: string, maxConcurrentPerModel: number) {
const state = getQueueState();
return getRunningCountForModel(modelKey) < maxConcurrentPerModel;
function hasModelCapacity(
modelKey: string,
userId: string,
maxConcurrentPerModel: number
) {
return (
getRunningCountForModel(modelKey) < maxConcurrentPerModel &&
getRunningCountForModelUser(modelKey, userId) <
getMaxConcurrentPerUser(maxConcurrentPerModel)
);
}
async function claimNextPendingTask(
@ -188,7 +253,7 @@ async function claimNextPendingTask(
if (!entry) continue;
if (state.running.has(entry.taskId)) continue;
if (isModelBatchDispatchDelayed(entry.modelKey)) continue;
if (!hasModelCapacity(entry.modelKey, maxConcurrentPerModel)) {
if (!hasModelCapacity(entry.modelKey, entry.userId, maxConcurrentPerModel)) {
markModelAwaitingNextBatch(entry.modelKey);
continue;
}
@ -203,6 +268,7 @@ async function claimNextPendingTask(
.select({
id: canvasImageTask.id,
request: canvasImageTask.request,
userId: canvasImageTask.userId,
})
.from(canvasImageTask)
.where(
@ -221,7 +287,8 @@ async function claimNextPendingTask(
if (isModelBatchDispatchDelayed(modelKey)) {
continue;
}
if (!hasModelCapacity(modelKey, maxConcurrentPerModel)) {
const userId = String(row.userId || "__anonymous__");
if (!hasModelCapacity(modelKey, userId, maxConcurrentPerModel)) {
markModelAwaitingNextBatch(modelKey);
continue;
}
@ -229,6 +296,7 @@ async function claimNextPendingTask(
return {
taskId: row.id,
modelKey,
userId,
};
}
@ -251,13 +319,17 @@ async function runSingleTask(entry: PendingTaskEntry) {
.returning({
id: canvasImageTask.id,
request: canvasImageTask.request,
userId: canvasImageTask.userId,
});
if (!task) {
return;
}
const result = await generateCanvasImage(task.request as CanvasGenerateImageRequest);
const result = await generateCanvasImage({
...(task.request as CanvasGenerateImageRequest),
ownerKey: task.userId,
});
await db
.update(canvasImageTask)
.set({
@ -301,7 +373,7 @@ export async function drainCanvasImageTaskQueue() {
const nextTask = await claimNextPendingTask(maxConcurrentPerModel);
if (!nextTask) break;
if (state.running.has(nextTask.taskId)) continue;
markRunningTask(nextTask.taskId, nextTask.modelKey);
markRunningTask(nextTask.taskId, nextTask.modelKey, nextTask.userId);
void runSingleTask(nextTask);
}
} finally {
@ -379,7 +451,10 @@ export async function getCanvasImageTaskForUser(params: {
lt(canvasImageTask.startedAt, staleBefore)
)
);
await enqueueLocalTask(task.id, normalizeTaskModelKey(task.request));
await enqueueLocalTask(task.id, {
modelKey: normalizeTaskModelKey(task.request),
userId: params.userId,
});
await drainCanvasImageTaskQueue();
task.status = "pending" as CanvasImageTaskStatus;
task.startedAt = null;
@ -388,7 +463,10 @@ export async function getCanvasImageTaskForUser(params: {
}
if (task.status === "pending") {
await enqueueLocalTask(task.id, normalizeTaskModelKey(task.request));
await enqueueLocalTask(task.id, {
modelKey: normalizeTaskModelKey(task.request),
userId: params.userId,
});
await drainCanvasImageTaskQueue();
}

File diff suppressed because one or more lines are too long