feat: cache asset visual memory summaries
This commit is contained in:
parent
17567df76b
commit
c1fea459cd
@ -1049,6 +1049,21 @@ function extractMessageImageUrls(parts: any[]): string[] {
|
|||||||
.filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url));
|
.filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function extractMessageImageAssetIds(parts: any[]): string[] {
|
||||||
|
return Array.from(
|
||||||
|
new Set(
|
||||||
|
(parts || [])
|
||||||
|
.map((part: any) => {
|
||||||
|
if (!part || typeof part !== "object") return null;
|
||||||
|
return typeof part.assetId === "string" && part.assetId.trim()
|
||||||
|
? part.assetId.trim()
|
||||||
|
: null;
|
||||||
|
})
|
||||||
|
.filter((assetId: any): assetId is string => typeof assetId === "string" && assetId.length > 0),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function shouldEnrichImageAnalysisText(message?: string): boolean {
|
function shouldEnrichImageAnalysisText(message?: string): boolean {
|
||||||
if (!message) return false;
|
if (!message) return false;
|
||||||
if (!/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) return false;
|
if (!/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) return false;
|
||||||
@ -2242,6 +2257,12 @@ export async function POST(request: Request) {
|
|||||||
text: requestTextContent,
|
text: requestTextContent,
|
||||||
hasImage,
|
hasImage,
|
||||||
imageUrls,
|
imageUrls,
|
||||||
|
assetIds: Array.from(
|
||||||
|
new Set([
|
||||||
|
...extractMessageImageAssetIds(message.parts as any[]),
|
||||||
|
...((contextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||||||
|
]),
|
||||||
|
),
|
||||||
imageFromHistory: smartAutoHistoryImages.length > 0,
|
imageFromHistory: smartAutoHistoryImages.length > 0,
|
||||||
historyMessages,
|
historyMessages,
|
||||||
context: autoContextText || undefined,
|
context: autoContextText || undefined,
|
||||||
@ -2425,6 +2446,12 @@ export async function POST(request: Request) {
|
|||||||
rescueHasImage &&
|
rescueHasImage &&
|
||||||
autoCurrentImageUrlsForFallback.length === 0 &&
|
autoCurrentImageUrlsForFallback.length === 0 &&
|
||||||
smartAutoHistoryImages.length > 0;
|
smartAutoHistoryImages.length > 0;
|
||||||
|
const rescueAssetIds = Array.from(
|
||||||
|
new Set([
|
||||||
|
...extractMessageImageAssetIds(message.parts as any[]),
|
||||||
|
...((contextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||||||
|
]),
|
||||||
|
);
|
||||||
const rescueTimeoutMs = getBudgetedTimeout({
|
const rescueTimeoutMs = getBudgetedTimeout({
|
||||||
startedAt: requestStartedAt,
|
startedAt: requestStartedAt,
|
||||||
preferredMs: SMART_AUTO_FALLBACK_TIMEOUT_MS,
|
preferredMs: SMART_AUTO_FALLBACK_TIMEOUT_MS,
|
||||||
@ -2436,6 +2463,7 @@ export async function POST(request: Request) {
|
|||||||
text: autoTextForFallback,
|
text: autoTextForFallback,
|
||||||
hasImage: rescueHasImage,
|
hasImage: rescueHasImage,
|
||||||
imageUrls: rescueImageUrls,
|
imageUrls: rescueImageUrls,
|
||||||
|
assetIds: rescueAssetIds,
|
||||||
imageFromHistory: rescueFromHistory || autoFromHistoryForFallback,
|
imageFromHistory: rescueFromHistory || autoFromHistoryForFallback,
|
||||||
historyMessages: [],
|
historyMessages: [],
|
||||||
context: undefined,
|
context: undefined,
|
||||||
@ -2486,6 +2514,12 @@ export async function POST(request: Request) {
|
|||||||
|
|
||||||
if (summaryImageUrls.length > 0) {
|
if (summaryImageUrls.length > 0) {
|
||||||
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
||||||
|
const summaryAssetIds = Array.from(
|
||||||
|
new Set([
|
||||||
|
...extractMessageImageAssetIds(message.parts as any[]),
|
||||||
|
...((contextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||||||
|
]),
|
||||||
|
);
|
||||||
const fallbackSummaryTimeoutMs = getBudgetedTimeout({
|
const fallbackSummaryTimeoutMs = getBudgetedTimeout({
|
||||||
startedAt: requestStartedAt,
|
startedAt: requestStartedAt,
|
||||||
preferredMs: SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS,
|
preferredMs: SMART_AUTO_IMAGE_SUMMARY_TIMEOUT_MS,
|
||||||
@ -2498,6 +2532,7 @@ export async function POST(request: Request) {
|
|||||||
imageUrls: summaryImageUrls,
|
imageUrls: summaryImageUrls,
|
||||||
context: typeof autoContext === "string" ? autoContext : undefined,
|
context: typeof autoContext === "string" ? autoContext : undefined,
|
||||||
preferredModelId: preferredAgentModelId,
|
preferredModelId: preferredAgentModelId,
|
||||||
|
assetIds: summaryAssetIds,
|
||||||
}),
|
}),
|
||||||
"fallbackVisualSummary",
|
"fallbackVisualSummary",
|
||||||
fallbackSummaryTimeoutMs,
|
fallbackSummaryTimeoutMs,
|
||||||
@ -2724,6 +2759,12 @@ export async function POST(request: Request) {
|
|||||||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||||||
.map((part) => part.text)
|
.map((part) => part.text)
|
||||||
.join(" ");
|
.join(" ");
|
||||||
|
const summaryAssetIds = Array.from(
|
||||||
|
new Set([
|
||||||
|
...extractMessageImageAssetIds(message.parts as any[]),
|
||||||
|
...((contextPayload?.selectedAssetIds || []).filter(Boolean) as string[]),
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
|
||||||
let summary: string | null = null;
|
let summary: string | null = null;
|
||||||
const summaryTimeoutMs = getBudgetedTimeout({
|
const summaryTimeoutMs = getBudgetedTimeout({
|
||||||
@ -2738,6 +2779,7 @@ export async function POST(request: Request) {
|
|||||||
imageUrls: summaryImageUrls,
|
imageUrls: summaryImageUrls,
|
||||||
context: typeof autoContext === "string" ? autoContext : undefined,
|
context: typeof autoContext === "string" ? autoContext : undefined,
|
||||||
preferredModelId: preferredAgentModelId,
|
preferredModelId: preferredAgentModelId,
|
||||||
|
assetIds: summaryAssetIds,
|
||||||
}),
|
}),
|
||||||
"enrichImageAnalysisSummary",
|
"enrichImageAnalysisSummary",
|
||||||
summaryTimeoutMs,
|
summaryTimeoutMs,
|
||||||
|
|||||||
@ -51,6 +51,7 @@ export interface AutoAgentInput {
|
|||||||
text: string;
|
text: string;
|
||||||
hasImage: boolean;
|
hasImage: boolean;
|
||||||
imageUrls?: string[];
|
imageUrls?: string[];
|
||||||
|
assetIds?: string[];
|
||||||
imageFromHistory?: boolean;
|
imageFromHistory?: boolean;
|
||||||
historyMessages?: HistoryMessage[];
|
historyMessages?: HistoryMessage[];
|
||||||
context?: string;
|
context?: string;
|
||||||
@ -70,6 +71,7 @@ const AutoAgentState = Annotation.Root({
|
|||||||
text: Annotation<string>,
|
text: Annotation<string>,
|
||||||
hasImage: Annotation<boolean>,
|
hasImage: Annotation<boolean>,
|
||||||
imageUrls: Annotation<string[]>,
|
imageUrls: Annotation<string[]>,
|
||||||
|
assetIds: Annotation<string[]>,
|
||||||
imageFromHistory: Annotation<boolean>,
|
imageFromHistory: Annotation<boolean>,
|
||||||
historyMessages: Annotation<HistoryMessage[]>,
|
historyMessages: Annotation<HistoryMessage[]>,
|
||||||
context: Annotation<string | null>,
|
context: Annotation<string | null>,
|
||||||
@ -1182,6 +1184,7 @@ const analyzeNode = async (state: AutoAgentStateType) => {
|
|||||||
text: state.text,
|
text: state.text,
|
||||||
hasImage: state.hasImage,
|
hasImage: state.hasImage,
|
||||||
imageUrls: state.imageUrls,
|
imageUrls: state.imageUrls,
|
||||||
|
assetIds: state.assetIds,
|
||||||
imageFromHistory: state.imageFromHistory,
|
imageFromHistory: state.imageFromHistory,
|
||||||
historyMessages: state.historyMessages,
|
historyMessages: state.historyMessages,
|
||||||
context: state.context ?? undefined,
|
context: state.context ?? undefined,
|
||||||
@ -1332,6 +1335,7 @@ export async function runAutoAgentGraph(params: AutoAgentInput): Promise<AutoAge
|
|||||||
text: params.text,
|
text: params.text,
|
||||||
hasImage: params.hasImage,
|
hasImage: params.hasImage,
|
||||||
imageUrls: params.imageUrls ?? [],
|
imageUrls: params.imageUrls ?? [],
|
||||||
|
assetIds: params.assetIds ?? [],
|
||||||
imageFromHistory: params.imageFromHistory ?? false,
|
imageFromHistory: params.imageFromHistory ?? false,
|
||||||
historyMessages: params.historyMessages ?? [],
|
historyMessages: params.historyMessages ?? [],
|
||||||
context: params.context ?? null,
|
context: params.context ?? null,
|
||||||
|
|||||||
@ -447,6 +447,7 @@ async function analyzeWithHardcodedQwen3VlFallback(params: {
|
|||||||
text: string;
|
text: string;
|
||||||
hasImage: boolean;
|
hasImage: boolean;
|
||||||
imageUrls?: string[];
|
imageUrls?: string[];
|
||||||
|
assetIds?: string[];
|
||||||
visionImages?: Array<{ type: "image"; image: string }>;
|
visionImages?: Array<{ type: "image"; image: string }>;
|
||||||
historyMessages?: HistoryMessage[];
|
historyMessages?: HistoryMessage[];
|
||||||
context?: string;
|
context?: string;
|
||||||
@ -905,16 +906,155 @@ async function summarizeWithZhipuOfficialApi(params: {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function uniqueNonEmptyStrings(values?: string[] | null): string[] {
|
||||||
|
return Array.from(
|
||||||
|
new Set(
|
||||||
|
(values || [])
|
||||||
|
.map((value) => String(value || "").trim())
|
||||||
|
.filter(Boolean),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractVisualSummarySection(summary: string, label: string): string {
|
||||||
|
const normalized = summary.replace(/\s+/g, " ").trim();
|
||||||
|
if (!normalized) return "";
|
||||||
|
const match = normalized.match(new RegExp(`${label}\\s*[::]\\s*([^;;]+)`, "i"));
|
||||||
|
return match?.[1]?.trim() || "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function tokenizeVisualSummarySection(section: string, maxTags = 6): string[] {
|
||||||
|
const normalized = section
|
||||||
|
.replace(/[()()]/g, " ")
|
||||||
|
.replace(/\s+/g, " ")
|
||||||
|
.trim();
|
||||||
|
if (!normalized) return [];
|
||||||
|
|
||||||
|
const parts = normalized
|
||||||
|
.split(/[、,,/|]/)
|
||||||
|
.map((item) => item.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
|
||||||
|
if (parts.length === 0) {
|
||||||
|
return normalized ? [normalized.slice(0, 80)] : [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return Array.from(new Set(parts)).slice(0, maxTags);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildVisualSummaryMemoryPayload(summary: string) {
|
||||||
|
return {
|
||||||
|
subjectTags: tokenizeVisualSummarySection(extractVisualSummarySection(summary, "画面")),
|
||||||
|
sceneTags: tokenizeVisualSummarySection(extractVisualSummarySection(summary, "场景")),
|
||||||
|
styleTags: tokenizeVisualSummarySection(extractVisualSummarySection(summary, "风格")),
|
||||||
|
lightingTags: tokenizeVisualSummarySection(extractVisualSummarySection(summary, "光线")),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getCachedVisualSummaryByAssetIds(assetIds?: string[]): Promise<string | null> {
|
||||||
|
const uniqueAssetIds = uniqueNonEmptyStrings(assetIds);
|
||||||
|
if (uniqueAssetIds.length !== 1) return null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { getAssetMemorySnapshotsByAssetIds } = await import("@/lib/db/queries");
|
||||||
|
const snapshots = await getAssetMemorySnapshotsByAssetIds({
|
||||||
|
assetIds: uniqueAssetIds,
|
||||||
|
snapshotType: "visual_summary",
|
||||||
|
status: "ready",
|
||||||
|
});
|
||||||
|
const latest = snapshots.find(
|
||||||
|
(snapshot) =>
|
||||||
|
typeof snapshot.summaryText === "string" &&
|
||||||
|
snapshot.summaryText.trim().length > 0,
|
||||||
|
);
|
||||||
|
const normalized = normalizeVisualSummaryText(latest?.summaryText);
|
||||||
|
if (normalized && !isWeakVisualSummary(normalized)) {
|
||||||
|
console.log("[Smart Analyzer] Asset memory cache hit:", uniqueAssetIds[0]);
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[Smart Analyzer] Failed to read asset memory snapshot cache:", error);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function persistVisualSummarySnapshot(params: {
|
||||||
|
assetIds?: string[];
|
||||||
|
summary: string;
|
||||||
|
preferredModelId?: string;
|
||||||
|
text?: string;
|
||||||
|
context?: string;
|
||||||
|
imageUrls?: string[];
|
||||||
|
}) {
|
||||||
|
const uniqueAssetIds = uniqueNonEmptyStrings(params.assetIds);
|
||||||
|
if (uniqueAssetIds.length !== 1) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { getMediaAssetsByIds, upsertAssetMemorySnapshot } = await import("@/lib/db/queries");
|
||||||
|
const assets = await getMediaAssetsByIds({ ids: uniqueAssetIds });
|
||||||
|
const asset = assets.find((item) => item.id === uniqueAssetIds[0]);
|
||||||
|
if (!asset?.userId || !asset.normalizedUrl) return;
|
||||||
|
|
||||||
|
const normalizedSummary = normalizeVisualSummaryText(params.summary);
|
||||||
|
if (!normalizedSummary || isWeakVisualSummary(normalizedSummary)) return;
|
||||||
|
|
||||||
|
const memoryPayload = buildVisualSummaryMemoryPayload(normalizedSummary);
|
||||||
|
await upsertAssetMemorySnapshot({
|
||||||
|
userId: asset.userId,
|
||||||
|
assetId: asset.id,
|
||||||
|
chatId: asset.chatId ?? null,
|
||||||
|
projectId: asset.projectId ?? null,
|
||||||
|
snapshotType: "visual_summary",
|
||||||
|
status: "ready",
|
||||||
|
normalizedUrl: asset.normalizedUrl,
|
||||||
|
summaryText: normalizedSummary,
|
||||||
|
subjectTags: memoryPayload.subjectTags,
|
||||||
|
sceneTags: memoryPayload.sceneTags,
|
||||||
|
styleTags: memoryPayload.styleTags,
|
||||||
|
lightingTags: memoryPayload.lightingTags,
|
||||||
|
confidence: 88,
|
||||||
|
sourceModelId: params.preferredModelId || asset.modelId || null,
|
||||||
|
metadata: {
|
||||||
|
generator: "summarizeImageForExecutionReport",
|
||||||
|
promptText: params.text || null,
|
||||||
|
context: params.context || null,
|
||||||
|
imageUrls: uniqueNonEmptyStrings(params.imageUrls),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[Smart Analyzer] Failed to persist asset memory snapshot:", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function summarizeImageForExecutionReport(params: {
|
export async function summarizeImageForExecutionReport(params: {
|
||||||
text?: string;
|
text?: string;
|
||||||
imageUrls?: string[];
|
imageUrls?: string[];
|
||||||
context?: string;
|
context?: string;
|
||||||
preferredModelId?: string;
|
preferredModelId?: string;
|
||||||
|
assetIds?: string[];
|
||||||
}): Promise<string | null> {
|
}): Promise<string | null> {
|
||||||
const { text, imageUrls, context, preferredModelId } = params;
|
const { text, imageUrls, context, preferredModelId, assetIds } = params;
|
||||||
|
const uniqueAssetIds = uniqueNonEmptyStrings(assetIds);
|
||||||
|
const cachedSummary = await getCachedVisualSummaryByAssetIds(uniqueAssetIds);
|
||||||
|
if (cachedSummary) return cachedSummary;
|
||||||
const visionImages = await buildAnalyzerVisionImages(imageUrls);
|
const visionImages = await buildAnalyzerVisionImages(imageUrls);
|
||||||
if (visionImages.length === 0) return null;
|
if (visionImages.length === 0) return null;
|
||||||
|
|
||||||
|
const finalizeVisualSummary = async (summary?: string | null): Promise<string | null> => {
|
||||||
|
const normalized = normalizeVisualSummaryText(summary);
|
||||||
|
if (!normalized || isWeakVisualSummary(normalized)) return null;
|
||||||
|
await persistVisualSummarySnapshot({
|
||||||
|
assetIds: uniqueAssetIds,
|
||||||
|
summary: normalized,
|
||||||
|
preferredModelId,
|
||||||
|
text,
|
||||||
|
context,
|
||||||
|
imageUrls,
|
||||||
|
});
|
||||||
|
return normalized;
|
||||||
|
};
|
||||||
|
|
||||||
const {
|
const {
|
||||||
getEnabledAiModelsWithProvider,
|
getEnabledAiModelsWithProvider,
|
||||||
getAgentAiModelWithProvider,
|
getAgentAiModelWithProvider,
|
||||||
@ -977,7 +1117,7 @@ export async function summarizeImageForExecutionReport(params: {
|
|||||||
imageUrls,
|
imageUrls,
|
||||||
visionImages,
|
visionImages,
|
||||||
});
|
});
|
||||||
if (forcedSummary) return forcedSummary;
|
if (forcedSummary) return await finalizeVisualSummary(forcedSummary);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (zhipuProvider?.apiKey && zhipuProvider?.baseUrl) {
|
if (zhipuProvider?.apiKey && zhipuProvider?.baseUrl) {
|
||||||
@ -987,7 +1127,7 @@ export async function summarizeImageForExecutionReport(params: {
|
|||||||
promptText,
|
promptText,
|
||||||
visionImages,
|
visionImages,
|
||||||
});
|
});
|
||||||
if (summary) return summary;
|
if (summary) return await finalizeVisualSummary(summary);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (volcanoProvider?.apiKey && volcanoProvider?.baseUrl) {
|
if (volcanoProvider?.apiKey && volcanoProvider?.baseUrl) {
|
||||||
@ -997,7 +1137,7 @@ export async function summarizeImageForExecutionReport(params: {
|
|||||||
promptText,
|
promptText,
|
||||||
imageUrls: (imageUrls || []).filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT),
|
imageUrls: (imageUrls || []).filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT),
|
||||||
});
|
});
|
||||||
if (summary) return summary;
|
if (summary) return await finalizeVisualSummary(summary);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@ -1046,7 +1186,7 @@ export async function summarizeImageForExecutionReport(params: {
|
|||||||
console.log(
|
console.log(
|
||||||
`[Smart Analyzer] Visual summary model used: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`,
|
`[Smart Analyzer] Visual summary model used: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`,
|
||||||
);
|
);
|
||||||
return normalized;
|
return await finalizeVisualSummary(normalized);
|
||||||
}
|
}
|
||||||
if (normalized) {
|
if (normalized) {
|
||||||
console.warn(
|
console.warn(
|
||||||
@ -1072,7 +1212,7 @@ export async function summarizeImageForExecutionReport(params: {
|
|||||||
imageUrls,
|
imageUrls,
|
||||||
visionImages,
|
visionImages,
|
||||||
});
|
});
|
||||||
if (qwenFallbackSummary) return qwenFallbackSummary;
|
if (qwenFallbackSummary) return await finalizeVisualSummary(qwenFallbackSummary);
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@ -1264,12 +1404,13 @@ export async function analyzeIntent(params: {
|
|||||||
text: string;
|
text: string;
|
||||||
hasImage: boolean;
|
hasImage: boolean;
|
||||||
imageUrls?: string[];
|
imageUrls?: string[];
|
||||||
|
assetIds?: string[];
|
||||||
imageFromHistory?: boolean; // 🆕 图片是否来自历史记录
|
imageFromHistory?: boolean; // 🆕 图片是否来自历史记录
|
||||||
historyMessages?: HistoryMessage[]; // 🆕 历史对话消息(用于上下文理解)
|
historyMessages?: HistoryMessage[]; // 🆕 历史对话消息(用于上下文理解)
|
||||||
context?: string; // 🆕 画布上下文(仅用于理解)
|
context?: string; // 🆕 画布上下文(仅用于理解)
|
||||||
preferredModelId?: string;
|
preferredModelId?: string;
|
||||||
}): Promise<AnalysisResult> {
|
}): Promise<AnalysisResult> {
|
||||||
const { text, hasImage, imageUrls, imageFromHistory, historyMessages, context, preferredModelId } = params;
|
const { text, hasImage, imageUrls, assetIds, imageFromHistory, historyMessages, context, preferredModelId } = params;
|
||||||
|
|
||||||
console.log("[Smart Analyzer] Analyzing intent:", {
|
console.log("[Smart Analyzer] Analyzing intent:", {
|
||||||
text,
|
text,
|
||||||
@ -1292,6 +1433,7 @@ export async function analyzeIntent(params: {
|
|||||||
text,
|
text,
|
||||||
hasImage,
|
hasImage,
|
||||||
imageUrls,
|
imageUrls,
|
||||||
|
assetIds,
|
||||||
historyMessages,
|
historyMessages,
|
||||||
context,
|
context,
|
||||||
preferredModelId,
|
preferredModelId,
|
||||||
@ -1330,6 +1472,7 @@ async function analyzWithLLM(
|
|||||||
text: string,
|
text: string,
|
||||||
hasImage: boolean,
|
hasImage: boolean,
|
||||||
imageUrls?: string[],
|
imageUrls?: string[],
|
||||||
|
assetIds?: string[],
|
||||||
historyMessages?: HistoryMessage[],
|
historyMessages?: HistoryMessage[],
|
||||||
context?: string,
|
context?: string,
|
||||||
preferredModelId?: string
|
preferredModelId?: string
|
||||||
@ -1500,6 +1643,7 @@ async function analyzWithLLM(
|
|||||||
text,
|
text,
|
||||||
hasImage,
|
hasImage,
|
||||||
imageUrls,
|
imageUrls,
|
||||||
|
assetIds,
|
||||||
visionImages,
|
visionImages,
|
||||||
historyMessages,
|
historyMessages,
|
||||||
context,
|
context,
|
||||||
@ -1509,11 +1653,17 @@ async function analyzWithLLM(
|
|||||||
|
|
||||||
// 🆕 构建包含历史上下文的消息
|
// 🆕 构建包含历史上下文的消息
|
||||||
let contextMessage = "";
|
let contextMessage = "";
|
||||||
|
const cachedVisualSummary = hasImage
|
||||||
|
? await getCachedVisualSummaryByAssetIds(assetIds)
|
||||||
|
: null;
|
||||||
|
|
||||||
// 🆕 画布上下文(仅用于理解,不要复述)
|
// 🆕 画布上下文(仅用于理解,不要复述)
|
||||||
if (context && context.trim().length > 0) {
|
if (context && context.trim().length > 0) {
|
||||||
contextMessage = `## 画布上下文(仅用于理解,不要在对话回复中复述):\n${context.trim()}\n\n`;
|
contextMessage = `## 画布上下文(仅用于理解,不要在对话回复中复述):\n${context.trim()}\n\n`;
|
||||||
}
|
}
|
||||||
|
if (cachedVisualSummary) {
|
||||||
|
contextMessage += `## 已缓存图片摘要(可直接作为视觉上下文):\n${cachedVisualSummary}\n\n`;
|
||||||
|
}
|
||||||
|
|
||||||
// 添加历史对话上下文(最多取最近5轮对话)
|
// 添加历史对话上下文(最多取最近5轮对话)
|
||||||
if (historyMessages && historyMessages.length > 0) {
|
if (historyMessages && historyMessages.length > 0) {
|
||||||
@ -1550,6 +1700,7 @@ ${historyText}
|
|||||||
const forceSingleMultimodalAgent = Boolean(hasImage && multimodalAgentVisionModel);
|
const forceSingleMultimodalAgent = Boolean(hasImage && multimodalAgentVisionModel);
|
||||||
|
|
||||||
if (
|
if (
|
||||||
|
!cachedVisualSummary &&
|
||||||
!forceSingleMultimodalAgent &&
|
!forceSingleMultimodalAgent &&
|
||||||
hasImage &&
|
hasImage &&
|
||||||
visionImages.length > 0 &&
|
visionImages.length > 0 &&
|
||||||
@ -1579,6 +1730,7 @@ ${historyText}
|
|||||||
|
|
||||||
// 调用模型分析(有图时携带图片,确保真实视觉分析)
|
// 调用模型分析(有图时携带图片,确保真实视觉分析)
|
||||||
const requestPayload = hasImage && visionImages.length > 0
|
const requestPayload = hasImage && visionImages.length > 0
|
||||||
|
&& !cachedVisualSummary
|
||||||
? {
|
? {
|
||||||
model,
|
model,
|
||||||
system: ANALYZER_SYSTEM_PROMPT,
|
system: ANALYZER_SYSTEM_PROMPT,
|
||||||
@ -1615,6 +1767,7 @@ ${historyText}
|
|||||||
text,
|
text,
|
||||||
hasImage,
|
hasImage,
|
||||||
imageUrls,
|
imageUrls,
|
||||||
|
assetIds,
|
||||||
visionImages,
|
visionImages,
|
||||||
historyMessages,
|
historyMessages,
|
||||||
context,
|
context,
|
||||||
@ -1633,6 +1786,7 @@ ${historyText}
|
|||||||
text,
|
text,
|
||||||
hasImage,
|
hasImage,
|
||||||
imageUrls,
|
imageUrls,
|
||||||
|
assetIds,
|
||||||
visionImages,
|
visionImages,
|
||||||
historyMessages,
|
historyMessages,
|
||||||
context,
|
context,
|
||||||
|
|||||||
56
lib/db/migrations/0052_add_asset_memory_snapshots.sql
Normal file
56
lib/db/migrations/0052_add_asset_memory_snapshots.sql
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
CREATE TABLE IF NOT EXISTS "AssetMemorySnapshot" (
|
||||||
|
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
|
||||||
|
"userId" uuid NOT NULL,
|
||||||
|
"assetId" uuid,
|
||||||
|
"chatId" uuid,
|
||||||
|
"projectId" uuid,
|
||||||
|
"snapshotType" varchar(32) DEFAULT 'visual_summary' NOT NULL,
|
||||||
|
"status" varchar(32) DEFAULT 'ready' NOT NULL,
|
||||||
|
"normalizedUrl" text NOT NULL,
|
||||||
|
"summaryText" text,
|
||||||
|
"subjectTags" jsonb DEFAULT '[]'::jsonb NOT NULL,
|
||||||
|
"sceneTags" jsonb DEFAULT '[]'::jsonb NOT NULL,
|
||||||
|
"styleTags" jsonb DEFAULT '[]'::jsonb NOT NULL,
|
||||||
|
"lightingTags" jsonb DEFAULT '[]'::jsonb NOT NULL,
|
||||||
|
"confidence" integer DEFAULT 0 NOT NULL,
|
||||||
|
"sourceModelId" varchar(128),
|
||||||
|
"metadata" jsonb,
|
||||||
|
"createdAt" timestamp DEFAULT now() NOT NULL,
|
||||||
|
"updatedAt" timestamp DEFAULT now() NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
DO $$ BEGIN
|
||||||
|
ALTER TABLE "AssetMemorySnapshot" ADD CONSTRAINT "AssetMemorySnapshot_userId_User_id_fk" FOREIGN KEY ("userId") REFERENCES "public"."User"("id") ON DELETE cascade ON UPDATE no action;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN duplicate_object THEN null;
|
||||||
|
END $$;
|
||||||
|
--> statement-breakpoint
|
||||||
|
DO $$ BEGIN
|
||||||
|
ALTER TABLE "AssetMemorySnapshot" ADD CONSTRAINT "AssetMemorySnapshot_assetId_MediaAsset_id_fk" FOREIGN KEY ("assetId") REFERENCES "public"."MediaAsset"("id") ON DELETE cascade ON UPDATE no action;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN duplicate_object THEN null;
|
||||||
|
END $$;
|
||||||
|
--> statement-breakpoint
|
||||||
|
DO $$ BEGIN
|
||||||
|
ALTER TABLE "AssetMemorySnapshot" ADD CONSTRAINT "AssetMemorySnapshot_chatId_Chat_id_fk" FOREIGN KEY ("chatId") REFERENCES "public"."Chat"("id") ON DELETE cascade ON UPDATE no action;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN duplicate_object THEN null;
|
||||||
|
END $$;
|
||||||
|
--> statement-breakpoint
|
||||||
|
DO $$ BEGIN
|
||||||
|
ALTER TABLE "AssetMemorySnapshot" ADD CONSTRAINT "AssetMemorySnapshot_projectId_CanvasProject_id_fk" FOREIGN KEY ("projectId") REFERENCES "public"."CanvasProject"("id") ON DELETE cascade ON UPDATE no action;
|
||||||
|
EXCEPTION
|
||||||
|
WHEN duplicate_object THEN null;
|
||||||
|
END $$;
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS "AssetMemorySnapshot_user_url_type_uidx"
|
||||||
|
ON "AssetMemorySnapshot"("userId", "normalizedUrl", "snapshotType");
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX IF NOT EXISTS "AssetMemorySnapshot_asset_type_idx"
|
||||||
|
ON "AssetMemorySnapshot"("assetId", "snapshotType");
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX IF NOT EXISTS "AssetMemorySnapshot_chat_created_idx"
|
||||||
|
ON "AssetMemorySnapshot"("chatId", "createdAt");
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX IF NOT EXISTS "AssetMemorySnapshot_project_created_idx"
|
||||||
|
ON "AssetMemorySnapshot"("projectId", "createdAt");
|
||||||
@ -267,6 +267,13 @@
|
|||||||
"when": 1772792400000,
|
"when": 1772792400000,
|
||||||
"tag": "0051_add_media_assets",
|
"tag": "0051_add_media_assets",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 38,
|
||||||
|
"version": "7",
|
||||||
|
"when": 1772796000000,
|
||||||
|
"tag": "0052_add_asset_memory_snapshots",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@ -60,6 +60,8 @@ import {
|
|||||||
type MediaAsset,
|
type MediaAsset,
|
||||||
mediaAssetRelation,
|
mediaAssetRelation,
|
||||||
type MediaAssetRelation,
|
type MediaAssetRelation,
|
||||||
|
assetMemorySnapshot,
|
||||||
|
type AssetMemorySnapshot,
|
||||||
canvasNode,
|
canvasNode,
|
||||||
canvasProject,
|
canvasProject,
|
||||||
} from "./schema";
|
} from "./schema";
|
||||||
@ -985,6 +987,28 @@ export async function getMediaAssetsByUserAndUrls(params: {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function getMediaAssetsByIds(params: {
|
||||||
|
ids: string[];
|
||||||
|
}): Promise<MediaAsset[]> {
|
||||||
|
const ids = Array.from(
|
||||||
|
new Set(
|
||||||
|
(params.ids || [])
|
||||||
|
.map((id) => String(id || "").trim())
|
||||||
|
.filter(Boolean),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
if (ids.length === 0) return [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await db.select().from(mediaAsset).where(inArray(mediaAsset.id, ids));
|
||||||
|
} catch (_error) {
|
||||||
|
throw new ChatSDKError(
|
||||||
|
"bad_request:database",
|
||||||
|
"Failed to get media assets by ids"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function upsertMediaAsset(params: {
|
export async function upsertMediaAsset(params: {
|
||||||
userId: string;
|
userId: string;
|
||||||
chatId?: string | null;
|
chatId?: string | null;
|
||||||
@ -1126,6 +1150,160 @@ export async function createMediaAssetRelation(params: {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function getAssetMemorySnapshotsByAssetIds(params: {
|
||||||
|
assetIds: string[];
|
||||||
|
snapshotType?: string | null;
|
||||||
|
status?: string | null;
|
||||||
|
}): Promise<AssetMemorySnapshot[]> {
|
||||||
|
const assetIds = Array.from(
|
||||||
|
new Set(
|
||||||
|
(params.assetIds || [])
|
||||||
|
.map((id) => String(id || "").trim())
|
||||||
|
.filter(Boolean),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
if (assetIds.length === 0) return [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
const conditions = [inArray(assetMemorySnapshot.assetId, assetIds)];
|
||||||
|
if (params.snapshotType) {
|
||||||
|
conditions.push(eq(assetMemorySnapshot.snapshotType, params.snapshotType));
|
||||||
|
}
|
||||||
|
if (params.status) {
|
||||||
|
conditions.push(eq(assetMemorySnapshot.status, params.status));
|
||||||
|
}
|
||||||
|
|
||||||
|
return await db
|
||||||
|
.select()
|
||||||
|
.from(assetMemorySnapshot)
|
||||||
|
.where(and(...conditions))
|
||||||
|
.orderBy(desc(assetMemorySnapshot.updatedAt));
|
||||||
|
} catch (_error) {
|
||||||
|
throw new ChatSDKError(
|
||||||
|
"bad_request:database",
|
||||||
|
"Failed to get asset memory snapshots by asset ids"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getAssetMemorySnapshotsByUserAndUrls(params: {
|
||||||
|
userId: string;
|
||||||
|
urls: string[];
|
||||||
|
snapshotType?: string | null;
|
||||||
|
status?: string | null;
|
||||||
|
}): Promise<AssetMemorySnapshot[]> {
|
||||||
|
const normalizedUrls = Array.from(
|
||||||
|
new Set(
|
||||||
|
(params.urls || [])
|
||||||
|
.map((url) => normalizeMediaUrl(String(url || "").trim()))
|
||||||
|
.filter(Boolean),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
if (!params.userId || normalizedUrls.length === 0) return [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
const conditions = [
|
||||||
|
eq(assetMemorySnapshot.userId, params.userId),
|
||||||
|
inArray(assetMemorySnapshot.normalizedUrl, normalizedUrls),
|
||||||
|
];
|
||||||
|
if (params.snapshotType) {
|
||||||
|
conditions.push(eq(assetMemorySnapshot.snapshotType, params.snapshotType));
|
||||||
|
}
|
||||||
|
if (params.status) {
|
||||||
|
conditions.push(eq(assetMemorySnapshot.status, params.status));
|
||||||
|
}
|
||||||
|
|
||||||
|
return await db
|
||||||
|
.select()
|
||||||
|
.from(assetMemorySnapshot)
|
||||||
|
.where(and(...conditions))
|
||||||
|
.orderBy(desc(assetMemorySnapshot.updatedAt));
|
||||||
|
} catch (_error) {
|
||||||
|
throw new ChatSDKError(
|
||||||
|
"bad_request:database",
|
||||||
|
"Failed to get asset memory snapshots by urls"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function upsertAssetMemorySnapshot(params: {
|
||||||
|
userId: string;
|
||||||
|
assetId?: string | null;
|
||||||
|
chatId?: string | null;
|
||||||
|
projectId?: string | null;
|
||||||
|
snapshotType?: string | null;
|
||||||
|
status?: string | null;
|
||||||
|
normalizedUrl: string;
|
||||||
|
summaryText?: string | null;
|
||||||
|
subjectTags?: string[] | null;
|
||||||
|
sceneTags?: string[] | null;
|
||||||
|
styleTags?: string[] | null;
|
||||||
|
lightingTags?: string[] | null;
|
||||||
|
confidence?: number | null;
|
||||||
|
sourceModelId?: string | null;
|
||||||
|
metadata?: any;
|
||||||
|
}): Promise<AssetMemorySnapshot | null> {
|
||||||
|
const normalizedUrl = normalizeMediaUrl(String(params.normalizedUrl || "").trim());
|
||||||
|
if (!params.userId || !normalizedUrl) return null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const now = new Date();
|
||||||
|
const snapshotType = String(params.snapshotType || "visual_summary").trim() || "visual_summary";
|
||||||
|
const status = String(params.status || "ready").trim() || "ready";
|
||||||
|
const [result] = await db
|
||||||
|
.insert(assetMemorySnapshot)
|
||||||
|
.values({
|
||||||
|
userId: params.userId,
|
||||||
|
assetId: params.assetId ?? null,
|
||||||
|
chatId: params.chatId ?? null,
|
||||||
|
projectId: params.projectId ?? null,
|
||||||
|
snapshotType,
|
||||||
|
status,
|
||||||
|
normalizedUrl,
|
||||||
|
summaryText: params.summaryText ?? null,
|
||||||
|
subjectTags: normalizeStringArray(params.subjectTags),
|
||||||
|
sceneTags: normalizeStringArray(params.sceneTags),
|
||||||
|
styleTags: normalizeStringArray(params.styleTags),
|
||||||
|
lightingTags: normalizeStringArray(params.lightingTags),
|
||||||
|
confidence: Math.max(0, Math.min(100, Math.round(params.confidence ?? 0))),
|
||||||
|
sourceModelId: params.sourceModelId ?? null,
|
||||||
|
metadata: params.metadata ?? null,
|
||||||
|
createdAt: now,
|
||||||
|
updatedAt: now,
|
||||||
|
})
|
||||||
|
.onConflictDoUpdate({
|
||||||
|
target: [
|
||||||
|
assetMemorySnapshot.userId,
|
||||||
|
assetMemorySnapshot.normalizedUrl,
|
||||||
|
assetMemorySnapshot.snapshotType,
|
||||||
|
],
|
||||||
|
set: {
|
||||||
|
assetId: params.assetId ?? undefined,
|
||||||
|
chatId: params.chatId ?? undefined,
|
||||||
|
projectId: params.projectId ?? undefined,
|
||||||
|
status,
|
||||||
|
summaryText: params.summaryText ?? null,
|
||||||
|
subjectTags: normalizeStringArray(params.subjectTags),
|
||||||
|
sceneTags: normalizeStringArray(params.sceneTags),
|
||||||
|
styleTags: normalizeStringArray(params.styleTags),
|
||||||
|
lightingTags: normalizeStringArray(params.lightingTags),
|
||||||
|
confidence: Math.max(0, Math.min(100, Math.round(params.confidence ?? 0))),
|
||||||
|
sourceModelId: params.sourceModelId ?? null,
|
||||||
|
metadata: params.metadata ?? null,
|
||||||
|
updatedAt: now,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.returning();
|
||||||
|
|
||||||
|
return result ?? null;
|
||||||
|
} catch (_error) {
|
||||||
|
throw new ChatSDKError(
|
||||||
|
"bad_request:database",
|
||||||
|
"Failed to upsert asset memory snapshot"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const UUID_REGEX =
|
const UUID_REGEX =
|
||||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||||
|
|
||||||
|
|||||||
@ -810,3 +810,35 @@ export const mediaAssetRelation = pgTable("MediaAssetRelation", {
|
|||||||
});
|
});
|
||||||
|
|
||||||
export type MediaAssetRelation = InferSelectModel<typeof mediaAssetRelation>;
|
export type MediaAssetRelation = InferSelectModel<typeof mediaAssetRelation>;
|
||||||
|
|
||||||
|
// 资产记忆快照表(视觉摘要 / 主体标签 / 场景标签等可复用缓存)
|
||||||
|
export const assetMemorySnapshot = pgTable("AssetMemorySnapshot", {
|
||||||
|
id: uuid("id").primaryKey().notNull().defaultRandom(),
|
||||||
|
userId: uuid("userId")
|
||||||
|
.notNull()
|
||||||
|
.references(() => user.id, { onDelete: "cascade" }),
|
||||||
|
assetId: uuid("assetId").references(() => mediaAsset.id, {
|
||||||
|
onDelete: "cascade",
|
||||||
|
}),
|
||||||
|
chatId: uuid("chatId").references(() => chat.id, { onDelete: "cascade" }),
|
||||||
|
projectId: uuid("projectId").references(() => canvasProject.id, {
|
||||||
|
onDelete: "cascade",
|
||||||
|
}),
|
||||||
|
snapshotType: varchar("snapshotType", { length: 32 })
|
||||||
|
.notNull()
|
||||||
|
.default("visual_summary"),
|
||||||
|
status: varchar("status", { length: 32 }).notNull().default("ready"),
|
||||||
|
normalizedUrl: text("normalizedUrl").notNull(),
|
||||||
|
summaryText: text("summaryText"),
|
||||||
|
subjectTags: jsonb("subjectTags").$type<string[]>().notNull().default([]),
|
||||||
|
sceneTags: jsonb("sceneTags").$type<string[]>().notNull().default([]),
|
||||||
|
styleTags: jsonb("styleTags").$type<string[]>().notNull().default([]),
|
||||||
|
lightingTags: jsonb("lightingTags").$type<string[]>().notNull().default([]),
|
||||||
|
confidence: integer("confidence").notNull().default(0),
|
||||||
|
sourceModelId: varchar("sourceModelId", { length: 128 }),
|
||||||
|
metadata: jsonb("metadata"),
|
||||||
|
createdAt: timestamp("createdAt").notNull().defaultNow(),
|
||||||
|
updatedAt: timestamp("updatedAt").notNull().defaultNow(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type AssetMemorySnapshot = InferSelectModel<typeof assetMemorySnapshot>;
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user