fix: resolve unselected canvas image targets
This commit is contained in:
parent
eab3f0372f
commit
6a2b928080
@ -857,6 +857,18 @@ function isInternalCanvasImageGenerationFailureText(parts: Array<any>) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getCanvasImageNodeUrl(node: CanvasNode | null | undefined) {
|
||||||
|
if (!node || node.type !== "image") return "";
|
||||||
|
const data = (node.data || {}) as Record<string, any>;
|
||||||
|
return String(
|
||||||
|
data.src ||
|
||||||
|
data.previewImage ||
|
||||||
|
data.imageUrl ||
|
||||||
|
data.renderArtifacts?.screenshotUrl ||
|
||||||
|
""
|
||||||
|
).trim();
|
||||||
|
}
|
||||||
|
|
||||||
function findCanvasImageAnchorNode(
|
function findCanvasImageAnchorNode(
|
||||||
nodes: CanvasNode[],
|
nodes: CanvasNode[],
|
||||||
params: {
|
params: {
|
||||||
@ -873,7 +885,7 @@ function findCanvasImageAnchorNode(
|
|||||||
if (sourceKeys.size > 0) {
|
if (sourceKeys.size > 0) {
|
||||||
const matchedNode = nodes.find((node) => {
|
const matchedNode = nodes.find((node) => {
|
||||||
if (node.type !== "image") return false;
|
if (node.type !== "image") return false;
|
||||||
const src = String((node.data as any)?.src || "").trim();
|
const src = getCanvasImageNodeUrl(node);
|
||||||
return sourceKeys.has(normalizeImageIdentityUrl(src));
|
return sourceKeys.has(normalizeImageIdentityUrl(src));
|
||||||
});
|
});
|
||||||
if (matchedNode) return matchedNode;
|
if (matchedNode) return matchedNode;
|
||||||
@ -7871,7 +7883,7 @@ export function CanvasChatPanel({
|
|||||||
pendingTarget: pendingCanvasAgentAnnotationEditTargetRef.current,
|
pendingTarget: pendingCanvasAgentAnnotationEditTargetRef.current,
|
||||||
isEligibleNode: (node) =>
|
isEligibleNode: (node) =>
|
||||||
node.type === "image" &&
|
node.type === "image" &&
|
||||||
isValidChatHttpUrl(String((node.data as any)?.src || "").trim()),
|
isValidChatHttpUrl(getCanvasImageNodeUrl(node)),
|
||||||
});
|
});
|
||||||
if (
|
if (
|
||||||
pendingCanvasAgentAnnotationEditTargetRef.current &&
|
pendingCanvasAgentAnnotationEditTargetRef.current &&
|
||||||
@ -7888,7 +7900,7 @@ export function CanvasChatPanel({
|
|||||||
const imageNodes = nodes.filter(
|
const imageNodes = nodes.filter(
|
||||||
(node) =>
|
(node) =>
|
||||||
node.type === "image" &&
|
node.type === "image" &&
|
||||||
isValidChatHttpUrl(String((node.data as any)?.src || "").trim()),
|
isValidChatHttpUrl(getCanvasImageNodeUrl(node)),
|
||||||
);
|
);
|
||||||
|
|
||||||
const effectiveComposerMode = activeComposerModeRef.current || activeComposerMode;
|
const effectiveComposerMode = activeComposerModeRef.current || activeComposerMode;
|
||||||
@ -7923,7 +7935,7 @@ export function CanvasChatPanel({
|
|||||||
new Map(
|
new Map(
|
||||||
[
|
[
|
||||||
...selectedImageNodes.filter((node) =>
|
...selectedImageNodes.filter((node) =>
|
||||||
isValidChatHttpUrl(String((node.data as any)?.src || "").trim())
|
isValidChatHttpUrl(getCanvasImageNodeUrl(node))
|
||||||
),
|
),
|
||||||
...(pendingAnnotationTargetNode ? [pendingAnnotationTargetNode] : []),
|
...(pendingAnnotationTargetNode ? [pendingAnnotationTargetNode] : []),
|
||||||
].map((node) => [node.id, node])
|
].map((node) => [node.id, node])
|
||||||
@ -8201,7 +8213,7 @@ export function CanvasChatPanel({
|
|||||||
assetId: isValidChatUuid((node.data as any)?.assetId)
|
assetId: isValidChatUuid((node.data as any)?.assetId)
|
||||||
? String((node.data as any).assetId).trim()
|
? String((node.data as any).assetId).trim()
|
||||||
: undefined,
|
: undefined,
|
||||||
url: String((node.data as any)?.src || "").trim(),
|
url: getCanvasImageNodeUrl(node),
|
||||||
title:
|
title:
|
||||||
typeof (node.data as any)?.title === "string"
|
typeof (node.data as any)?.title === "string"
|
||||||
? String((node.data as any).title).slice(0, 256)
|
? String((node.data as any).title).slice(0, 256)
|
||||||
|
|||||||
@ -308,6 +308,7 @@ export function resolveChatMediaInput(
|
|||||||
resolvedImageRefs,
|
resolvedImageRefs,
|
||||||
currentMessageImageUrls: currentMessageImageRefs.map((ref) => ref.url),
|
currentMessageImageUrls: currentMessageImageRefs.map((ref) => ref.url),
|
||||||
selectedImageUrls: selectedImageRefs.map((ref) => ref.url),
|
selectedImageUrls: selectedImageRefs.map((ref) => ref.url),
|
||||||
|
canvasImageUrls: canvasImageRefs.map((ref) => ref.url),
|
||||||
historyUserImageUrls: history.latestUser.map((ref) => ref.url),
|
historyUserImageUrls: history.latestUser.map((ref) => ref.url),
|
||||||
historyAssistantImageUrls: history.latestAssistant.map((ref) => ref.url),
|
historyAssistantImageUrls: history.latestAssistant.map((ref) => ref.url),
|
||||||
resolvedImageUrls: resolvedImageRefs.map((ref) => ref.url),
|
resolvedImageUrls: resolvedImageRefs.map((ref) => ref.url),
|
||||||
|
|||||||
@ -497,6 +497,21 @@ export function resolveTargetAssets(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
canvasRefs.length === 1 &&
|
||||||
|
(hasCanvasAgentActionableImageEditIntent(text) || focusPolicy === "auto")
|
||||||
|
) {
|
||||||
|
return buildResolvedTargetAssets({
|
||||||
|
targetRefs: [canvasRefs[0]],
|
||||||
|
candidateRefs,
|
||||||
|
resolutionMode: "text_reference",
|
||||||
|
confidence: 84,
|
||||||
|
needsClarification: false,
|
||||||
|
reason: "用户使用画布/图片指代表达,且当前画布仅有一张可处理图片",
|
||||||
|
fromHistory: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const allowsImplicitHistoryTarget =
|
const allowsImplicitHistoryTarget =
|
||||||
IMPLICIT_HISTORY_TARGET_REGEX.test(text) &&
|
IMPLICIT_HISTORY_TARGET_REGEX.test(text) &&
|
||||||
(hasCanvasAgentActionableImageEditIntent(text) ||
|
(hasCanvasAgentActionableImageEditIntent(text) ||
|
||||||
@ -559,6 +574,18 @@ export function resolveTargetAssets(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (hasCanvasAgentActionableImageEditIntent(text)) {
|
if (hasCanvasAgentActionableImageEditIntent(text)) {
|
||||||
|
if (canvasRefs.length === 1) {
|
||||||
|
return buildResolvedTargetAssets({
|
||||||
|
targetRefs: [canvasRefs[0]],
|
||||||
|
candidateRefs,
|
||||||
|
resolutionMode: "text_reference",
|
||||||
|
confidence: 82,
|
||||||
|
needsClarification: false,
|
||||||
|
reason: "用户表达了图片编辑意图,且当前画布仅有一张可处理图片",
|
||||||
|
fromHistory: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
return buildResolvedTargetAssets({
|
return buildResolvedTargetAssets({
|
||||||
targetRefs: [],
|
targetRefs: [],
|
||||||
candidateRefs,
|
candidateRefs,
|
||||||
|
|||||||
@ -57,6 +57,7 @@ export interface ResolvedChatMediaInput {
|
|||||||
resolvedImageRefs: ResolvedImageRef[];
|
resolvedImageRefs: ResolvedImageRef[];
|
||||||
currentMessageImageUrls: string[];
|
currentMessageImageUrls: string[];
|
||||||
selectedImageUrls: string[];
|
selectedImageUrls: string[];
|
||||||
|
canvasImageUrls: string[];
|
||||||
historyUserImageUrls: string[];
|
historyUserImageUrls: string[];
|
||||||
historyAssistantImageUrls: string[];
|
historyAssistantImageUrls: string[];
|
||||||
resolvedImageUrls: string[];
|
resolvedImageUrls: string[];
|
||||||
|
|||||||
@ -653,6 +653,7 @@ export function pickAgentSourceImageUrls(params: {
|
|||||||
params.requestTargetResolution?.targetImageUrls || [],
|
params.requestTargetResolution?.targetImageUrls || [],
|
||||||
params.requestResolvedMediaInput?.selectedImageUrls || [],
|
params.requestResolvedMediaInput?.selectedImageUrls || [],
|
||||||
params.requestResolvedMediaInput?.currentMessageImageUrls || [],
|
params.requestResolvedMediaInput?.currentMessageImageUrls || [],
|
||||||
|
params.requestResolvedMediaInput?.canvasImageUrls || [],
|
||||||
params.requestWorkingState?.lastResolvedTargetImageUrls || [],
|
params.requestWorkingState?.lastResolvedTargetImageUrls || [],
|
||||||
params.requestWorkingState?.lastGeneratedImageUrls || [],
|
params.requestWorkingState?.lastGeneratedImageUrls || [],
|
||||||
params.requestResolvedMediaInput?.historyAssistantImageUrls || [],
|
params.requestResolvedMediaInput?.historyAssistantImageUrls || [],
|
||||||
|
|||||||
@ -29,6 +29,13 @@ export function isCanvasImageReadyForAi(node: CanvasNode | null | undefined) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
const src = String(((node.data as Record<string, any>)?.src || "")).trim();
|
const data = (node.data || {}) as Record<string, any>;
|
||||||
|
const src = String(
|
||||||
|
data.src ||
|
||||||
|
data.previewImage ||
|
||||||
|
data.imageUrl ||
|
||||||
|
data.renderArtifacts?.screenshotUrl ||
|
||||||
|
""
|
||||||
|
).trim();
|
||||||
return /^https?:\/\//i.test(src);
|
return /^https?:\/\//i.test(src);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1339,6 +1339,66 @@ test.describe("canvas chat pure helpers", () => {
|
|||||||
expect(directContext?.targetImageUrl).toBe(latestResultUrl);
|
expect(directContext?.targetImageUrl).toBe(latestResultUrl);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("falls back to canvas images for direct agent edit tools", () => {
|
||||||
|
const canvasImageUrl = "https://cdn.example.com/canvas-cat.png";
|
||||||
|
const mediaInput = {
|
||||||
|
imageRefs: [],
|
||||||
|
currentMessageImageRefs: [],
|
||||||
|
selectedImageRefs: [],
|
||||||
|
canvasImageRefs: [
|
||||||
|
{
|
||||||
|
url: canvasImageUrl,
|
||||||
|
normalizedUrl: canvasImageUrl,
|
||||||
|
nodeId: "cat-node",
|
||||||
|
source: "canvas_context",
|
||||||
|
role: "reference",
|
||||||
|
x: 120,
|
||||||
|
y: 80,
|
||||||
|
width: 640,
|
||||||
|
height: 640,
|
||||||
|
isSelected: false,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
historyUserImageRefs: [],
|
||||||
|
historyAssistantImageRefs: [],
|
||||||
|
resolvedImageRefs: [],
|
||||||
|
currentMessageImageUrls: [],
|
||||||
|
selectedImageUrls: [],
|
||||||
|
canvasImageUrls: [canvasImageUrl],
|
||||||
|
historyUserImageUrls: [],
|
||||||
|
historyAssistantImageUrls: [],
|
||||||
|
resolvedImageUrls: [],
|
||||||
|
selectedAssetIds: [],
|
||||||
|
candidateAssetIds: [],
|
||||||
|
uploadedAssetIds: [],
|
||||||
|
hasImage: false,
|
||||||
|
source: "none",
|
||||||
|
fromHistory: false,
|
||||||
|
} as any;
|
||||||
|
|
||||||
|
const directContext = resolveDirectAgentToolContext({
|
||||||
|
execution: {
|
||||||
|
mode: "edit_image",
|
||||||
|
routeIntent: "image_edit",
|
||||||
|
routeSubIntent: "remove_background",
|
||||||
|
targetSource: "text_reference",
|
||||||
|
preferredCapability: "remove_background",
|
||||||
|
shouldUseStructuredReply: true,
|
||||||
|
shouldOptimizePrompt: true,
|
||||||
|
shouldApplyImagePipeline: true,
|
||||||
|
shouldApplyVideoPipeline: false,
|
||||||
|
shouldStayConversational: false,
|
||||||
|
shouldPublishToCanvas: false,
|
||||||
|
toolCalls: [],
|
||||||
|
availableTools: ["removeBackground"],
|
||||||
|
} as any,
|
||||||
|
requestResolvedMediaInput: mediaInput,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(directContext?.execution.toolName).toBe("removeBackground");
|
||||||
|
expect(directContext?.targetImageUrl).toBe(canvasImageUrl);
|
||||||
|
});
|
||||||
|
|
||||||
test("only runs direct image edit batches for explicit safe batch requests", () => {
|
test("only runs direct image edit batches for explicit safe batch requests", () => {
|
||||||
expect(
|
expect(
|
||||||
shouldRunDirectAgentBatchImageEdit({
|
shouldRunDirectAgentBatchImageEdit({
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user