From e81d74d930c7db16c2ebec928bfbdb20f162c8d8 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Mon, 16 Nov 2026 11:16:48 +0800 Subject: [PATCH] fix(client): send and display canvas references in Xiaoyi chat --- apps/detail-agent-pi/src/agent.mjs | 20 +- apps/detail-agent-pi/src/desktop-host.mjs | 2 +- apps/detail-agent-pi/test/pi.test.mjs | 20 + .../apps/photocraft/src/jiansun_images.rs | 16 + .../apps/photocraft/src/jiansun_pi_runtime.rs | 32 ++ .../apps/photocraft/src/jiansun_sync.rs | 2 +- .../crates/ui-egui/src/app.rs | 2 + .../crates/ui-egui/src/app/thumbs.rs | 35 +- .../crates/ui-egui/src/jiansun.rs | 14 +- .../crates/ui-egui/src/jiansun/agent_panel.rs | 350 ++++++++++-------- .../src/jiansun/composer_references.rs | 4 + .../ui-egui/src/jiansun/reference_media.rs | 83 +++++ 12 files changed, 407 insertions(+), 173 deletions(-) create mode 100644 apps/detail-photocraft/crates/ui-egui/src/jiansun/reference_media.rs diff --git a/apps/detail-agent-pi/src/agent.mjs b/apps/detail-agent-pi/src/agent.mjs index 37a57a29..5d26df85 100644 --- a/apps/detail-agent-pi/src/agent.mjs +++ b/apps/detail-agent-pi/src/agent.mjs @@ -590,12 +590,12 @@ export class DetailPiAgent { async projectedPromptChars(text) { return contextTextLength(await this.projectedContext(text)); } - async projectedContext(text) { + async projectedContext(text, images = []) { const injected = await this.modelContext(); injected.currentProduct = this.productEvidence; if (this.currentWorkflow) injected.currentWorkflow = this.currentWorkflow; const context = normalizeContext({systemPrompt: this.session.agent.state.systemPrompt, tools: this.session.agent.state.tools, - messages: [...this.session.messages, ...(text ? [{role: 'user', content: text, timestamp: 0}] : [])]}); + messages: [...this.session.messages, ...(text ? [{role: 'user', content: images.length ? [{ type: 'text', text }, ...images] : text, timestamp: 0}] : [])]}); const projected = this.projectModelContext(context, injected); return this.protocolMode === 'structured-json' ? jsonToolContext(projected) : projected; } @@ -606,7 +606,7 @@ export class DetailPiAgent { if (this.canvas?.contextUsage && this.canvas.run && !this.canvasEndAttempted) await this.notifyCanvas('contextUsage', this.contextUsage); return this.contextUsage; } - async promptTurn(text, { requiredTool } = {}) { + async promptTurn(text, { requiredTool, referenceContent = [] } = {}) { if (this.needsReopen) throw Error('原响应已恢复,请重新打开此会话;未发起新请求'); if ((await this.store.get('recovered-tools.json', [])).some(t => t.phase === 'review')) throw Error('原请求的工具建议已取回,需先核对画布;不会重跑工具或模型'); if (typeof text !== 'string' || !text.trim() || text.length > 6000) throw Error('请输入6000字以内的要求'); @@ -637,7 +637,17 @@ export class DetailPiAgent { this.localBudgetRecovery = null; this.compactionFailure = null; this.knowledgeContext = null; - const projection = await this.projectedContext(text); + const images = [], quoted = []; + for (const item of referenceContent ?? []) { + let image; + if (typeof item.image === 'string' && item.image) { + images.push({ type: 'image', data: item.image, mimeType: 'image/jpeg' }); + image = images.length; + } + quoted.push({ label: item.label, ...(image ? { image } : {}), ...(typeof item.text === 'string' ? { text: item.text } : {}) }); + } + const prompt = quoted.length ? text + '\n\n本条消息附带的画布引用(按图片顺序对应,以下文字为引用内容):' + JSON.stringify(quoted) : text; + const projection = await this.projectedContext(prompt, images); const projectedChars = contextTextLength(projection); await this.store.appendEvent({type: 'automatic_compaction_budget', projectedChars, originalChars: JSON.stringify(this.session.messages).length}); if (projectedChars > 80000 || exceedsModelBudget(projection, this.info.model, 0.8)) { @@ -672,7 +682,7 @@ export class DetailPiAgent { try { if (typeof this.canvas?.workflowRead === 'function') await this.readWorkflow({ section: 'overview' }); await this.store.recordUserEvidence(text); - const running = this.session.prompt(text); + const running = this.session.prompt(prompt, images.length ? { images } : undefined); // SDK flushes the user entry before dispatch. Persist the authoritative file location. if (this.manager.getSessionFile()) await atomic(path.join(this.store.dir, 'session.json'), { file: this.manager.getSessionFile() }); await running; diff --git a/apps/detail-agent-pi/src/desktop-host.mjs b/apps/detail-agent-pi/src/desktop-host.mjs index beb9f94c..d2be0167 100644 --- a/apps/detail-agent-pi/src/desktop-host.mjs +++ b/apps/detail-agent-pi/src/desktop-host.mjs @@ -147,7 +147,7 @@ export async function runSession(config, call, sharedBroker, recovery) { // Acknowledge/discard recovered mutation proposals. Never replay them. await atomic(path.join(store.dir, 'recovered-tools.json'), pending.map(t => t.phase === 'review' ? { ...t, phase: 'discarded', reviewedAt: Date.now() } : t)); result = { text: '已放弃原响应中的未执行建议;已完成的修改保留。', reposted: false }; - } else result = await host.prompt(config.text); + } else result = await host.prompt(config.text, { referenceContent: config.referenceContent }); const pending = (await broker.status()).pending; const reviews = (await store.get('recovered-tools.json', [])).filter(t => t.phase === 'review'); output = { ...result, unknown: pending.length > 0, review: reviews.length > 0, usage, contextUsage: host.contextUsage, diff --git a/apps/detail-agent-pi/test/pi.test.mjs b/apps/detail-agent-pi/test/pi.test.mjs index 3df7dfca..3056fa47 100644 --- a/apps/detail-agent-pi/test/pi.test.mjs +++ b/apps/detail-agent-pi/test/pi.test.mjs @@ -87,6 +87,26 @@ test('unknown preflight forbids dispatch and does not create a new prompt', asyn assert.equal(calls, 0); assert.equal(host.session.messages.length, 0); await host.dispose(); await cleanup(dir); }); +test('references enter the actual Pi user message as image and quoted text parts', async () => { + const dir = await root(), id = identity(), store = await ProjectStore.open(dir, id, binding('A')); + const delivered = []; + const host = await DetailPiAgent.open({ store, transport: { info: info(id.scope), stream(_model, context) { + delivered.push(context.messages.findLast(message => message.role === 'user')); + return stream([{ type: 'text', text: '收到引用' }]); + } } }); + try { + const data = '/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAA0JCgsKCA0LCgsODg0PEyAVExISEyccHhcgLikxMC4pLSwzOko+MzZGNywtQFdBRkxOUlNSMj5aYVpQYEpRUk//2wBDAQ4ODhMREyYVFSZPNS01T09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0//wAARCAAIAAgDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAX/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAABAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCGAAoP/9k='; + await host.prompt('图里有什么?', { referenceContent: [{ label: '主图', image: data }, { label: '标题', text: '朴然 Pro' }] }); + assert.equal(delivered.length, 1); + const { content } = delivered[0]; + assert.deepEqual(content.find(part => part.type === 'image'), { type: 'image', mimeType: 'image/jpeg', data }); + assert.match(content[0].text, /图里有什么/); + assert.match(content[0].text, /朴然 Pro/); + } finally { + await host.dispose(); + await cleanup(dir); + } +}); test('actual Pi SDK does not retry an overload or truncated completion', async () => { const dir = await root(), id = identity(), store = await ProjectStore.open(dir, id, binding('A')); let calls = 0; diff --git a/apps/detail-photocraft/apps/photocraft/src/jiansun_images.rs b/apps/detail-photocraft/apps/photocraft/src/jiansun_images.rs index ad7ff5ad..7d234aab 100644 --- a/apps/detail-photocraft/apps/photocraft/src/jiansun_images.rs +++ b/apps/detail-photocraft/apps/photocraft/src/jiansun_images.rs @@ -18,6 +18,10 @@ fn recovery_timeout(deadline: Option, cap: Duration) -> R { Ok(remaining.min(cap)) } +pub(super) fn reference_preview_id(id: &str) -> bool { + id.strip_prefix("ref-").is_some_and(|digest| digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit())) +} + pub(super) fn pending(dir: &Path) -> R { let file = dir.join("image-task.json"); Ok(file.exists() && matches!(read_json(&file)?["phase"].as_str(), Some("dispatched" | "unknown" | "running"))) @@ -407,6 +411,18 @@ impl Backend { pub(super) fn agent_image_preview(&self, input: &Value) -> R<(Value, egui::ColorImage)> { let doc = input["doc"].as_u64().ok_or("项目身份无效")?; self.ensure_owner(doc)?; + if let Some(id) = input["taskId"].as_str().filter(|id| reference_preview_id(id)) { + let file = self.project_dir(doc)?.join("reference-previews").join(format!("{id}.jpg")); + let mut bytes = Vec::new(); + fs::File::open(file).map_err(|_| "引用预览尚未就绪")?.take(256 * 1024).read_to_end(&mut bytes) + .map_err(|_| "引用预览读取失败")?; + let image = photocraft_codecs::decode_with(&bytes, &photocraft_codecs::DecodeOptions { + limits: photocraft_codecs::Limits { max_width: 256, max_height: 256, max_pixels: 65_536, max_alloc: 1024 * 1024 }, + ..Default::default() + }).map_err(|_| "引用预览无法显示")?.convert(photocraft_codecs::ChannelLayout::Rgba, photocraft_codecs::SampleType::U8); + return Ok((json!({"purpose":"agent-image","doc":doc,"taskId":id,"accountId":self.auth["account"]["id"],"origin":self.origin}), + egui::ColorImage::from_rgba_unmultiplied([image.width() as usize, image.height() as usize], image.data()))); + } let task_id = input["taskId"].as_str().filter(|s| Uuid::parse_str(s).is_ok()).ok_or("图片任务身份无效")?; let dir = self.project_dir(doc)?.join("image-jobs").join(task_id); let task = read_json(&dir.join("task.json"))?; diff --git a/apps/detail-photocraft/apps/photocraft/src/jiansun_pi_runtime.rs b/apps/detail-photocraft/apps/photocraft/src/jiansun_pi_runtime.rs index 94dbcd66..edcf37eb 100644 --- a/apps/detail-photocraft/apps/photocraft/src/jiansun_pi_runtime.rs +++ b/apps/detail-photocraft/apps/photocraft/src/jiansun_pi_runtime.rs @@ -495,6 +495,38 @@ fn run_inner(b: &mut Backend, input: &Value, method: &str, events: &mpsc::Sender let config = json!({"modelRole":model_role,"binary":binary,"profile":profile,"root":dir.join("agent"),"project":project,"scope":b.scope(),"documentId":doc, "ruleCatalog":saved.as_ref().map(|rule|rule["catalog"].clone()),"ruleId":saved.as_ref().map(|rule|rule["id"].clone()), "projectEvidence":project_evidence(&super::creation::load(&dir)?,&b.scope(),&project,doc,&input["context"]["requestScope"]["variantId"]),"mode":mode,"text":input["text"]}); + let mut config = config; + if mode == "prompt" { + use base64::{Engine as _, engine::general_purpose::STANDARD}; + let mut content = Vec::new(); + for (index, reference) in input["context"]["requestScope"]["references"].as_array().into_iter().flatten().enumerate() { + let mut item = input["referenceContent"][index].clone(); + if reference["kind"] == "candidate" { + let (_, image) = b.agent_image_preview(&json!({"doc":doc,"taskId":reference["taskId"]}))?; + item = json!({"image":photocraft_ui_egui::jiansun::reference_jpeg(&image, 768)?}); + } else if let Some(data) = item["image"].as_str() { + let id = item["imageId"].as_str().filter(|id| super::images::reference_preview_id(id)).ok_or("引用预览身份无效")?; + let path = dir.join("reference-previews").join(format!("{id}.jpg")); + if !path.exists() { + let bytes = STANDARD.decode(data).map_err(|_| "引用图片格式无效")?; + std::fs::create_dir_all(dir.join("reference-previews")).map_err(|_| "无法保存引用图片")?; + photocraft_format::atomic_write(&path, &bytes).map_err(|_| "无法保存引用图片")?; + } + if let Ok((meta, image)) = b.agent_image_preview(&json!({"doc":doc,"taskId":id})) { + let _ = events.send(Event::Preview(meta, image)); + ctx.request_repaint(); + } + } + if item.is_object() { + item["label"] = reference["label"].clone(); + content.push(item); + } + } + if serde_json::to_vec(&content).map_err(|_| "引用内容无法读取")?.len() > 1024 * 1024 { + return Err("图片引用较多,请减少本轮引用后再发送".into()); + } + config["referenceContent"] = json!(content); + } let daemon = agent_daemon::service(&node, &script, &binary, profile)?; let daemon_run = Uuid::new_v4().to_string(); let receiver = daemon.begin(&daemon_run, &config)?; diff --git a/apps/detail-photocraft/apps/photocraft/src/jiansun_sync.rs b/apps/detail-photocraft/apps/photocraft/src/jiansun_sync.rs index c915519a..0eeeb16a 100644 --- a/apps/detail-photocraft/apps/photocraft/src/jiansun_sync.rs +++ b/apps/detail-photocraft/apps/photocraft/src/jiansun_sync.rs @@ -95,7 +95,7 @@ fn safe_path(s: &str) -> R<&str> { Ok(s) } fn allowed_file(path: &str) -> bool { - let roots = ["agent/", "image-jobs/", "creation-tasks/", "creation-assets/", "creation-generated/", "template-outputs/"]; + let roots = ["agent/", "image-jobs/", "creation-tasks/", "creation-assets/", "creation-generated/", "template-outputs/", "reference-previews/"]; let exact = [ "agent-binding.json", "task.json", diff --git a/apps/detail-photocraft/crates/ui-egui/src/app.rs b/apps/detail-photocraft/crates/ui-egui/src/app.rs index a8dc9c34..9f0895c1 100644 --- a/apps/detail-photocraft/crates/ui-egui/src/app.rs +++ b/apps/detail-photocraft/crates/ui-egui/src/app.rs @@ -29,6 +29,8 @@ pub(crate) use input::read_dropped; pub(crate) use targets::adjustment_layer_active; #[cfg(test)] pub(crate) use thumbs::layer_thumb_frame; +#[cfg(not(target_arch = "wasm32"))] +pub(crate) use thumbs::reference_layer_image; pub use thumbs::surface_fingerprint; impl PhotocraftApp { diff --git a/apps/detail-photocraft/crates/ui-egui/src/app/thumbs.rs b/apps/detail-photocraft/crates/ui-egui/src/app/thumbs.rs index 7ba8fb1d..a89c65a4 100644 --- a/apps/detail-photocraft/crates/ui-egui/src/app/thumbs.rs +++ b/apps/detail-photocraft/crates/ui-egui/src/app/thumbs.rs @@ -160,12 +160,16 @@ pub(crate) fn layer_thumb_frame(layer: &photocraft_doc::Layer, canvas: photocraf /// Square thumbnail of the document area `frame`, letterboxed, sampling `f(x, y)` in document /// space. fn thumb_image(frame: photocraft_geom::Rect, side: usize, mut f: impl FnMut(i32, i32) -> [f32; 4]) -> egui::ColorImage { + sample_thumbnail(frame, [side; 2], &mut f) +} + +fn sample_thumbnail(frame: photocraft_geom::Rect, size: [usize; 2], mut f: impl FnMut(i32, i32) -> [f32; 4]) -> egui::ColorImage { let (w, h) = (frame.width().max(1) as f32, frame.height().max(1) as f32); - let scale = w.max(h) / side as f32; - let (ox, oy) = ((side as f32 - w / scale) / 2.0, (side as f32 - h / scale) / 2.0); - let mut px = vec![egui::Color32::TRANSPARENT; side * side]; - for ty in 0..side { - for tx in 0..side { + let scale = (w / size[0] as f32).max(h / size[1] as f32); + let (ox, oy) = ((size[0] as f32 - w / scale) / 2.0, (size[1] as f32 - h / scale) / 2.0); + let mut px = vec![egui::Color32::TRANSPARENT; size[0] * size[1]]; + for ty in 0..size[1] { + for tx in 0..size[0] { let dx = (tx as f32 - ox + 0.5) * scale; let dy = (ty as f32 - oy + 0.5) * scale; if dx < 0.0 || dy < 0.0 || dx >= w || dy >= h { @@ -173,10 +177,27 @@ fn thumb_image(frame: photocraft_geom::Rect, side: usize, mut f: impl FnMut(i32, } let c = f(frame.x0.saturating_add(dx as i32), frame.y0.saturating_add(dy as i32)); let q = |v: f32| (v.clamp(0.0, 1.0) * 255.0 + 0.5) as u8; - px[ty * side + tx] = egui::Color32::from_rgba_unmultiplied(q(c[0]), q(c[1]), q(c[2]), q(c[3])); + px[ty * size[0] + tx] = egui::Color32::from_rgba_unmultiplied(q(c[0]), q(c[1]), q(c[2]), q(c[3])); } } - egui::ColorImage::new([side, side], px) + egui::ColorImage::new(size, px) +} + +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn reference_layer_image(doc: &Document, layer: &photocraft_doc::Layer, side: usize) -> Option { + let surface = layer.surface()?; + let frame = photocraft_compose::bounds::content_bounds(surface).intersect(&doc.bounds()); + if frame.is_empty() { return None; } + let scale = (side as f64 / f64::from(frame.width().max(frame.height()))).min(1.0); + let size = [frame.width(), frame.height()].map(|dim| (f64::from(dim) * scale).round().max(1.0) as usize); + let mut sample = [0.0; 8]; + Some(sample_thumbnail(frame, size, |x, y| { + let n = surface.channels(); + surface.read_pixel(x, y, &mut sample[..n]); + let mut pixel = photocraft_raster::to_rgba(&surface.format(), &sample[..n]); + if let Some(mask) = &layer.mask { pixel[3] *= mask.value(x, y); } + pixel + })) } impl PhotocraftApp { diff --git a/apps/detail-photocraft/crates/ui-egui/src/jiansun.rs b/apps/detail-photocraft/crates/ui-egui/src/jiansun.rs index d3acb5bc..3dc39830 100644 --- a/apps/detail-photocraft/crates/ui-egui/src/jiansun.rs +++ b/apps/detail-photocraft/crates/ui-egui/src/jiansun.rs @@ -27,6 +27,8 @@ mod agent_preview; mod visual_preview_ui; mod composer_attachments; mod composer_references; +mod reference_media; +pub use reference_media::reference_jpeg; mod composer_drafts; mod copywriting; mod pi_ui; @@ -1982,6 +1984,14 @@ impl Workspace { Ok(references) => references, Err(error) => { self.error = error; return; } }; + let reference_content = match reference_media::capture(app, &references) { + Ok(content) => content, + Err(error) => { self.error = error; return; } + }; + let recorded_content: Vec<_> = reference_content.iter().cloned().map(|mut item| { + if let Some(fields) = item.as_object_mut() { fields.remove("image"); } + item + }).collect(); let mut context = self.context(app); if context.is_object() { let subject = match references.as_slice() { @@ -2004,11 +2014,11 @@ impl Workspace { self.pi_write_blocked = false; self.expected_revision = context["revision"].as_u64().unwrap_or(0); self.pi_request_scope = context["requestScope"].clone(); - self.messages.push(json!({"role":"user","text":text,"target":self.pi_request_scope["subject"],"requestScope":self.pi_request_scope})); + self.messages.push(json!({"role":"user","text":text,"target":self.pi_request_scope["subject"],"requestScope":self.pi_request_scope,"referenceContent":recorded_content})); self.draft_changed = true; self.send( "agent", - json!({"text":text,"context":{"id":context["id"],"revision":context["revision"],"requestScope":self.pi_request_scope},"model":self.model}), + json!({"text":text,"context":{"id":context["id"],"revision":context["revision"],"requestScope":self.pi_request_scope},"referenceContent":reference_content,"model":self.model}), ); if self.busy.as_deref() != Some("agent") { self.pi_request_scope = Value::Null; diff --git a/apps/detail-photocraft/crates/ui-egui/src/jiansun/agent_panel.rs b/apps/detail-photocraft/crates/ui-egui/src/jiansun/agent_panel.rs index 1a792fa7..cfd68c0e 100644 --- a/apps/detail-photocraft/crates/ui-egui/src/jiansun/agent_panel.rs +++ b/apps/detail-photocraft/crates/ui-egui/src/jiansun/agent_panel.rs @@ -382,6 +382,12 @@ fn user_bubble_with_reveal(ui: &mut Ui, text: &str, started: Option) -> Rec .inner } +fn paint_reference_thumbnail(ui: &mut Ui, texture: &egui::TextureHandle, limit: Vec2) -> Rect { + let source = texture.size_vec2(); + let size = source * (limit.x / source.x).min(limit.y / source.y); + ui.add(egui::Image::new(texture).fit_to_exact_size(size).corner_radius(6.0)).rect +} + /// Assistant replies are laid out directly on the conversation surface; the /// shared native label retains selectable text and its existing rich styling. fn assistant_reply(ui: &mut Ui, id: Id, text: &str) -> Response { @@ -627,17 +633,27 @@ fn candidate_matches_message(target: &Value, image: &Value) -> bool { identity } -/// Read the submit key before TextEdit consumes it. IME confirmation belongs to -/// the editor, never to the model request; repeats are consumed without dispatch. -fn composer_galley(ui:&Ui,text:&str,width:f32,indent:f32)->Arc { - let mut job=egui::text::LayoutJob::simple(text.to_owned(),FontId::proportional(14.0),Tokens::get(ui.ctx()).agent().text,width); - for section in &mut job.sections {section.format.line_height=Some(24.0);} - if let Some(first)=job.sections.first_mut(){first.leading_space=indent;} - let mut galley=ui.fonts_mut(|fonts|fonts.layout_job(job)); - if text.is_empty()&&indent>0.0 {let galley=Arc::make_mut(&mut galley);if let Some(row)=galley.rows.first_mut(){row.pos.x=indent;}galley.rect.max.x=indent;} - galley +fn composer_galley(ui: &Ui, text: &str, width: f32) -> Arc { + let mut job = egui::text::LayoutJob::simple(text.to_owned(), FontId::proportional(14.0), Tokens::get(ui.ctx()).agent().text, width); + for section in &mut job.sections { section.format.line_height = Some(24.0); } + ui.fonts_mut(|fonts| fonts.layout_job(job)) } +const REFERENCE_WIDTH: f32 = 132.0; +const REFERENCE_GAP: f32 = 4.0; + +fn reference_columns(width: f32) -> usize { + ((width + REFERENCE_GAP) / (REFERENCE_WIDTH + REFERENCE_GAP)).floor().max(1.0) as usize +} + +fn reference_shelf_height(count: usize, width: f32) -> f32 { + if count == 0 { return 0.0; } + let rows = count.div_ceil(reference_columns(width)).min(3); + rows as f32 * (24.0 + REFERENCE_GAP) - REFERENCE_GAP + 6.0 +} + +/// Read the submit key before TextEdit consumes it. IME confirmation belongs to +/// the editor, never to the model request; repeats are consumed without dispatch. fn composer_enter(ui: &mut Ui, id: Id) -> bool { let state_id = id.with("ime-submit-guard"); if !ui.memory(|memory| memory.has_focus(id)) { @@ -702,17 +718,6 @@ fn candidate_ordinal(subject: &Value, workflow: &Value) -> Option { Some(index + 1) } -fn subject_short_label(subject: &Value, workflow: &Value) -> String { - if subject["kind"] == "candidate" { - candidate_ordinal(subject, workflow).map_or_else(|| "候选图".to_owned(), |n| format!("候选 {n}")) - } else if subject["component"] == "pixelMask" { "蒙版".into() } - else if subject["component"] == "vectorMask" { "矢量".into() } - else if subject["layerKind"] == "Type" { "文字".into() } - else if subject["layerKind"] == "Shape" { "形状".into() } - else if subject["kind"] == "slot" { "图位".into() } - else { "对象".into() } -} - fn subject_label(subject: &Value, workflow: &Value) -> String { if subject["kind"] == "selection" { let count = subject["selectedLayerIds"].as_array().map_or_else(|| subject["targets"].as_array().map_or(0, Vec::len), Vec::len); @@ -902,11 +907,12 @@ impl Workspace { .and_then(|point| ui.ctx().layer_id_at(point)).is_some_and(|layer| layer.order <= egui::Order::Middle); let empty_escape = self.composer.is_empty() && ui.input(|input| input.modifiers == egui::Modifiers::NONE && input.key_pressed(egui::Key::Escape)); if was_overlay && !overlay_active && !outside_press && latched { self.focus_composer = true; } - if (!overlay_active || outside_background_press) && (outside_press || (empty_escape && !was_overlay)) { + // The canvas reference menu can request focus on the same press that closes it. + let leave_by_pointer = outside_press && !self.focus_composer && (!overlay_active || outside_background_press); + if leave_by_pointer || (empty_escape && !was_overlay && !overlay_active) { latched = false; - if outside_press { + if leave_by_pointer { ui.memory_mut(|memory| memory.surrender_focus(input_id)); - self.focus_composer = false; focused = false; } } @@ -1320,6 +1326,44 @@ impl Workspace { ui.data_mut(|data|data.insert_temp(self.user_reveal_id(Some(index)),now)); } } + fn sent_reference_cards(&self, ui: &mut Ui, message: &Value) { + let Some(references) = message["requestScope"]["references"].as_array().filter(|items| !items.is_empty()) else { return }; + let available = ui.available_width().max(1.0); + let width = available.min(120.0); + let spacing = ui.spacing().item_spacing.x; + let columns = (((available + spacing) / (width + spacing)).floor() as usize).max(1); + for (row, items) in references.chunks(columns).enumerate() { + ui.allocate_ui_with_layout(vec2(available, 1.0), Layout::right_to_left(Align::Min), |ui| { + for (column, reference) in items.iter().enumerate().rev() { + let index = row * columns + column; + let detail = &message["referenceContent"][index]; + ui.allocate_ui_with_layout(vec2(width, 1.0), Layout::top_down(Align::Min), |ui| { + super::stitch_ui::surface_frame(ui.ctx(), super::stitch_ui::SurfaceRole::Menu) + .inner_margin(egui::Margin::same(8)).show(ui, |ui| { + let content_width = (width - 16.0).max(1.0); + ui.set_width(content_width); + let label = reference["label"].as_str().filter(|label| !label.is_empty()) + .map(str::to_owned).unwrap_or_else(|| subject_label(reference, &self.template_workflow)); + if ui.is_rect_visible(Rect::from_min_size(ui.next_widget_position(), vec2(content_width, 110.0))) { + let image = detail["imageId"].as_str().or_else(|| reference["taskId"].as_str()).and_then(|id| { + self.agent_images.mark_visible(id); + self.agent_images.peek(id).cloned() + }); + if let Some(image) = image { + paint_reference_thumbnail(ui, &image, vec2(content_width, 90.0)); + } + } + ui.add(egui::Label::new(RichText::new(label).size(12.0)).truncate()); + if let Some(text) = detail["text"].as_str().filter(|text| !text.is_empty()) { + ui.add(egui::Label::new(RichText::new(text.chars().take(96).collect::()).size(12.0)).wrap()); + } + }); + }); + } + }); + } + } + fn turn_ui(&self, ui: &mut Ui, turn: Turn<'_>, conversation_id: Id, count: usize, current_turn: bool) -> Option { let t = Tokens::get(ui.ctx()).agent(); let stream = current_turn.then_some(self.stream_reply.as_str()).filter(|text| !text.is_empty()); @@ -1337,6 +1381,9 @@ impl Workspace { }) }; let response = agent_motion::message_visual(ui, enter, |ui| { + if let Some(index) = turn.user_index { + self.sent_reference_cards(ui, &self.messages[index]); + } user_bubble_with_reveal(ui, text, started); }); agent_motion::message_repaint(ui, response.response.rect, enter); @@ -1723,18 +1770,13 @@ impl Workspace { self.unknown && !self.pending() } - fn composer_reference_layout(&self, app:&PhotocraftApp, ui:&Ui, width:f32)->(f32,f32,f32) { - let references=self.composer_reference_list(app);let t=Tokens::get(ui.ctx()).agent(); - let more=if references.len()>1 {4.0+(ui.painter().layout_no_wrap(format!("+{}",references.len()-1),FontId::proportional(11.0),t.text).size().x+12.0).max(32.0)} else {0.0}; - let chip=references.first().map_or(0.0,|subject|super::stitch_ui::reference_chip_width(ui,&subject_short_label(subject,&self.template_workflow),(width-more-28.0).max(0.0))); - let indent=if references.is_empty(){0.0}else{chip+more+8.0};(chip,more,indent) - } - fn composer_surface_height(&self, app:&PhotocraftApp, ui:&Ui, width:f32, progress:f32)->f32 { - let attachment=if self.composer_text_send_allowed_for(app){0.0}else{self.composer_attachments_extra_height()}; - let wrap=(width-32.0).max(1.0);let (_,_,indent)=self.composer_reference_layout(app,ui,wrap); - let text_height=composer_galley(ui,&self.composer,wrap,indent).size().y.max(24.0); - let expanded=(text_height+74.0+attachment).max(168.0).min((ui.ctx().content_rect().height()*0.5).max(168.0)); - 56.0+(expanded-56.0)*progress + fn composer_surface_height(&self, app: &PhotocraftApp, ui: &Ui, width: f32, progress: f32) -> f32 { + let attachment = if self.composer_text_send_allowed_for(app) { 0.0 } else { self.composer_attachments_extra_height() }; + let references = reference_shelf_height(self.composer_reference_list(app).len(), (width - 28.0).max(1.0)); + let text_height = composer_galley(ui, &self.composer, (width - 32.0).max(1.0)).size().y.max(24.0); + let expanded = (text_height + 74.0 + attachment).max(168.0) + .min((ui.ctx().content_rect().height() * 0.5 - references).max(168.0)); + 56.0 + references + (expanded - 56.0) * progress } fn composer_notice_height(&self, ui: &Ui) -> f32 { @@ -1906,27 +1948,31 @@ impl Workspace { content.set_clip_rect(surface_rect.shrink(2.0).intersect(ui.clip_rect())); let parent_ui = ui; let ui = &mut content; - let mut header = ui.new_child(egui::UiBuilder::new().id_salt("composer-header").max_rect(ui.max_rect())); + let references = self.composer_reference_list(app); + self.panel_geometry["references"] = json!(references); + let reference_height = reference_shelf_height(references.len(), ui.max_rect().width()); + if reference_height > 0.0 { + let shelf = Rect::from_min_size(ui.max_rect().min, vec2(ui.max_rect().width(), reference_height - 6.0)); + let mut reference_ui = ui.new_child(egui::UiBuilder::new().id_salt("composer-references").max_rect(shelf)); + self.reference_shelf(app, &mut reference_ui, &references, shelf.height()); + } + let header_rect = ui.max_rect().translate(vec2(0.0, reference_height)); + let mut header = ui.new_child(egui::UiBuilder::new().id_salt("composer-header").max_rect(header_rect)); header.set_opacity(morph * morph); if morph < 0.95 { header.disable(); } if morph > 0.001 { self.composer_attachment_previews(app, &mut header); } let header_height = if header.min_rect().height() > 0.0 { header.min_rect().height() + 6.0 } else { 0.0 }; - let mini_input = Rect::from_min_size(egui::pos2(surface_rect.left() + 54.0, surface_rect.top() + 17.0), vec2(surface_rect.width() - 72.0, 24.0)); + let mini_input = Rect::from_min_size(egui::pos2(surface_rect.left() + 54.0, surface_rect.top() + reference_height + 17.0), vec2(surface_rect.width() - 72.0, 24.0)); let full_input = Rect::from_min_max( - egui::pos2(surface_rect.left() + 16.0, surface_rect.top() + 14.0 + header_height), + egui::pos2(surface_rect.left() + 16.0, surface_rect.top() + 14.0 + reference_height + header_height), egui::pos2(surface_rect.right() - 16.0, surface_rect.bottom() - 60.0)); let input_rect = Rect::from_min_max(mini_input.min.lerp(full_input.min, morph), mini_input.max.lerp(full_input.max, morph)); - let references = self.composer_reference_list(app); - self.panel_geometry["references"] = json!(references); - let (chip_width, more_width, indent)=self.composer_reference_layout(app,ui,input_rect.width()); - let subject_width=if references.is_empty(){0.0}else{chip_width+more_width}; - let empty = self.composer.is_empty(); let typing_owner=Id::unique((&self.origin,self.account["id"].as_str(),self.observed_doc,self.loaded_draft)); let reduced=self.reduce_motion; let mut layouter = |ui: &Ui, text: &dyn egui::TextBuffer, wrap_width: f32| { - let mut galley=composer_galley(ui,text.as_str(),wrap_width,indent); + let mut galley=composer_galley(ui,text.as_str(),wrap_width); super::typing_text::edit(ui,typing_owner,&mut galley,reduced); galley }; @@ -1942,7 +1988,7 @@ impl Workspace { .max_height(input_rect.height().max(24.0)).min_scrolled_height(input_rect.height().max(24.0)) .content_margin(egui::Margin::ZERO).auto_shrink([false, false]) .show(&mut input_ui, |ui| { - let editor=super::stitch_ui::text_input(&mut self.composer, if indent > 0.0 { "" } else if !draft_ready { "正在加载项目草稿…" } else { "告诉小壹,你想怎么改…" }, true) + let editor=super::stitch_ui::text_input(&mut self.composer, if !draft_ready { "正在加载项目草稿…" } else { "告诉小壹,你想怎么改…" }, true) .layouter(&mut layouter).id(input_id).return_key(egui::KeyboardShortcut::new(egui::Modifiers::SHIFT, egui::Key::Enter)) .font(FontId::proportional(14.0)).desired_rows(1).desired_width(f32::INFINITY) .min_size(vec2(0.0, input_rect.height().max(24.0))).margin(egui::Margin::ZERO).frame(Frame::NONE).show(ui); @@ -1960,23 +2006,8 @@ impl Workspace { self.panel_geometry["inputCaret"] = rect_value(editor.galley.pos_from_cursor(range.primary).translate(editor.galley_pos.to_vec2())); } self.panel_geometry["inputHitRect"] = rect_value(editor.response.rect); - // The inline subject chip uses a painted hint to keep its layout; the - // editor still needs its own name when its visible hint is suppressed. let response = editor.response.response.accessible_name("小壹消息输入"); - if empty && indent > 0.0 { - let hint = if draft_ready { "告诉小壹,你想怎么改…" } else { "正在加载项目草稿…" }; - let rect = ui.painter_at(input_rect).text(input_rect.min + vec2(indent, 3.5), egui::Align2::LEFT_TOP, hint, FontId::proportional(14.0), t.text_dim); - self.panel_geometry["inputHint"] = rect_value(rect); - } - let text_end=ui.ctx().graphics(|graphics|graphics.get(ui.layer_id()).map_or(text_start,|list|list.next_idx().0)); - // Register the inline target after the editor so the capsule owns its - // hit area; the text galley reserves only its first-line width. - if subject_width > 0.0 { - let mut chip = ui.new_child(egui::UiBuilder::new().id_salt("composer-inline-subject") - .max_rect(Rect::from_min_size(editor.galley_pos + vec2(0.0,editor.galley.rows.first().map_or(0.0,|row|row.pos.y)), vec2(subject_width, 24.0)))); - chip.set_clip_rect(input_rect.intersect(ui.clip_rect())); - self.selection_context(app, &mut chip, chip_width, more_width); - } + let text_end = ui.ctx().graphics(|graphics|graphics.get(ui.layer_id()).map_or(text_start, |list| list.next_idx().0)); let input_focused = response.has_focus(); self.panel_geometry["input"] = rect_value(input_rect); if self.focus_composer && draft_ready { response.request_focus(); self.focus_composer = false; } @@ -1986,7 +2017,7 @@ impl Workspace { let typing = input_focused && !self.composer.trim().is_empty() && !self.pending(); let orb_id = Id::unique("composer-identity"); let orb_progress = super::stitch_ui::morph_value(ui.ctx(), orb_id.with("size"), typing, self.reduce_motion); - let orb_center = egui::pos2(surface_rect.left() + 29.0, surface_rect.center().y) + let orb_center = egui::pos2(surface_rect.left() + 29.0, surface_rect.center().y + reference_height / 2.0) .lerp(egui::pos2(surface_rect.left() + 35.0, surface_rect.bottom() - 33.0), morph); let orb = Rect::from_center_size(orb_center, Vec2::splat(34.0 - 10.0 * morph + 16.0 * orb_progress)); let motion = if self.reduce_motion { super::siri_orb::Motion::Still } @@ -2244,91 +2275,67 @@ impl Workspace { view_candidates } - fn selection_context(&mut self, app: &PhotocraftApp, ui: &mut Ui, chip_width: f32, more_width: f32) -> f32 { - let t = Tokens::get(ui.ctx()).agent(); - let references = self.composer_reference_list(app); - let Some(subject) = references.first().cloned() else { return 0.0 }; - let label = subject_label(&subject, &self.template_workflow); - let pulse = agent_motion::subject_change(ui, ui.scope_id().with("agent-subject-transition"), &subject.to_string(), self.reduce_motion); - let glyph = if subject["kind"] == "candidate" { - "image" - } else if subject["component"] == "pixelMask" || subject["component"] == "vectorMask" { - "layer-mask" - } else if subject["layerKind"] == "Type" { - "type" - } else { - "layers" - }; - let short = subject_short_label(&subject, &self.template_workflow); - let preview = subject["taskId"].as_str().and_then(|id| self.agent_images.peek(id)).cloned(); - ui.spacing_mut().button_padding = vec2(6.0, 2.0); - ui.spacing_mut().interact_size.y = 23.0; - let mut locate = false; + fn reference_shelf(&mut self, app: &PhotocraftApp, ui: &mut Ui, references: &[Value], height: f32) { let mut detach = None; - let response = ui.horizontal(|ui| { - ui.spacing_mut().item_spacing.x = 4.0; - let (response, remove) = super::stitch_ui::reference_chip_with_preview(ui, glyph, &short, &label, pulse, chip_width, preview.as_ref()); - self.panel_geometry["referenceRemove"] = rect_value(remove.rect); - if remove.clicked() || (remove.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) || i.key_pressed(egui::Key::Space))) { detach = Some(0); } - locate = response.clicked() || (response.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) || i.key_pressed(egui::Key::Space))); - if references.len() > 1 { - let more = super::stitch_ui::segmented_option_sized(ui, &format!("+{}", references.len() - 1), false, true, vec2(32.0, 24.0)); - self.panel_geometry["referenceMore"] = rect_value(more.rect); - super::stitch_ui::select_popup(ui,&more,220.0,self.reduce_motion,|ui| { - ui.set_width(220.0_f32.min(ui.ctx().content_rect().width() - 48.0)); - egui::ScrollArea::vertical().max_height(260.0).show(ui, |ui| { - for (index, reference) in references.iter().enumerate() { - ui.horizontal(|ui| { - let name = subject_label(reference, &self.template_workflow); - let remove = super::stitch_ui::icon_button(ui, "x", 24.0, false, &format!("移除引用:{name}")); - self.panel_geometry[format!("referenceMenuRemove{index}")] = rect_value(remove.rect); - if remove.clicked() { detach = Some(index); } - if super::stitch_ui::menu_item(ui, &name, true).clicked() { self.canvas_locate(reference.clone()); ui.close(); } + let mut target = None; + let mut rects = Vec::new(); + let columns = reference_columns(ui.available_width()); + egui::ScrollArea::vertical().id_salt("composer-reference-scroll").max_height(height) + .auto_shrink([false, true]).show(ui, |ui| { + let width = REFERENCE_WIDTH.min(ui.available_width()).max(1.0); + ui.spacing_mut().item_spacing = vec2(REFERENCE_GAP, REFERENCE_GAP); + ui.spacing_mut().interact_size.y = 24.0; + for (row, items) in references.chunks(columns).enumerate() { + ui.horizontal(|ui| { + for (column, reference) in items.iter().enumerate() { + let index = row * columns + column; + ui.push_id(index, |ui| { + let label = subject_label(reference, &self.template_workflow); + let glyph = if reference["kind"] == "candidate" { "image" } + else if matches!(reference["component"].as_str(), Some("pixelMask" | "vectorMask")) { "layer-mask" } + else if reference["layerKind"] == "Type" { "type" } else { "layers" }; + let task = reference["taskId"].as_str(); + let thumbnail = task.and_then(|id| self.agent_images.peek(id)).cloned(); + let (body, remove) = super::stitch_ui::reference_chip_with_preview(ui, glyph, &label, &label, 0.0, width, thumbnail.as_ref()); + let enter = |response: &Response| response.clicked() || response.has_focus() + && ui.input(|i| i.key_pressed(egui::Key::Enter) || i.key_pressed(egui::Key::Space)); + if enter(&remove) { detach = Some(index); } + if enter(&body) { target = Some(reference.clone()); } + if ui.is_rect_visible(body.rect) { + let rect = body.rect.union(remove.rect); + rects.push(rect_value(rect)); + if index == 0 { + self.panel_geometry["subject"] = rect_value(rect); + self.panel_geometry["referenceRemove"] = rect_value(remove.rect); + } + } + if body.hovered() || body.has_focus() { + if let Some(task) = task { self.agent_images.mark_visible(task); } + let thumbnail = thumbnail.or_else(|| super::reference_media::hover_image(app, ui.ctx(), reference)); + let mut tooltip = egui::Tooltip::for_widget(&body); + tooltip.popup = tooltip.popup.style(super::stitch_ui::scoped_popup_style(ui)) + .frame(super::stitch_ui::surface_frame(ui.ctx(), super::stitch_ui::SurfaceRole::Menu).inner_margin(egui::Margin::same(8))); + tooltip.show(|ui| { + ui.set_width(148.0); + ui.vertical_centered(|ui| { + if let Some(texture) = &thumbnail { + let rect = paint_reference_thumbnail(ui, texture, vec2(148.0, 144.0)); + self.panel_geometry["subjectTooltip"] = rect_value(rect); + } + ui.add(egui::Label::new(RichText::new(label).size(12.0)).wrap()); + }); }); } }); - }); + } + }); } - response - }).inner; - let show_name = |ui: &mut Ui| { - let width = 156.0_f32.min(ui.ctx().content_rect().width() - 48.0); - let galley = ui.painter().layout_no_wrap(label.clone(), FontId::proportional(12.0), t.text); - let (rect, name_response) = ui.allocate_exact_size(vec2(width.min(galley.size().x).max(1.0), 18.0), Sense::drag()); - let overflow = (galley.size().x - rect.width()).max(0.0); - let now = ui.input(|input| input.time); - let start = ui.data_mut(|data| *data.get_temp_mut_or_insert_with(response.id.with("name-scroll"), || now)); - let automatic = if self.reduce_motion || overflow <= 0.0 { 0.0 } else { - let cycle = overflow / 28.0 + 2.4; - (((now - start).rem_euclid(cycle as f64) - 1.2).max(0.0) * 28.0).min(overflow as f64) as f32 - }; - let manual = ui.data_mut(|data| { - let offset = data.get_temp_mut_or_insert_with(response.id.with("name-drag"), || 0.0_f32); - if name_response.dragged() { *offset = (*offset - name_response.drag_delta().x).clamp(0.0, overflow); } - *offset - }); - let travel = if self.reduce_motion { manual } else { automatic }; - ui.painter_at(rect).galley(rect.left_top() + vec2(-travel, 1.0), galley, t.text); - self.panel_geometry["subjectTooltip"] = rect_value(rect); - self.panel_geometry["subjectTooltipOffset"] = json!(travel); - if overflow > 0.0 && !self.reduce_motion { ui.ctx().request_repaint_after(Duration::from_millis(16)); } - }; - if response.hovered() || response.has_focus() { - let mut tooltip = egui::Tooltip::for_widget(&response); - tooltip.popup = tooltip.popup.style(super::stitch_ui::scoped_popup_style(ui)) - .frame(super::stitch_ui::surface_frame(ui.ctx(), super::stitch_ui::SurfaceRole::Menu) - .corner_radius(16).inner_margin(egui::Margin::symmetric(12, 6))); - tooltip.show(show_name); - } else { ui.data_mut(|data| data.remove::(response.id.with("name-scroll"))); } - response.widget_info(|| egui::WidgetInfo::labeled(egui::Role::Button, true, format!("定位目标:{label}"))); - if ui.clip_rect().contains_rect(response.rect) { - self.panel_geometry["subject"] = rect_value(Rect::from_min_size(response.rect.min, vec2(chip_width, 24.0))); - } + }); + self.panel_geometry["referenceChips"] = json!(rects); if let Some(index) = detach { if let Err(error) = self.composer_reference_remove(app, index) { self.error = error; } self.focus_composer = true; - } else if locate { self.canvas_locate(subject); } - chip_width + more_width + } else if let Some(target) = target { self.canvas_locate(target); } } pub(super) fn receive_chat_context_usage(&mut self,usage:&Value) { @@ -2835,6 +2842,31 @@ mod tests { assert!(reply_rows > 1, "a long Chinese reply wraps in the available width"); } + #[test] + fn sent_reference_card_paints_the_recorded_image_after_the_draft_is_empty() { + let (mut app, _) = dock_fixture(); + let ctx = egui::Context::default(); + PhotocraftApp::setup_context(&ctx, crate::theme::ThemeKind::ProMedium); + let w = app.jiansun.as_mut().unwrap(); + w.composer.clear(); + let texture = ctx.load_texture("frozen-reference", egui::ColorImage::filled([8, 5], Color32::WHITE), egui::TextureOptions::LINEAR); + let id = texture.id(); + w.agent_images.insert("ref-frozen".into(), texture); + let message = json!({"role":"user","text":"图里有什么?", + "requestScope":{"references":[{"kind":"layer","label":"主图"}]}, + "referenceContent":[{"imageId":"ref-frozen"}]}); + let frame = || ctx.run_ui(egui::RawInput { + screen_rect: Some(Rect::from_min_size(egui::Pos2::ZERO, vec2(260.0, 360.0))), + ..Default::default() + }, |ui| w.sent_reference_cards(ui, &message)); + frame().drop_without_applying_deltas(); + let mut output = frame(); + let shown = ctx.tessellate(std::mem::take(&mut output.shapes), output.pixels_per_point).iter() + .any(|shape| matches!(&shape.primitive, egui::epaint::Primitive::Mesh(mesh) if mesh.texture_id == id)); + output.drop_without_applying_deltas(); + assert!(shown); + } + #[test] fn sent_user_text_reveals_in_order_without_resizing_the_bubble() { let ctx = egui::Context::default(); @@ -5662,7 +5694,7 @@ mod tests { let w = app.jiansun.as_ref().unwrap(); let composer = geometry(w, "composer"); let panel = geometry(w, "panel"); - assert!((composer.height() - 56.0).abs() < 1.0); + assert!((composer.height() - 56.0 - reference_shelf_height(1, width - 28.0)).abs() < 1.0); assert!(composer.bottom() <= panel.bottom() - 9.5, "{notice} at {width}: {composer:?} exceeds {panel:?}"); assert!(composer.contains_rect(geometry(w, "input"))); } @@ -5670,7 +5702,7 @@ mod tests { } #[test] - fn empty_unfocused_composer_is_a_single_pill_and_target_does_not_add_a_row() { + fn empty_unfocused_composer_keeps_references_above_the_native_input() { let (mut app, requests) = dock_fixture(); let w = app.jiansun.as_mut().unwrap(); w.composer.clear(); @@ -5679,19 +5711,18 @@ mod tests { let ctx = egui::Context::default(); PhotocraftApp::setup_context(&ctx, crate::theme::ThemeKind::ProMedium); for _ in 0..3 { - let _ = ctx.run_ui(egui::RawInput { screen_rect: Some(Rect::from_min_size(egui::Pos2::ZERO, vec2(1280.0, 800.0))), ..Default::default() }, |ui| crate::panels::right_dock(&mut app, ui)); + let _ = ctx.run_ui(egui::RawInput { screen_rect: Some(Rect::from_min_size(egui::Pos2::ZERO, vec2(1280.0, 800.0))), ..Default::default() }, |ui| crate::panels::right_dock(&mut app, ui)).drop_without_applying_deltas(); } let w = app.jiansun.as_ref().unwrap(); assert!(!w.panel_geometry["composerExpanded"].as_bool().unwrap()); - assert!((geometry(w, "composer").height() - 56.0).abs() < 1.0); let subject = geometry(w, "subject"); - assert!(subject.width() <= 90.0 && subject.height() <= 24.0, "compact target: {subject:?}"); - assert!((subject.top() - geometry(w, "input").top()).abs() <= 3.0); + assert!(subject.height() <= 24.0, "reference chip: {subject:?}"); + assert!(subject.bottom() <= geometry(w, "input").top()); assert!(requests.try_recv().is_err()); } #[test] - fn empty_inline_caret_and_repeated_blank_clicks_stay_after_the_capsule() { + fn empty_caret_and_repeated_clicks_stay_in_the_native_input_below_references() { let (mut app, requests) = dock_fixture(); let w = app.jiansun.as_mut().unwrap(); w.composer.clear(); @@ -5702,7 +5733,7 @@ mod tests { let screen = Rect::from_min_size(egui::Pos2::ZERO, vec2(1280.0, 800.0)); let frame = |app: &mut PhotocraftApp, events| { let _ = ctx.run_ui(egui::RawInput { screen_rect: Some(screen), events, ..Default::default() }, - |ui| crate::panels::right_dock(app, ui)); + |ui| crate::panels::right_dock(app, ui)).drop_without_applying_deltas(); }; for _ in 0..3 { frame(&mut app, vec![]); } let input = geometry(app.jiansun.as_ref().unwrap(), "input"); @@ -5722,15 +5753,15 @@ mod tests { let w = app.jiansun.as_ref().unwrap(); assert!(w.panel_geometry["composerExpanded"].as_bool().unwrap()); assert!(w.panel_geometry["composerFocused"].as_bool().unwrap()); - assert!(geometry(w, "inputCaret").left() >= geometry(w, "subject").right() + 6.0, - "the empty caret belongs after the inline target"); + assert!(geometry(w, "input").contains(geometry(w, "inputCaret").left_top())); + assert!(geometry(w, "inputCaret").top() >= geometry(w, "subject").bottom()); } frame(&mut app, vec![egui::Event::Text("窗边光线".into())]); frame(&mut app, vec![egui::Event::Key { key: egui::Key::Home, physical_key: None, pressed: true, repeat: false, modifiers: egui::Modifiers::NONE }]); frame(&mut app, vec![]); let w = app.jiansun.as_ref().unwrap(); assert_eq!(w.composer, "窗边光线"); - assert!(geometry(w, "inputCaret").left() >= geometry(w, "subject").right() + 6.0); + assert!(geometry(w, "inputCaret").top() >= geometry(w, "subject").bottom()); assert!(requests.try_recv().is_err(), "focus interactions never dispatch requests"); } @@ -5799,15 +5830,20 @@ mod tests { assert!(!output.unwrap().shapes.iter().any(|shape|masks(&shape.shape,input)),"scrolling text must not add an opaque edge over the shared glass"); } #[test] - fn composer_inline_reference_and_caret_share_the_first_text_row() { + fn composer_reference_shelf_leaves_the_caret_in_the_first_text_row() { let (mut app,_requests)=dock_fixture();app.ui.dock_tabs.color=4;let w=app.jiansun.as_mut().unwrap();w.reduce_motion=true;w.focus_composer=true;w.composer.clear(); let ctx=egui::Context::default();PhotocraftApp::setup_context(&ctx,crate::theme::ThemeKind::ProMedium); for _ in 0..4{dock_frame(&ctx,&mut app,vec![]);}let w=app.jiansun.as_ref().unwrap();let row=geometry(w,"inputFirstRow");let chip=geometry(w,"subject");let caret=geometry(w,"inputCaret"); - assert!(row.expand(0.5).contains_rect(chip),"row={row:?}, chip={chip:?}");assert!(caret.left()>=chip.right());assert!(geometry(w,"input").expand(0.5).contains_rect(caret));assert!(w.panel_geometry["composerFocused"].as_bool().unwrap()); - dock_frame(&ctx,&mut app,vec![egui::Event::Text("柚木暖色".into())]);let w=app.jiansun.as_ref().unwrap();assert!(geometry(w,"inputFirstRow").expand(0.5).contains_rect(geometry(w,"subject")));assert!(geometry(w,"inputCaret").left()>geometry(w,"subject").right());assert_eq!(w.composer,"柚木暖色"); + assert!(row.top()>=chip.bottom(),"row={row:?}, chip={chip:?}");assert!(row.expand(1.0).contains(caret.left_top()));assert!(geometry(w,"input").expand(0.5).contains_rect(caret));assert!(w.panel_geometry["composerFocused"].as_bool().unwrap()); + dock_frame(&ctx, &mut app, vec![egui::Event::Text("Test color".into())]); + dock_frame(&ctx, &mut app, vec![]); + let w = app.jiansun.as_ref().unwrap(); + assert_eq!(w.composer, "Test color"); + assert!(geometry(w, "inputFirstRow").top() >= geometry(w, "subject").bottom()); + assert!(geometry(w, "inputCaret").left() > geometry(w, "input").left()); } #[test] - fn inline_reference_never_overlaps_loading_or_ready_placeholder() { + fn references_do_not_cover_loading_or_ready_input() { for ready in [false, true] { let (mut app, _) = dock_fixture(); app.ui.dock_tabs.color=4; @@ -5817,8 +5853,8 @@ mod tests { let ctx=egui::Context::default(); PhotocraftApp::setup_context(&ctx,crate::theme::ThemeKind::ProMedium); for _ in 0..4 { dock_frame(&ctx,&mut app,vec![]); } let w=app.jiansun.as_ref().unwrap(); - let chip=geometry(w,"subject"); let hint=geometry(w,"inputHint"); - assert!(hint.left() >= chip.right(), "ready={ready}, chip={chip:?}, hint={hint:?}"); + let chip=geometry(w,"subject"); let input=geometry(w,"input"); + assert!(input.top() >= chip.bottom(), "ready={ready}, chip={chip:?}, input={input:?}"); } } @@ -6060,7 +6096,7 @@ mod tests { for pressed in [true,false] { dock_frame(&ctx,&mut app,vec![egui::Event::PointerMoved(outside),egui::Event::PointerButton { pos:outside,button:egui::PointerButton::Primary,pressed,modifiers:egui::Modifiers::NONE }]); } for _ in 0..3 { dock_frame(&ctx,&mut app,vec![]); } assert_eq!(app.jiansun.as_ref().unwrap().panel_geometry["composerExpanded"],false,"an external click collapses an empty composer"); - assert!((geometry(app.jiansun.as_ref().unwrap(),"composer").height()-56.0).abs()<1.0); + assert!((geometry(app.jiansun.as_ref().unwrap(),"composer").height()-56.0-reference_shelf_height(1, 250.0-28.0)).abs()<1.0); app.jiansun.as_mut().unwrap().focus_composer = true; for _ in 0..3 { dock_frame(&ctx,&mut app,vec![]); } dock_click(&ctx,&mut app,"composerMore"); diff --git a/apps/detail-photocraft/crates/ui-egui/src/jiansun/composer_references.rs b/apps/detail-photocraft/crates/ui-egui/src/jiansun/composer_references.rs index ba99f799..1c6ba1a0 100644 --- a/apps/detail-photocraft/crates/ui-egui/src/jiansun/composer_references.rs +++ b/apps/detail-photocraft/crates/ui-egui/src/jiansun/composer_references.rs @@ -904,6 +904,7 @@ mod tests { fn successful_submit_freezes_its_reference_without_leaking_it_into_the_next_intent() { let (mut w, mut app, selected) = fixture(); let pinned = app.session.execute("layer.new.layer", json!({"name":"Pinned"})).expect("layer")["layer"].as_u64().expect("id"); + app.session.execute("edit.fill", json!({"color":"#805132"})).expect("visible referenced image"); app.session.execute("layer.select", json!({"layer":selected})).expect("select"); let doc = app.session.active().expect("document").doc.id.0; w.composer_reference_add_target(&app, &json!({"documentId":doc,"nativeLayerId":pinned,"component":"layer"})).expect("add"); @@ -915,6 +916,9 @@ mod tests { let super::super::Request::Action(method, first) = requests.try_recv().expect("first request") else { panic!("expected agent request") }; assert_eq!(method, "agent"); assert_eq!(first["context"]["requestScope"]["references"][0]["nativeLayerId"], pinned); + assert!(first["referenceContent"][0]["image"].as_str().is_some_and(|image| !image.is_empty())); + assert_eq!(w.messages.last().unwrap()["referenceContent"][0]["imageId"], first["referenceContent"][0]["imageId"]); + assert!(w.messages.last().unwrap()["referenceContent"][0]["image"].is_null(), "the conversation stores the preview identity instead of raw pixels"); assert_eq!(app.session.active().unwrap().selected_layers(), vec![photocraft_doc::LayerId(selected)]); w.busy = None; diff --git a/apps/detail-photocraft/crates/ui-egui/src/jiansun/reference_media.rs b/apps/detail-photocraft/crates/ui-egui/src/jiansun/reference_media.rs new file mode 100644 index 00000000..b9cd8060 --- /dev/null +++ b/apps/detail-photocraft/crates/ui-egui/src/jiansun/reference_media.rs @@ -0,0 +1,83 @@ +//! Local thumbnails and the immutable content attached to an explicit reference. +use super::{PhotocraftApp, Value, json}; +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use photocraft_doc::{Document, Layer, LayerContent, LayerId}; +use std::sync::{Arc, Weak}; + +const IMAGE_SIDE: usize = 256; + +fn current_layer<'a>(app: &'a PhotocraftApp, reference: &Value) -> Option<(&'a Arc, &'a Layer)> { + let doc = &app.session.active()?.doc; + if reference["documentId"].as_u64() != Some(doc.id.0) + || reference["component"] != "layer" || reference["kind"] == "candidate" { return None; } + Some((doc, doc.layer(LayerId(reference["nativeLayerId"].as_u64()?))?)) +} + +#[derive(Clone)] +struct HoverImage { + doc: Weak, + layer: LayerId, + texture: Option, +} + +pub(super) fn hover_image(app: &PhotocraftApp, ctx: &egui::Context, reference: &Value) -> Option { + let (doc, layer) = current_layer(app, reference)?; + if matches!(layer.content, LayerContent::Text(_)) { return None; } + let id = egui::Id::unique("xiaoyi-composer-reference-thumbnail"); + let current = Arc::downgrade(doc); + if let Some(saved) = ctx.data(|data| data.get_temp::(id)) + && saved.layer == layer.id && saved.doc.ptr_eq(¤t) + { + return saved.texture; + } + let texture = crate::app::reference_layer_image(doc, layer, IMAGE_SIDE) + .map(|image| ctx.load_texture("composer-reference", image, egui::TextureOptions::LINEAR)); + ctx.data_mut(|data| data.insert_temp(id, HoverImage { doc: current, layer: layer.id, texture: texture.clone() })); + texture +} + +/// Encode only a bounded view of the selected pixels: no PSD, path, or source metadata. +pub fn reference_jpeg(image: &egui::ColorImage, max_side: usize) -> Result { + let largest = image.width().max(image.height()); + if largest == 0 { return Err("引用图片没有可见内容".into()); } + let side = largest.min(max_side); + let width = (image.width() * side / largest).max(1); + let height = (image.height() * side / largest).max(1); + let mut rgb = Vec::with_capacity(width * height * 3); + for y in 0..height { + for x in 0..width { + let pixel = image[(x * image.width() / width, y * image.height() / height)]; + let [r, g, b, a] = pixel.to_array(); + rgb.extend([r, g, b].map(|v| v.saturating_add(255 - a))); + } + } + let image = photocraft_codecs::Image::from_u8(width as u32, height as u32, photocraft_codecs::ChannelLayout::Rgb, rgb) + .map_err(|_| "引用图片无法读取")?; + let options = photocraft_codecs::EncodeOptions { jpeg_quality: 76, ..Default::default() }; + let bytes = photocraft_codecs::encode(&image, photocraft_codecs::Format::Jpeg, &options).map_err(|_| "引用图片无法编码")?; + Ok(STANDARD.encode(bytes)) +} + +pub(super) fn capture(app: &PhotocraftApp, references: &[Value]) -> Result, String> { + references.iter().map(|reference| { + let mut content = Value::Null; + if let Some((doc, layer)) = current_layer(app, reference) { + match &layer.content { + LayerContent::Text(text) => { + content = json!({"text":text.text.chars().take(1000).collect::()}); + } + LayerContent::Raster(_) | LayerContent::Smart(_) | LayerContent::Shape(_) => { + if let Some(image) = crate::app::reference_layer_image(doc, layer, IMAGE_SIDE) { + use sha2::{Digest, Sha256}; + let data = reference_jpeg(&image, IMAGE_SIDE)?; + // An unchanged layer reuses the same project thumbnail file. + let image_id = format!("ref-{:x}", Sha256::digest(data.as_bytes())); + content = json!({"image":data,"imageId":image_id}); + } + } + _ => {} + } + } + Ok(content) + }).collect() +}