fix(history): count fill caches and smart filter masks in undo memory (#4972) (#5075)

crates/ops/src/lib.rs

@@ -248,7 +248,7 @@ impl History {
     }
 }
 
-/// Bytes of the pixel tiles of `doc` (layers, masks, alpha channels, selection) not already in `seen`.
+/// Bytes of the pixel tiles of `doc` (layers, masks, fill caches, smart filter masks, alpha channels, selection) not already in `seen`.
 fn tile_bytes(doc: &Document, seen: &mut HashSet<usize>) -> usize {
     let mut n = 0usize;
     tiles::surfaces(doc, |surface| {
@@ -408,6 +408,42 @@ mod tests {
         assert_eq!(h.unique_bytes(&cur), 256 * 256 * 4);
     }
 
+    #[test]
+    fn fill_caches_and_smart_filter_masks_count_toward_history_bytes() {
+        use photocraft_doc::{Affine, Fill, FillCache, LayerContent, LayerMask, PixelFormat, SmartObject, SmartSource, Surface};
+        let mut doc = Document::new("fills", Size::new(64, 64), ColorMode::Rgb, SampleType::U8);
+        let mut fill = Layer::new("Fill", LayerContent::Fill(Fill::Solid(Color::WHITE)));
+        let mut cache = Surface::new(PixelFormat::GRAY8);
+        cache.write_pixel(1, 1, &[1.0]);
+        fill.fill_cache = Some(FillCache { fill: Fill::Solid(Color::WHITE), surface: cache });
+        doc.insert_above(None, fill);
+        let mut smart_obj = SmartObject::new(SmartSource::Linked { path: "a.png".into() }, Affine::IDENTITY, None);
+        let mut mask = LayerMask::reveal_all();
+        mask.surface.write_pixel(1, 1, &[0.5]);
+        smart_obj.filter_mask = Some(mask);
+        doc.insert_above(None, Layer::new("Smart", LayerContent::Smart(smart_obj)));
+        let tile = 256 * 256;
+        let mut cur = Arc::new(doc);
+        let mut h = History { max_bytes: usize::MAX, ..History::default() };
+        assert_eq!(h.pixel_bytes(&cur), 2 * tile, "fill cache and smart filter mask are counted");
+        edit(&mut h, &mut cur, "Rename", |doc| doc.name = "renamed".into());
+        assert_eq!(h.pixel_bytes(&cur), 2 * tile, "shared tiles count once");
+        edit(&mut h, &mut cur, "Fill cache", |doc| {
+            let id = doc.layers[0].id;
+            if let Some(cache) = doc.layer_mut(id).and_then(|l| l.fill_cache.as_mut()) {
+                cache.surface.write_pixel(1, 1, &[0.0]);
+            }
+        });
+        assert_eq!(h.pixel_bytes(&cur), 3 * tile, "the copied fill cache tile is history-only");
+        assert_eq!(h.unique_bytes(&cur), tile);
+        h.trim(&cur);
+        assert_counter(&h, &cur);
+        h.max_bytes = 2 * tile;
+        assert_eq!(h.trim(&cur), 1, "the copied fill cache tile pushes history over budget");
+        assert_eq!(h.pixel_bytes(&cur), 3 * tile, "the most recent undo state is always kept");
+        assert_counter(&h, &cur);
+    }
+
     #[test]
     fn selection_tiles_count_current_and_history_without_double_counting() {
         use photocraft_doc::{PixelFormat, Surface};

crates/ops/src/tiles.rs

@@ -1,7 +1,7 @@
 //! Tile references owned by history snapshots. The snapshots keep these addresses alive;
 //! COW edits cannot change a retained tile in place. Count occurrences (including aliases
 //! within one document), so removing a state releases a tile only after its last reference.
-use photocraft_doc::{Document, Surface};
+use photocraft_doc::{Document, LayerContent, Surface};
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 
@@ -19,6 +19,14 @@ pub(crate) fn surfaces(doc: &Document, mut visit: impl FnMut(&Surface)) {
         if let Some(mask) = &layer.mask {
             visit(&mask.surface);
         }
+        if let Some(cache) = &layer.fill_cache {
+            visit(&cache.surface);
+        }
+        if let LayerContent::Smart(smart) = &layer.content
+            && let Some(mask) = &smart.filter_mask
+        {
+            visit(&mask.surface);
+        }
         if let Some(video) = &layer.video {
             // Undo retains the whole clip, not only the displayed frame. Visit aliases too:
             // Tiles tracks every owner and counts their shared allocations only once.