UI per text section picking (#22047)

# Objective

Allow picking text per section.

Fixes #17477

## Solution

In `ui_picking` whenever a node with text components is below a pointer,
don't add the `Text` node's entity to the `hit_nodes` list. Instead,
iterate the list of text section bounding rects in its `TextLayoutInfo`
component to find which text section the pointer is over, and add that
section's entity id to the `hit_nodes` list.

* Add optional text components to `NodeQuery`.
* New helper function, `pick_ui_text`. Finds if a text section is
hovered and returns its id.
* Add the entities of text sections to `hit_nodes` when hovered.
* Add the target camera entity to `hit_nodes`, instead of querying for
it a second time.
* When a picked node is a text node and `require_markers` is enabled,
check if the hit section entity has a `Pickable` component, instead of
the text node.

#

This design might need to be changed later to allow for text selection
and copying. But there doesn't seem any need to overcomplicate things
for now by considering it.

## Testing

I added observers to the `TextBackgroundColors` example's text entities.
The observers turn a text section's text color to white when the pointer
hovers them.

```
cargo run --example text_background_colors
```

Author: ickshonpe

crates/bevy_ui/src/picking_backend.rs

@@ -30,6 +30,7 @@ use bevy_ecs::{prelude::*, query::QueryData};
 use bevy_math::Vec2;
 use bevy_platform::collections::HashMap;
 use bevy_reflect::{std_traits::ReflectDefault, Reflect};
+use bevy_text::{ComputedTextBlock, TextLayoutInfo};
 use bevy_window::PrimaryWindow;
 
 use bevy_picking::backend::prelude::*;
@@ -92,6 +93,7 @@ pub struct NodeQuery {
     pickable: Option<&'static Pickable>,
     inherited_visibility: Option<&'static InheritedVisibility>,
     target_camera: &'static ComputedUiTargetCamera,
+    text_node: Option<(&'static TextLayoutInfo, &'static ComputedTextBlock)>,
 }
 
 /// Computes the UI node entities under each pointer.
@@ -108,6 +110,7 @@ pub fn ui_picking(
     mut output: MessageWriter<PointerHits>,
     clipping_query: Query<(&ComputedNode, &UiGlobalTransform, &Node)>,
     child_of_query: Query<&ChildOf, Without<OverrideClip>>,
+    pickable_query: Query<&Pickable>,
 ) {
     // Map from each camera to its active pointers and their positions in viewport space
     let mut pointer_pos_by_camera = HashMap::<Entity, HashMap<PointerId, Vec2>>::default();
@@ -143,7 +146,7 @@ pub fn ui_picking(
     }
 
     // The list of node entities hovered for each (camera, pointer) combo
-    let mut hit_nodes = HashMap::<(Entity, PointerId), Vec<(Entity, Vec2)>>::default();
+    let mut hit_nodes = HashMap::<(Entity, PointerId), Vec<(Entity, Entity, Vec2)>>::default();
 
     // prepare an iterator that contains all the nodes that have the cursor in their rect,
     // from the top node to the bottom one. this will also reset the interaction to `None`
@@ -175,7 +178,8 @@ pub fn ui_picking(
                 continue;
             };
 
-            if settings.require_markers && node.pickable.is_none() {
+            // Nodes with Display::None have a (0., 0.) logical rect and can be ignored
+            if node.node.size() == Vec2::ZERO {
                 continue;
             }
 
@@ -188,16 +192,38 @@ pub fn ui_picking(
                 continue;
             }
 
-            // Nodes with Display::None have a (0., 0.) logical rect and can be ignored
-            if node.node.size() == Vec2::ZERO {
+            // If this is a text node, need to do this check per section.
+            if node.text_node.is_none() && settings.require_markers && node.pickable.is_none() {
                 continue;
             }
 
             // Find the normalized cursor position relative to the node.
             // (±0., 0.) is the center with the corners at points (±0.5, ±0.5).
             // Coordinates are relative to the entire node, not just the visible region.
             for (pointer_id, cursor_position) in pointers_on_this_cam.iter() {
-                if node.node.contains_point(*node.transform, *cursor_position)
+                if let Some((text_layout_info, text_block)) = node.text_node {
+                    if let Some(text_entity) = pick_ui_text_section(
+                        node.node,
+                        node.transform,
+                        *cursor_position,
+                        text_layout_info,
+                        text_block,
+                    ) {
+                        if settings.require_markers && !pickable_query.contains(text_entity) {
+                            continue;
+                        }
+
+                        hit_nodes
+                            .entry((camera_entity, *pointer_id))
+                            .or_default()
+                            .push((
+                                text_entity,
+                                camera_entity,
+                                node.transform.inverse().transform_point2(*cursor_position)
+                                    / node.node.size(),
+                            ));
+                    }
+                } else if node.node.contains_point(*node.transform, *cursor_position)
                     && clip_check_recursive(
                         *cursor_position,
                         node_entity,
@@ -210,6 +236,7 @@ pub fn ui_picking(
                         .or_default()
                         .push((
                             node_entity,
+                            camera_entity,
                             node.transform.inverse().transform_point2(*cursor_position)
                                 / node.node.size(),
                         ));
@@ -224,19 +251,13 @@ pub fn ui_picking(
         let mut picks = Vec::new();
         let mut depth = 0.0;
 
-        for (hovered_node, position) in hovered {
-            let node = node_query.get(*hovered_node).unwrap();
-
-            let Some(camera_entity) = node.target_camera.get() else {
-                continue;
-            };
-
+        for (hovered_node, camera_entity, position) in hovered {
             picks.push((
-                node.entity,
-                HitData::new(camera_entity, depth, Some(position.extend(0.0)), None),
+                *hovered_node,
+                HitData::new(*camera_entity, depth, Some(position.extend(0.0)), None),
             ));
 
-            if let Some(pickable) = node.pickable {
+            if let Ok(pickable) = pickable_query.get(*hovered_node) {
                 // If an entity has a `Pickable` component, we will use that as the source of truth.
                 if pickable.should_block_lower {
                     break;
@@ -258,3 +279,22 @@ pub fn ui_picking(
         output.write(PointerHits::new(*pointer, picks, order));
     }
 }
+
+fn pick_ui_text_section(
+    uinode: &ComputedNode,
+    global_transform: &UiGlobalTransform,
+    point: Vec2,
+    text_layout_info: &TextLayoutInfo,
+    text_block: &ComputedTextBlock,
+) -> Option<Entity> {
+    let local_point = global_transform
+        .try_inverse()
+        .map(|transform| transform.transform_point2(point) + 0.5 * uinode.size())?;
+
+    for run in text_layout_info.run_geometry.iter() {
+        if run.bounds.contains(local_point) {
+            return text_block.entities().get(run.span_index).map(|e| e.entity);
+        }
+    }
+    None
+}

examples/ui/text_background_colors.rs

@@ -49,15 +49,30 @@ fn setup(mut commands: Commands) {
                 ))
                 .with_children(|commands| {
                     for (i, section_str) in message_text.iter().enumerate() {
-                        commands.spawn((
-                            TextSpan::new(*section_str),
-                            TextColor::BLACK,
-                            TextFont {
-                                font_size: 100.,
-                                ..default()
-                            },
-                            TextBackgroundColor(PALETTE[i % PALETTE.len()]),
-                        ));
+                        commands
+                            .spawn((
+                                TextSpan::new(*section_str),
+                                TextColor::BLACK,
+                                TextFont {
+                                    font_size: 100.,
+                                    ..default()
+                                },
+                                TextBackgroundColor(PALETTE[i % PALETTE.len()]),
+                            ))
+                            .observe(
+                                |event: On<Pointer<Over>>, mut query: Query<&mut TextColor>| {
+                                    if let Ok(mut text_color) = query.get_mut(event.entity) {
+                                        text_color.0 = Color::WHITE;
+                                    }
+                                },
+                            )
+                            .observe(
+                                |event: On<Pointer<Out>>, mut query: Query<&mut TextColor>| {
+                                    if let Ok(mut text_color) = query.get_mut(event.entity) {
+                                        text_color.0 = Color::BLACK;
+                                    }
+                                },
+                            );
                     }
                 });
         });

release-content/migration-guides/the_non-text_areas_of_text_nodes_are_no_longer_pickable.md

@@ -0,0 +1,7 @@
+---
+title: "The non-text areas of UI `Text` nodes are no longer pickable"
+pull_requests: [22047]
+---
+
+Only the sections of `Text` node's containing text are pickable now, the non-text areas of the node do not register pointer hits.
+To replicate Bevy 0.17's picking behavior, use an intermediate parent node to intercept the pointer hits.

release-content/release-notes/per_text_section_picking.md

@@ -0,0 +1,7 @@
+---
+title: "UI per text section picking"
+authors: ["@ickshonpe"]
+pull_requests: [22047]
+---
+
+Individual text sections belonging to UI text nodes are now pickable and can be given observers.