Building Custom XR Tools

Note

Applies to: Spatial Extensions, Kit 109.0.3+, CloudXR 6

This custom tools reference covers creating custom XR tools and GUI components using the XR component system.

Tool Architecture

XR tools are modular components that handle specific interactions (selection, teleport, grabbing, etc.). They follow a lifecycle pattern and integrate with the event system.

graph TD XRCore[XRCore] --> Components[Component Registry] Components --> Tools[Tool Components] Components --> GUILayers[GUI Layer Components] Tools --> Teleport[Teleport Tool] Tools --> Select[Select Tool] Tools --> Grab[Grab Tool] Tools --> Move[Move Tool] Tools --> Custom[Custom Tool] GUILayers --> Menu[Menu Layer] GUILayers --> Tooltips[Tooltips Layer] GUILayers --> CustomGUI[Custom GUI Layer]

Base Component Classes

XRComponentBase

Base class for all XR components.

from omni.kit.xr.core import XRComponentBase

class MyXRComponent(XRComponentBase):
    def __init__(self, component_name):
        super().__init__(component_name)
    
    def on_start(self):
        """Called when component is registered."""
        pass
    
    def on_enable(self):
        """Called when component is activated."""
        pass
    
    def on_update(self):
        """Called every frame while enabled."""
        pass
    
    def on_disable(self):
        """Called when component is deactivated."""
        pass
    
    def on_destroy(self):
        """Called when component is destroyed."""
        pass

XRToolComponentBase

Specialized for interactive tools.

from omni.kit.xr.core import XRToolComponentBase

class MyTool(XRToolComponentBase):
    def __init__(self):
        super().__init__("my_tool")
        self._subscriptions = []
    
    def on_enable(self):
        """Tool activated - set up input bindings."""
        # Bind input events
        self._setup_input()
    
    def on_disable(self):
        """Tool deactivated - clean up."""
        self._cleanup_input()
    
    def _setup_input(self):
        """Set up input event generators."""
        device = self.xr_core.get_input_device("/user/hand/left")
        if device:
            generator = device.bind_event_generator(
                "trigger", "my_tool_trigger",
                ["press", "release"]
            )
            
            # Subscribe to events
            message_bus = self.xr_core.get_message_bus()
            self._subscriptions.append(
                message_bus.create_subscription_to_pop_by_type(
                    carb.events.type_from_string("my_tool_trigger.press"),
                    self._on_trigger_press
                )
            )
    
    def _cleanup_input(self):
        """Clean up subscriptions."""
        self._subscriptions.clear()
    
    def _on_trigger_press(self, event):
        """Handle trigger press."""
        print("Tool triggered!")

XRGuiLayerComponentBase

Specialized for GUI layers (menus, HUDs, tooltips).

from omni.kit.xr.core import XRGuiLayerComponentBase

class MyGUILayer(XRGuiLayerComponentBase):
    def __init__(self):
        super().__init__("my_gui_layer")
    
    def on_enable(self):
        """GUI layer shown - create UI."""
        self._create_ui()
    
    def on_disable(self):
        """GUI layer hidden - clean up UI."""
        self._cleanup_ui()
    
    def _create_ui(self):
        """Create 3D UI in XRUsdLayer."""
        pass
    
    def _cleanup_ui(self):
        """Clean up UI elements."""
        pass

Complete Tool Example: Selection Tool

import omni.kit.xr.core
import carb.events
from omni.kit.xr.core import (
    XRToolComponentBase,
    XRRay,
    XRInputDeviceGeneratorEvent
)
from pxr import Gf

class SelectionTool(XRToolComponentBase):
    """Tool for selecting objects with controller raycast."""
    
    def __init__(self):
        super().__init__("selection_tool")
        
        self._subscriptions = []
        self._event_generators = []
        self._xr_usd_layer = None
        
        self._left_device = None
        self._right_device = None
        
        self._selection_beams = {}
        self._selected_objects = set()
    
    def on_enable(self):
        """Tool activated."""
        print("Selection tool enabled")
        
        # Create USD layer for beams
        self._xr_usd_layer = self.xr_core.create_xr_usd_layer(
            usd_path="/_xr/selection_tool",
            meters_per_unit=0.01,
            up_axis='y'
        )
        
        # Set up controllers
        self._left_device = self.xr_core.get_input_device("/user/hand/left")
        self._right_device = self.xr_core.get_input_device("/user/hand/right")
        
        # Create selection beams
        self._create_selection_beams()
        
        # Bind input
        self._setup_input()
        
        # Subscribe to frame updates
        message_bus = self.xr_core.get_message_bus()
        self._subscriptions.append(
            message_bus.create_subscription_to_pop_by_type(
                omni.kit.xr.core.XRCoreEventType.pre_sync_update,
                self._on_pre_sync
            )
        )
    
    def on_disable(self):
        """Tool deactivated."""
        print("Selection tool disabled")
        
        # Clean up
        self._cleanup_input()
        self._subscriptions.clear()
        
        # Clean up USD layer
        if self._xr_usd_layer:
            self._xr_usd_layer.clear()
            self._xr_usd_layer = None
        
        self._selected_objects.clear()
    
    def _create_selection_beams(self):
        """Create selection beams for each controller."""
        for device_name in ["/user/hand/left", "/user/hand/right"]:
            device_path = self._xr_usd_layer.ensure_device_prim_path(device_name)
            
            # Create beam
            beam_path = self._xr_usd_layer.add_beam(
                path=f"{device_path}/selection_beam",
                group="tools",
                material_reference="/World/Materials/SelectionBeam",
                max_length=1000.0,  # 10m
                tube_radius=0.3,
                visible=True
            )
            
            # Link to controller aim pose
            self._xr_usd_layer.add_link_to_input_device(
                path=beam_path,
                group="tools",
                input_device_name=device_name,
                pose_name="aim"
            )
            
            self._selection_beams[device_name] = beam_path
    
    def _setup_input(self):
        """Bind input events."""
        message_bus = self.xr_core.get_message_bus()
        
        # Bind trigger for selection
        for device_name, device in [
            ("/user/hand/left", self._left_device),
            ("/user/hand/right", self._right_device)
        ]:
            if not device:
                continue
            
            # Bind trigger
            generator = device.bind_event_generator(
                "trigger",
                f"select_{device_name.split('/')[-1]}",
                ["press", "release"],
                {"press": "Select Object"}
            )
            self._event_generators.append(generator)
            
            # Subscribe to press
            self._subscriptions.append(
                message_bus.create_subscription_to_pop_by_type(
                    carb.events.type_from_string(
                        f"select_{device_name.split('/')[-1]}.press"
                    ),
                    lambda e, d=device: self._on_select_press(e, d)
                )
            )
            
            # Subscribe to release
            self._subscriptions.append(
                message_bus.create_subscription_to_pop_by_type(
                    carb.events.type_from_string(
                        f"select_{device_name.split('/')[-1]}.release"
                    ),
                    lambda e, d=device: self._on_select_release(e, d)
                )
            )
    
    def _cleanup_input(self):
        """Clean up input bindings."""
        self._event_generators.clear()
    
    def _on_pre_sync(self, event):
        """Update beam lengths based on raycast."""
        for device_name, beam_path in self._selection_beams.items():
            device = self.xr_core.get_input_device(device_name)
            if not device:
                continue
            
            # Cast ray
            pose = device.get_pose("aim")
            origin = pose.ExtractTranslation()
            forward = pose.TransformDir(Gf.Vec3d(0, 0, -1))
            
            ray = XRRay(origin, forward, 0.0, 1000.0)
            
            # Submit raycast
            self.xr_core.submit_raycast_query(
                ray,
                lambda r, result, bp=beam_path: self._update_beam_length(bp, result)
            )
    
    def _update_beam_length(self, beam_path, result):
        """Update beam length based on raycast result."""
        if result.valid:
            # Hit something - set beam to hit distance
            distance = result.hit_t * 100.0  # Convert to cm
            self._xr_usd_layer.set_length(beam_path, distance)
        else:
            # No hit - use max length
            self._xr_usd_layer.set_length(beam_path, -1.0)
    
    def _on_select_press(self, event, device):
        """Handle selection trigger press."""
        # Cast ray to select
        pose = device.get_pose("aim")
        origin = pose.ExtractTranslation()
        forward = pose.TransformDir(Gf.Vec3d(0, 0, -1))
        
        ray = XRRay(origin, forward, 0.0, 1000.0)
        
        # Use async raycast
        import asyncio
        asyncio.ensure_future(self._select_object_async(ray))
    
    async def _select_object_async(self, ray):
        """Async selection."""
        result = await self.xr_core.execute_raycast_query_async(ray)
        
        if result.valid:
            target_path = result.get_target_usd_path()
            model_path = result.get_target_enclosing_model_usd_path()
            
            # Select model (not individual mesh)
            selected_path = model_path if model_path else target_path
            
            if selected_path:
                self._selected_objects.add(selected_path)
                print(f"Selected: {selected_path}")
                
                # Highlight selection (implementation depends on your app)
                self._highlight_selection(selected_path)
    
    def _on_select_release(self, event, device):
        """Handle selection trigger release."""
        pass
    
    def _highlight_selection(self, prim_path):
        """Highlight selected object."""
        # Implementation depends on your application
        # Could change material, add outline, etc.
        pass

# Register and use the tool
def setup_selection_tool(xr_core):
    """Set up and enable selection tool."""
    tool = SelectionTool()
    tool.on_enable()
    return tool

# Usage
# selection_tool = setup_selection_tool(xr_core)
# ... later ...
# selection_tool.on_disable()

Tool Registration Pattern

Tools are typically registered with the XR system for automatic lifecycle management.

class ToolManager:
    """Manage XR tools."""
    
    def __init__(self, xr_core):
        self.xr_core = xr_core
        self.tools = {}
        self.active_tool = None
    
    def register_tool(self, tool_name, tool_class):
        """Register tool class."""
        self.tools[tool_name] = tool_class
    
    def enable_tool(self, tool_name):
        """Enable specific tool."""
        # Disable current tool
        if self.active_tool:
            self.active_tool.on_disable()
        
        # Enable new tool
        if tool_name in self.tools:
            tool_class = self.tools[tool_name]
            self.active_tool = tool_class()
            self.active_tool.on_enable()
            
            print(f"Enabled tool: {tool_name}")
    
    def disable_current_tool(self):
        """Disable active tool."""
        if self.active_tool:
            self.active_tool.on_disable()
            self.active_tool = None
    
    def get_active_tool(self):
        """Get currently active tool."""
        return self.active_tool

# Usage
tool_manager = ToolManager(xr_core)

# Register tools
tool_manager.register_tool("select", SelectionTool)
tool_manager.register_tool("teleport", TeleportTool)
tool_manager.register_tool("grab", GrabTool)

# Enable tool
tool_manager.enable_tool("select")

GUI Layer Example: XR Menu

from omni.kit.xr.core import XRGuiLayerComponentBase
from pxr import Gf

class XRMenu(XRGuiLayerComponentBase):
    """3D menu that appears in front of user."""
    
    def __init__(self):
        super().__init__("xr_menu")
        
        self._xr_usd_layer = None
        self._menu_root = None
        self._buttons = {}
        self._subscriptions = []
    
    def on_enable(self):
        """Menu shown."""
        print("XR Menu enabled")
        
        # Create USD layer
        self._xr_usd_layer = self.xr_core.create_xr_usd_layer(
            usd_path="/_xr/menu",
            meters_per_unit=0.01,
            up_axis='y'
        )
        
        # Create menu UI
        self._create_menu()
        
        # Position in front of user
        self._position_menu()
        
        # Set up interaction
        self._setup_interaction()
    
    def on_disable(self):
        """Menu hidden."""
        print("XR Menu disabled")
        
        # Clean up
        self._subscriptions.clear()
        
        if self._xr_usd_layer:
            self._xr_usd_layer.clear()
            self._xr_usd_layer = None
    
    def _create_menu(self):
        """Create menu structure."""
        # Create root that faces user
        self._menu_root = self._xr_usd_layer.add_reorient(
            path="/_xr/menu/root",
            group="menu",
            alignment=omni.kit.xr.core.XROrientationAlignment.device_up_right,
            input_device="/user/head",
            visible=True
        )
        
        # Add background panel
        self._xr_usd_layer.add_asset(
            path=f"{self._menu_root}/background",
            group="menu",
            file_path="/assets/ui/menu_panel.usd",
            pickable=True
        )
        
        # Add buttons
        button_configs = [
            ("Select Tool", "select"),
            ("Teleport Tool", "teleport"),
            ("Grab Tool", "grab"),
            ("Close Menu", "close")
        ]
        
        button_height = 10.0  # 10cm
        button_spacing = 2.0   # 2cm
        
        for i, (label, action) in enumerate(button_configs):
            y_pos = -(i * (button_height + button_spacing))
            
            button_path = self._xr_usd_layer.add_asset(
                path=f"{self._menu_root}/button_{action}",
                group="menu",
                file_path="/assets/ui/button.usd",
                transform=Gf.Matrix4d().SetTranslate(
                    Gf.Vec3d(0, y_pos, 0)
                ),
                pickable=True
            )
            
            # Store button action in metadata
            self._xr_usd_layer.set_meta_data(
                f"button_{action}_label",
                label
            )
            
            self._buttons[action] = button_path
    
    def _position_menu(self):
        """Position menu in front of user."""
        head = self.xr_core.get_input_device("/user/head")
        if head:
            head_pose = head.get_virtual_world_pose()
            forward = head_pose.TransformDir(Gf.Vec3d(0, 0, -1))
            position = head_pose.ExtractTranslation() + forward * 100.0  # 1m forward
            
            self._xr_usd_layer.set_position(
                self._menu_root,
                position
            )
    
    def _setup_interaction(self):
        """Set up button interaction."""
        # Listen for trigger on either hand
        message_bus = self.xr_core.get_message_bus()
        
        for device_name in ["/user/hand/left", "/user/hand/right"]:
            device = self.xr_core.get_input_device(device_name)
            if device:
                generator = device.bind_event_generator(
                    "trigger",
                    f"menu_select_{device_name.split('/')[-1]}",
                    ["press"]
                )
                
                self._subscriptions.append(
                    message_bus.create_subscription_to_pop_by_type(
                        carb.events.type_from_string(
                            f"menu_select_{device_name.split('/')[-1]}.press"
                        ),
                        lambda e, d=device: self._on_menu_select(d)
                    )
                )
    
    def _on_menu_select(self, device):
        """Handle menu button selection."""
        # Cast ray from controller
        pose = device.get_pose("aim")
        origin = pose.ExtractTranslation()
        forward = pose.TransformDir(Gf.Vec3d(0, 0, -1))
        
        ray = omni.kit.xr.core.XRRay(origin, forward, 0.0, 1000.0)
        
        # Check hit
        import asyncio
        asyncio.ensure_future(self._check_button_hit_async(ray))
    
    async def _check_button_hit_async(self, ray):
        """Check which button was hit."""
        result = await self.xr_core.execute_raycast_query_async(ray)
        
        if result.valid:
            hit_path = result.get_target_usd_path()
            
            # Check if button
            for action, button_path in self._buttons.items():
                if button_path in hit_path:
                    self._handle_button_action(action)
                    break
    
    def _handle_button_action(self, action):
        """Handle button press."""
        print(f"Button pressed: {action}")
        
        if action == "close":
            # Close menu
            self.on_disable()
        elif action in ["select", "teleport", "grab"]:
            # Switch tool
            # (In real app, would call tool manager)
            print(f"Switch to {action} tool")

# Usage
xr_menu = XRMenu()
xr_menu.on_enable()  # Show menu
# ... later ...
xr_menu.on_disable()  # Hide menu

Tooltips Layer Example

from omni.kit.xr.core import XRGuiLayerComponentBase

class TooltipsLayer(XRGuiLayerComponentBase):
    """Display tooltips on controllers."""
    
    def __init__(self):
        super().__init__("tooltips_layer")
        
        self._xr_usd_layer = None
        self._tooltip_labels = {}
        self._subscriptions = []
    
    def on_enable(self):
        """Enable tooltips."""
        # Create USD layer
        self._xr_usd_layer = self.xr_core.create_xr_usd_layer(
            usd_path="/_xr/tooltips",
            meters_per_unit=0.01,
            up_axis='y'
        )
        
        # Create tooltips for each controller
        for device_name in ["/user/hand/left", "/user/hand/right"]:
            self._create_tooltip(device_name)
    
    def _create_tooltip(self, device_name):
        """Create tooltip for device."""
        device_path = self._xr_usd_layer.ensure_device_prim_path(device_name)
        
        # Create tooltip label
        tooltip_path = self._xr_usd_layer.add_asset(
            path=f"{device_path}/tooltip",
            group="tooltips",
            file_path="/assets/ui/tooltip.usd",
            transform=Gf.Matrix4d().SetTranslate(Gf.Vec3d(0, 5, 0)),  # Offset above controller
            visible=True
        )
        
        # Link to controller
        self._xr_usd_layer.add_link_to_input_device(
            path=tooltip_path,
            group="tooltips",
            input_device_name=device_name,
            pose_name="grip"
        )
        
        self._tooltip_labels[device_name] = tooltip_path
    
    def update_tooltip(self, device_name, text):
        """Update tooltip text."""
        # Implementation depends on your text rendering system
        pass
    
    def on_disable(self):
        """Disable tooltips."""
        if self._xr_usd_layer:
            self._xr_usd_layer.clear()
            self._xr_usd_layer = None

Best Practices

Tool Design

1. Single Responsibility: Each tool does one thing well

2. Clean Lifecycle: Proper setup/cleanup in on_enable/on_disable

3. Event-Driven: Use events, not polling

4. Reusable: Design for reuse across projects

Performance

1. USD-RT: Use XRUsdLayer for dynamic objects

2. Batch Operations: Group USD updates

3. Lazy Creation: Create UI only when needed

4. Efficient Raycasting: Cache ray results, use async

User Experience

1. Visual Feedback: Show tool state (beams, highlights)

2. Tooltips: Display control hints

3. Smooth Transitions: Fade in/out UI elements

4. Consistent Controls: Follow XR conventions

Next Steps

• Custom Bundles – Package tools as bundles

• Kit App Integration – Integrate into apps

• Event System – Advanced event patterns

• Scene Integration – USD integration details

Key Takeaways

• Tools inherit from XRToolComponentBase

• GUI layers inherit from XRGuiLayerComponentBase

• Lifecycle: on_enable → on_update → on_disable

• Use XRUsdLayer for 3D UI elements

• Event-driven architecture for responsive tools

• Clean up resources in on_disable