Babylon.js Market - Debugging the Invisible Game with Babylon.js/GUI

By Lawrence
Mon Jan 19 2026
13 minutes

Imagine clicking on a spaceship in your 3D world and having a detailed control panel appear right next to it. Or selecting a building and seeing its upgrade menu floating above it. World-attached GUIs bridge the gap between 3D game objects and 2D interface elements, creating immersive interactions that feel natural.

Let's build a system that can attach any 2D GUI to any 3D entity in your world, with full positioning control and dynamic content management.

The Magic of Spatial UI

Traditional game UIs live in screen space - they're always in the same place regardless of what's happening in your 3D world. World-attached GUIs live in world space, following entities around and providing contextual information exactly where you need it.

This opens up possibilities like:

Floating health bars above characters
Interactive control panels on machinery
Context menus that appear next to selected objects
Inventory displays that hover near containers
Dialog bubbles positioned above NPCs

The WorldGUI Component: Your Spatial Interface

src/components/WorldGUI.ts

import { Component } from "~/code/Component";
import {
  AdvancedDynamicTexture,
  Control,
  Container,
  Button,
  TextBlock,
  InputText,
  Image,
  Rectangle,
} from "@babylonjs/gui";
import { Vector3, Mesh, Matrix } from "@babylonjs/core";

export interface WorldGUIInput {
  width?: number;
  height?: number;
  offsetX?: number;
  offsetY?: number;
  offsetZ?: number;
  autoSize?: boolean;
  followMesh?: boolean;
  content?: GUIContentConfig[];
  visible?: boolean;
  backgroundAlpha?: number;
  backgroundColor?: string;
}

export interface GUIContentConfig {
  type: "button" | "label" | "input" | "image" | "panel";
  id: string;
  content?: string;
  x?: number;
  y?: number;
  width?: string;
  height?: string;
  onClick?: string; // Method name to call
  style?: GUIElementStyle;
}

export interface GUIElementStyle {
  fontSize?: number;
  fontFamily?: string;
  color?: string;
  backgroundColor?: string;
  borderColor?: string;
  borderThickness?: number;
  cornerRadius?: number;
}

export class WorldGUIComponent extends Component {
  public width: number;
  public height: number;
  public offsetX: number;
  public offsetY: number;
  public offsetZ: number;
  public autoSize: boolean;
  public followMesh: boolean;
  public content: GUIContentConfig[];
  public visible: boolean;
  public backgroundAlpha: number;
  public backgroundColor: string;

  // Runtime properties
  public texture: AdvancedDynamicTexture | null = null;
  public container: Container | null = null;
  public mesh: Mesh | null = null;
  public worldPosition: Vector3 = Vector3.Zero();
  public elements: Map<string, Control> = new Map();

  constructor(data: WorldGUIInput = {}) {
    super(data);
    this.width = data.width || 400;
    this.height = data.height || 300;
    this.offsetX = data.offsetX || 0;
    this.offsetY = data.offsetY || -50; // Default above the mesh
    this.offsetZ = data.offsetZ || 0;
    this.autoSize = data.autoSize ?? true;
    this.followMesh = data.followMesh ?? true;
    this.content = data.content || [];
    this.visible = data.visible ?? true;
    this.backgroundAlpha = data.backgroundAlpha || 0.8;
    this.backgroundColor = data.backgroundColor || "#000000";
  }
}

This component configuration covers everything needed for a world-attached GUI:

Positioning: Offset from the entity's mesh with X/Y/Z coordinates
Content Definition: Array of GUI elements to create
Styling: Background, colors, and element appearance
Behavior: Whether to follow the mesh and auto-size content

The WorldGUI System: Spatial Interface Manager

src/components/WorldGUI.ts

import { System } from "~/code/System";
import { Entity } from "~/code/Entity";
import { World } from "~/code/World";
import { Scene, Camera } from "@babylonjs/core";

export class WorldGUISystem extends System {
  private guiTextures: Map<string, AdvancedDynamicTexture> = new Map();

  constructor(world: World, componentClasses = [WorldGUIComponent]) {
    super(world, componentClasses);
  }

  load(): void {
    // Initialize system-wide resources if needed
  }

  loadEntity(entity: Entity): void {
    const worldGUI = entity.getComponent(WorldGUIComponent);
    if (!worldGUI) return;

    // The entity IS the mesh since Entity extends Mesh
    worldGUI.mesh = entity as Mesh;
    this.createGUITexture(entity, worldGUI);
  }

  processEntity(entity: Entity, deltaTime: number): void {
    const worldGUI = entity.getComponent(WorldGUIComponent);
    if (!worldGUI || !worldGUI.texture || !worldGUI.followMesh) return;

    this.updateGUIPosition(entity, worldGUI);
  }
}

The system manages GUI textures and handles the continuous positioning updates needed to keep GUIs attached to their 3D entities.

Creating Dynamic GUI Content

src/components/WorldGUI.ts

  private createGUITexture(entity: Entity, worldGUI: WorldGUIComponent) {
    // Create fullscreen GUI texture
    const texture = AdvancedDynamicTexture.CreateFullscreenUI(
      `WorldGUI_${entity.name}`,
      true,
      this.scene
    );

    // Create main container
    const container = new Container(`Container_${entity.name}`);
    container.widthInPixels = worldGUI.width;
    container.heightInPixels = worldGUI.height;
    container.background = worldGUI.backgroundColor;
    container.alpha = worldGUI.backgroundAlpha;
    container.cornerRadius = 10;
    container.thickness = 2;
    container.color = "#FFFFFF";

    texture.addControl(container);

    // Store references
    worldGUI.texture = texture;
    worldGUI.container = container;
    this.guiTextures.set(entity.name, texture);

    // Create content elements
    this.createGUIContent(worldGUI, container);

    // Initial positioning
    this.updateGUIPosition(entity, worldGUI);
  }

  private createGUIContent(worldGUI: WorldGUIComponent, container: Container) {
    worldGUI.content.forEach(contentConfig => {
      const element = this.createGUIElement(contentConfig, worldGUI);
      if (element) {
        container.addControl(element);
        worldGUI.elements.set(contentConfig.id, element);
      }
    });
  }

The system creates a fullscreen GUI texture but positions the container elements at specific world positions, giving us pixel-perfect control over placement.

GUI Element Factory

src/components/WorldGUI.ts

  private createGUIElement(config: GUIContentConfig, worldGUI: WorldGUIComponent): Control | null {
    let element: Control;

    switch (config.type) {
      case "button":
        element = this.createButton(config);
        break;
      case "label":
        element = this.createLabel(config);
        break;
      case "input":
        element = this.createInput(config);
        break;
      case "image":
        element = this.createImage(config);
        break;
      case "panel":
        element = this.createPanel(config);
        break;
      default:
        console.warn(`Unknown GUI element type: ${config.type}`);
        return null;
    }

    // Apply positioning
    if (config.x !== undefined) element.leftInPixels = config.x;
    if (config.y !== undefined) element.topInPixels = config.y;
    if (config.width) element.width = config.width;
    if (config.height) element.height = config.height;

    // Apply styling
    if (config.style) {
      this.applyElementStyle(element, config.style);
    }

    return element;
  }

  private createButton(config: GUIContentConfig): Button {
    const button = Button.CreateSimpleButton(config.id, config.content || "Button");

    // Set up click handler
    if (config.onClick) {
      button.onPointerClickObservable.add(() => {
        this.handleGUIEvent(config.onClick!, config.id);
      });
    }

    return button;
  }

  private createLabel(config: GUIContentConfig): TextBlock {
    const label = new TextBlock(config.id, config.content || "Label");
    label.color = "white";
    label.fontSize = 16;
    return label;
  }

  private createInput(config: GUIContentConfig): InputText {
    const input = new InputText(config.id, config.content || "");
    input.color = "white";
    input.background = "#333333";
    input.focusedBackground = "#555555";
    return input;
  }

  private createImage(config: GUIContentConfig): Image {
    const image = new Image(config.id, config.content || "");
    return image;
  }

  private createPanel(config: GUIContentConfig): Rectangle {
    const panel = new Rectangle(config.id);
    panel.background = "#333333";
    panel.alpha = 0.8;
    panel.cornerRadius = 5;
    return panel;
  }

The element factory creates different types of GUI controls based on configuration. Each element type has sensible defaults but can be fully customized through styling.

World-to-Screen Positioning Magic

src/components/WorldGUI.ts

  private updateGUIPosition(entity: Entity, worldGUI: WorldGUIComponent) {
    if (!worldGUI.mesh || !worldGUI.container) return;

    // Calculate world position with offset
    const meshPosition = worldGUI.mesh.getAbsolutePosition();
    const offsetPosition = new Vector3(
      meshPosition.x + worldGUI.offsetX,
      meshPosition.y + worldGUI.offsetY,
      meshPosition.z + worldGUI.offsetZ
    );

    const camera = this.scene.activeCamera;
    if (!camera) return;

    // Project 3D world position to 2D screen coordinates
    const screenPosition = Vector3.Project(
      offsetPosition,
      Matrix.Identity(),
      this.scene.getTransformMatrix(),
      camera.viewport.toGlobal(
        this.scene.getEngine().getRenderWidth(),
        this.scene.getEngine().getRenderHeight()
      )
    );

    // Check if position is behind camera
    if (screenPosition.z > 1.0) {
      worldGUI.container.isVisible = false;
      return;
    }

    // Update container position
    worldGUI.container.isVisible = worldGUI.visible;
    worldGUI.container.leftInPixels = screenPosition.x - (worldGUI.width / 2);
    worldGUI.container.topInPixels = screenPosition.y - (worldGUI.height / 2);

    // Update stored world position
    worldGUI.worldPosition = offsetPosition;
  }

This is the core positioning logic. It takes the 3D mesh position, applies the offset, and projects it to 2D screen coordinates. The GUI automatically hides when the entity goes behind the camera.

Dynamic Content Updates

src/components/WorldGUI.ts

  public updateGUIContent(entityName: string, elementId: string, newContent: string) {
    const worldGUI = this.getWorldGUIForEntity(entityName);
    if (!worldGUI) return;

    const element = worldGUI.elements.get(elementId);
    if (!element) return;

    // Update content based on element type
    if (element instanceof TextBlock) {
      element.text = newContent;
    } else if (element instanceof Button) {
      element.textBlock.text = newContent;
    } else if (element instanceof InputText) {
      element.text = newContent;
    } else if (element instanceof Image) {
      element.source = newContent;
    }
  }

  public showGUI(entityName: string) {
    const worldGUI = this.getWorldGUIForEntity(entityName);
    if (worldGUI && worldGUI.container) {
      worldGUI.visible = true;
      worldGUI.container.isVisible = true;
    }
  }

  public hideGUI(entityName: string) {
    const worldGUI = this.getWorldGUIForEntity(entityName);
    if (worldGUI && worldGUI.container) {
      worldGUI.visible = false;
      worldGUI.container.isVisible = false;
    }
  }

  private getWorldGUIForEntity(entityName: string): WorldGUIComponent | null {
    const entity = this.world.entities.get(entityName);
    return entity ? entity.getComponent(WorldGUIComponent) : null;
  }

These utility methods let you dynamically update GUI content, show/hide GUIs, and manage the interface state during gameplay.

Event Handling and Interaction

src/components/WorldGUI.ts

  private handleGUIEvent(eventName: string, elementId: string) {
    // Find the entity that owns this GUI element
    for (const [entityName, entity] of this.world.entities) {
      const worldGUI = entity.getComponent(WorldGUIComponent);
      if (worldGUI && worldGUI.elements.has(elementId)) {
        this.dispatchEntityEvent(entity, eventName, elementId);
        break;
      }
    }
  }

  private dispatchEntityEvent(entity: Entity, eventName: string, elementId: string) {
    // Check if entity has a method matching the event name
    const method = (entity as any)[eventName];
    if (typeof method === 'function') {
      method.call(entity, elementId);
    }

    // Also dispatch to other systems that might be listening
    this.world.events?.emit('gui-event', {
      entityName: entity.name,
      eventName,
      elementId
    });
  }

Continue Reading

Unlock the full course to access all content and examples.

Add to Cart View Course Details