Fix TUIO tangible detection and add comprehensive debug logging

- Fix connection settings to display all detected tangibles, not just configured ones
- Add visual indicators for configured vs unconfigured tangibles
- Add extensive debug logging throughout TUIO stack (WebSocket, client, handlers, integration)
- Add validation for TUIO 1.1 symbolId field to prevent errors
- Add warning message when unconfigured tangibles are detected

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

src/components/Config/TuioConnectionConfig.tsx

@@ -72,13 +72,16 @@ const TuioConnectionConfig = ({ isOpen, onClose }: Props) => {
       const client = new TuioClientManager(
         {
           onTangibleAdd: (hardwareId: string, info: TuioTangibleInfo) => {
+          console.log('[TUIO Config] Tangible added callback:', hardwareId, info);
           setTestActiveTangibles((prev) => {
             const newMap = new Map(prev);
             newMap.set(hardwareId, info);
+            console.log('[TUIO Config] Active tangibles count:', newMap.size);
             return newMap;
           });
         },
         onTangibleUpdate: (hardwareId: string, info: TuioTangibleInfo) => {
+          console.log('[TUIO Config] Tangible updated callback:', hardwareId, info);
           setTestActiveTangibles((prev) => {
             const newMap = new Map(prev);
             if (newMap.has(hardwareId)) {
@@ -88,13 +91,16 @@ const TuioConnectionConfig = ({ isOpen, onClose }: Props) => {
           });
         },
         onTangibleRemove: (hardwareId: string) => {
+          console.log('[TUIO Config] Tangible removed callback:', hardwareId);
           setTestActiveTangibles((prev) => {
             const newMap = new Map(prev);
             newMap.delete(hardwareId);
+            console.log('[TUIO Config] Active tangibles count:', newMap.size);
             return newMap;
           });
         },
         onConnectionChange: (connected: boolean, error?: string) => {
+          console.log('[TUIO Config] Connection state changed:', connected, error);
           setTestConnected(connected);
           if (error) {
             setTestConnectionError(error);
@@ -177,10 +183,16 @@ const TuioConnectionConfig = ({ isOpen, onClose }: Props) => {
 
   if (!isOpen) return null;
 
-  // Get active tangible configs from test connection
+  // Get all detected tangibles with their config status
-  const activeTangibleConfigs = Array.from(testActiveTangibles.keys())
+  const detectedTangibles = Array.from(testActiveTangibles.entries()).map(([hwId, info]) => {
-    .map((hwId) => tangibles.find((t) => t.hardwareId === hwId))
+    const config = tangibles.find((t) => t.hardwareId === hwId);
-    .filter((t): t is NonNullable<typeof t> => t !== undefined);
+    return {
+      hardwareId: hwId,
+      info,
+      config,
+      isConfigured: config !== undefined,
+    };
+  });
 
   return (
     <div className="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50">
@@ -292,37 +304,56 @@ const TuioConnectionConfig = ({ isOpen, onClose }: Props) => {
               </span>
             </label>
             <div className="border border-gray-200 rounded-md divide-y divide-gray-200 min-h-[100px]">
-              {activeTangibleConfigs.length > 0 ? (
+              {detectedTangibles.length > 0 ? (
-                activeTangibleConfigs.map((tangible) => {
+                detectedTangibles.map((tangible) => (
-                  const info = testActiveTangibles.get(tangible.hardwareId!);
+                  <div
-                  return (
+                    key={tangible.hardwareId}
-                    <div key={tangible.id} className="px-4 py-3 bg-gray-50">
+                    className={`px-4 py-3 ${tangible.isConfigured ? 'bg-green-50' : 'bg-yellow-50'}`}
-                      <div className="flex items-center justify-between">
+                  >
-                        <div>
+                    <div className="flex items-center justify-between">
+                      <div>
+                        <div className="flex items-center gap-2">
                           <p className="text-sm font-medium text-gray-900">
-                            {tangible.name}
+                            {tangible.config?.name || `Hardware ID: ${tangible.hardwareId}`}
-                          </p>
-                          <p className="text-xs text-gray-600">
-                            Hardware ID: {tangible.hardwareId} • Mode: {tangible.mode}
                           </p>
+                          <span
+                            className={`px-2 py-0.5 text-xs rounded-full ${
+                              tangible.isConfigured
+                                ? 'bg-green-100 text-green-800'
+                                : 'bg-yellow-100 text-yellow-800'
+                            }`}
+                          >
+                            {tangible.isConfigured ? 'Configured' : 'Not Configured'}
+                          </span>
                         </div>
-                        {info && (
+                        <p className="text-xs text-gray-600 mt-1">
-                          <div className="text-xs text-gray-500">
+                          Hardware ID: {tangible.hardwareId}
-                            Position: ({info.x.toFixed(2)}, {info.y.toFixed(2)})
+                          {tangible.config && ` • Mode: ${tangible.config.mode}`}
-                          </div>
+                        </p>
-                        )}
+                      </div>
+                      <div className="text-xs text-gray-500 text-right">
+                        <div>Position: ({tangible.info.x.toFixed(2)}, {tangible.info.y.toFixed(2)})</div>
+                        <div>Angle: {(tangible.info.angle * 180 / Math.PI).toFixed(1)}°</div>
                       </div>
                     </div>
-                  );
+                  </div>
-                })
+                ))
               ) : (
                 <div className="px-4 py-6 text-center text-sm text-gray-500">
                   {testConnected
-                    ? 'No tangibles detected. Place a configured tangible on the TUIO surface.'
+                    ? 'No tangibles detected. Place a tangible on the TUIO surface.'
                     : 'Connect to TUIO server to detect tangibles.'}
                 </div>
               )}
             </div>
+            {detectedTangibles.some((t) => !t.isConfigured) && (
+              <p className="text-xs text-yellow-700 mt-2 flex items-start gap-1">
+                <span className="font-bold">⚠️</span>
+                <span>
+                  Some detected tangibles are not configured. Go to Tangible Configuration to set up hardware IDs.
+                </span>
+              </p>
+            )}
           </div>
         </div>
 

src/hooks/useTuioIntegration.ts

@@ -29,6 +29,7 @@ export function useTuioIntegration() {
     if (!presentationMode) {
       // Disconnect if we're leaving presentation mode
       if (clientRef.current) {
+        console.log('[TUIO Integration] Presentation mode disabled, disconnecting');
         clientRef.current.disconnect();
         clientRef.current = null;
         useTuioStore.getState().clearActiveTangibles();
@@ -36,6 +37,8 @@ export function useTuioIntegration() {
       return;
     }
 
+    console.log('[TUIO Integration] Presentation mode enabled, connecting to TUIO server');
+
     // Create TUIO client if in presentation mode
     const client = new TuioClientManager(
       {
@@ -43,6 +46,7 @@ export function useTuioIntegration() {
         onTangibleUpdate: handleTangibleUpdate,
         onTangibleRemove: handleTangibleRemove,
         onConnectionChange: (connected, error) => {
+          console.log('[TUIO Integration] Connection state changed:', connected, error);
           useTuioStore.getState().setConnectionState(connected, error);
         },
       },
@@ -55,12 +59,13 @@ export function useTuioIntegration() {
     client
       .connect(websocketUrl)
       .catch((error) => {
-        console.error('Failed to connect to TUIO server:', error);
+        console.error('[TUIO Integration] Failed to connect to TUIO server:', error);
       });
 
     // Cleanup on unmount or when presentation mode changes
     return () => {
       if (clientRef.current) {
+        console.log('[TUIO Integration] Cleaning up, disconnecting');
         clientRef.current.disconnect();
         clientRef.current = null;
         useTuioStore.getState().clearActiveTangibles();
@@ -73,6 +78,8 @@ export function useTuioIntegration() {
  * Handle tangible add event
  */
 function handleTangibleAdd(hardwareId: string, info: TuioTangibleInfo): void {
+  console.log('[TUIO Integration] Tangible added:', hardwareId, info);
+
   // Update TUIO store
   useTuioStore.getState().addActiveTangible(hardwareId, info);
 
@@ -82,9 +89,12 @@ function handleTangibleAdd(hardwareId: string, info: TuioTangibleInfo): void {
 
   if (!tangibleConfig) {
     // Unknown hardware ID - silently ignore
+    console.log('[TUIO Integration] No configuration found for hardware ID:', hardwareId);
     return;
   }
 
+  console.log('[TUIO Integration] Tangible configuration found:', tangibleConfig.name, 'mode:', tangibleConfig.mode);
+
   // Trigger action based on tangible mode
   if (tangibleConfig.mode === 'filter') {
     applyFilterTangible(tangibleConfig);
@@ -99,6 +109,7 @@ function handleTangibleAdd(hardwareId: string, info: TuioTangibleInfo): void {
  * Currently just updates position/angle in store (for future stateDial support)
  */
 function handleTangibleUpdate(hardwareId: string, info: TuioTangibleInfo): void {
+  console.log('[TUIO Integration] Tangible updated:', hardwareId, info);
   useTuioStore.getState().updateActiveTangible(hardwareId, info);
 }
 
@@ -106,6 +117,8 @@ function handleTangibleUpdate(hardwareId: string, info: TuioTangibleInfo): void
  * Handle tangible remove event
  */
 function handleTangibleRemove(hardwareId: string): void {
+  console.log('[TUIO Integration] Tangible removed:', hardwareId);
+
   // Remove from TUIO store
   useTuioStore.getState().removeActiveTangible(hardwareId);
 
@@ -114,9 +127,12 @@ function handleTangibleRemove(hardwareId: string): void {
   const tangibleConfig = tangibles.find((t) => t.hardwareId === hardwareId);
 
   if (!tangibleConfig) {
+    console.log('[TUIO Integration] No configuration found for removed tangible:', hardwareId);
     return;
   }
 
+  console.log('[TUIO Integration] Handling removal for configured tangible:', tangibleConfig.name);
+
   // Handle removal based on tangible mode
   if (tangibleConfig.mode === 'filter') {
     removeFilterTangible(tangibleConfig);

src/lib/tuio/WebsocketTuioReceiver.ts

@@ -22,6 +22,8 @@ export class WebsocketTuioReceiver extends TuioReceiver {
   constructor(host: string, port: number) {
     super();
 
+    console.log(`[TUIO] Creating WebSocket receiver for ${host}:${port}`);
+
     // Create OSC WebSocket client
     this.osc = new OSC({
       plugin: new OSC.WebsocketClientPlugin({
@@ -32,17 +34,20 @@ export class WebsocketTuioReceiver extends TuioReceiver {
 
     // Forward all OSC messages to TUIO client
     this.osc.on('*', (message: OscMessage) => {
+      console.log('[TUIO] OSC message received:', message.address, message.args);
       this.onOscMessage(message);
     });
 
     // Listen for WebSocket connection events
     this.osc.on('open', () => {
+      console.log('[TUIO] WebSocket connection opened');
       if (this.onOpenCallback) {
         this.onOpenCallback();
       }
     });
 
     this.osc.on('close', () => {
+      console.log('[TUIO] WebSocket connection closed');
       if (this.onCloseCallback) {
         this.onCloseCallback();
       }
@@ -50,6 +55,7 @@ export class WebsocketTuioReceiver extends TuioReceiver {
 
     this.osc.on('error', (error: unknown) => {
       const errorMessage = error instanceof Error ? error.message : 'WebSocket error';
+      console.error('[TUIO] WebSocket error:', errorMessage);
       if (this.onErrorCallback) {
         this.onErrorCallback(errorMessage);
       }
@@ -81,6 +87,7 @@ export class WebsocketTuioReceiver extends TuioReceiver {
    * Open WebSocket connection to TUIO server
    */
   connect(): void {
+    console.log('[TUIO] Opening WebSocket connection...');
     this.osc.open();
   }
 
@@ -88,6 +95,7 @@ export class WebsocketTuioReceiver extends TuioReceiver {
    * Close WebSocket connection
    */
   disconnect(): void {
+    console.log('[TUIO] Closing WebSocket connection...');
     this.osc.close();
   }
 }

src/lib/tuio/tuioClient.ts

@@ -35,6 +35,8 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
   async connect(url: string): Promise<void> {
     return new Promise((resolve, reject) => {
       try {
+        console.log(`[TUIO] Connecting to ${url} with protocol version ${this.protocolVersion}`);
+
         // Parse WebSocket URL
         const wsUrl = new URL(url);
         const host = wsUrl.hostname;
@@ -47,9 +49,11 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
 
         // Create appropriate client based on protocol version
         if (this.protocolVersion === '1.1') {
+          console.log('[TUIO] Creating TUIO 1.1 client');
           this.client11 = new Tuio11Client(this.receiver);
           this.client20 = null;
         } else {
+          console.log('[TUIO] Creating TUIO 2.0 client');
           this.client20 = new Tuio20Client(this.receiver);
           this.client11 = null;
         }
@@ -57,6 +61,7 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
         // Set up connection event handlers
         this.receiver.setOnOpen(() => {
           // Connection successful
+          console.log('[TUIO] Connection successful');
           this.callbacks.onConnectionChange(true);
           resolve();
         });
@@ -80,10 +85,14 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
 
         // Add this manager as a listener
         if (this.client11) {
+          console.log('[TUIO] Adding listener to TUIO 1.1 client');
           this.client11.addTuioListener(this);
+          console.log('[TUIO] Connecting TUIO 1.1 client');
           this.client11.connect();
         } else if (this.client20) {
+          console.log('[TUIO] Adding listener to TUIO 2.0 client');
           this.client20.addTuioListener(this);
+          console.log('[TUIO] Connecting TUIO 2.0 client');
           this.client20.connect();
         }
       } catch (error) {
@@ -139,7 +148,16 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    * Called when a TUIO 1.1 object is added (tangible placed on surface)
    */
   addTuioObject(tuioObject: Tuio11Object): void {
+    console.log('[TUIO] 1.1 Object added - raw object:', tuioObject);
+
+    // Validate symbolId exists
+    if (tuioObject.symbolId === undefined || tuioObject.symbolId === null) {
+      console.warn('[TUIO] 1.1 Object has no symbolId, ignoring');
+      return;
+    }
+
     const info = this.extractTangibleInfo11(tuioObject);
+    console.log('[TUIO] 1.1 Object added - extracted info:', info);
     this.callbacks.onTangibleAdd(info.hardwareId, info);
   }
 
@@ -147,7 +165,16 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    * Called when a TUIO 1.1 object is updated (position/rotation changed)
    */
   updateTuioObject(tuioObject: Tuio11Object): void {
+    console.log('[TUIO] 1.1 Object updated - raw object:', tuioObject);
+
+    // Validate symbolId exists
+    if (tuioObject.symbolId === undefined || tuioObject.symbolId === null) {
+      console.warn('[TUIO] 1.1 Object has no symbolId, ignoring');
+      return;
+    }
+
     const info = this.extractTangibleInfo11(tuioObject);
+    console.log('[TUIO] 1.1 Object updated - extracted info:', info);
     this.callbacks.onTangibleUpdate(info.hardwareId, info);
   }
 
@@ -155,7 +182,16 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    * Called when a TUIO 1.1 object is removed (tangible removed from surface)
    */
   removeTuioObject(tuioObject: Tuio11Object): void {
+    console.log('[TUIO] 1.1 Object removed - raw object:', tuioObject);
+
+    // Validate symbolId exists
+    if (tuioObject.symbolId === undefined || tuioObject.symbolId === null) {
+      console.warn('[TUIO] 1.1 Object has no symbolId, ignoring');
+      return;
+    }
+
     const hardwareId = String(tuioObject.symbolId);
+    console.log('[TUIO] 1.1 Object removed - hardwareId:', hardwareId);
     this.callbacks.onTangibleRemove(hardwareId);
   }
 
@@ -163,6 +199,7 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    * Called when a TUIO 1.1 cursor is added (not used for tangibles)
    */
   addTuioCursor(): void {
+    console.log('[TUIO] 1.1 Cursor added (ignored)');
     // Ignore cursors (touch points)
   }
 
@@ -170,6 +207,7 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    * Called when a TUIO 1.1 cursor is updated (not used for tangibles)
    */
   updateTuioCursor(): void {
+    console.log('[TUIO] 1.1 Cursor updated (ignored)');
     // Ignore cursors
   }
 
@@ -177,6 +215,7 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    * Called when a TUIO 1.1 cursor is removed (not used for tangibles)
    */
   removeTuioCursor(): void {
+    console.log('[TUIO] 1.1 Cursor removed (ignored)');
     // Ignore cursors
   }
 
@@ -184,6 +223,7 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    * Called when a TUIO 1.1 blob is added (not used for tangibles)
    */
   addTuioBlob(): void {
+    console.log('[TUIO] 1.1 Blob added (ignored)');
     // Ignore blobs
   }
 
@@ -191,6 +231,7 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    * Called when a TUIO 1.1 blob is updated (not used for tangibles)
    */
   updateTuioBlob(): void {
+    console.log('[TUIO] 1.1 Blob updated (ignored)');
     // Ignore blobs
   }
 
@@ -198,6 +239,7 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    * Called when a TUIO 1.1 blob is removed (not used for tangibles)
    */
   removeTuioBlob(): void {
+    console.log('[TUIO] 1.1 Blob removed (ignored)');
     // Ignore blobs
   }
 
@@ -205,6 +247,7 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    * Called on TUIO 1.1 frame refresh (time sync)
    */
   refresh(): void {
+    console.log('[TUIO] 1.1 Frame refresh (ignored)');
     // Ignore refresh events
   }
 
@@ -215,9 +258,13 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    */
   tuioAdd(tuioObject: Tuio20Object): void {
     const token = tuioObject.token;
-    if (!token) return; // Only handle tokens (tangibles), not pointers
+    if (!token) {
+      console.log('[TUIO] 2.0 Add event ignored (not a token)');
+      return; // Only handle tokens (tangibles), not pointers
+    }
 
     const info = this.extractTangibleInfo(tuioObject);
+    console.log('[TUIO] 2.0 Token added:', info);
     this.callbacks.onTangibleAdd(info.hardwareId, info);
   }
 
@@ -226,9 +273,13 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    */
   tuioUpdate(tuioObject: Tuio20Object): void {
     const token = tuioObject.token;
-    if (!token) return;
+    if (!token) {
+      console.log('[TUIO] 2.0 Update event ignored (not a token)');
+      return;
+    }
 
     const info = this.extractTangibleInfo(tuioObject);
+    console.log('[TUIO] 2.0 Token updated:', info);
     this.callbacks.onTangibleUpdate(info.hardwareId, info);
   }
 
@@ -237,9 +288,13 @@ export class TuioClientManager implements Tuio11Listener, Tuio20Listener {
    */
   tuioRemove(tuioObject: Tuio20Object): void {
     const token = tuioObject.token;
-    if (!token) return;
+    if (!token) {
+      console.log('[TUIO] 2.0 Remove event ignored (not a token)');
+      return;
+    }
 
     const hardwareId = String(token.cId);
+    console.log('[TUIO] 2.0 Token removed:', hardwareId);
     this.callbacks.onTangibleRemove(hardwareId);
   }