diff --git a/docs/adr/ADR-342-cognitum-inspired-mobile-instrument-ui.md b/docs/adr/ADR-342-cognitum-inspired-mobile-instrument-ui.md index a527e4af..fc1382fe 100644 --- a/docs/adr/ADR-342-cognitum-inspired-mobile-instrument-ui.md +++ b/docs/adr/ADR-342-cognitum-inspired-mobile-instrument-ui.md @@ -337,6 +337,13 @@ consume only the existing fail-closed displayable track list. Do not add `expo-gl`, React Three Fiber, a WebView, a raw point-cloud schema, or a new sensor adapter in this UI-only decision. +Render explicit target-lock reticles, a relay-to-hidden depth scale, confidence, +and a persistent `RECONSTRUCTION / NOT RAW SCAN` boundary in the cloud surface. +The reticle geometry reuses bounded marker objects and updates their positions in +place so frame changes do not create a new WebGL context or allocate new marker +meshes. These cues improve operator target acquisition without promoting the +schematic cloud into physical sensor evidence. + ## Alternatives with quantified tradeoffs Alternatives are scored from 1, poor, to 5, strong. Weighted score is out of diff --git a/docs/screenshots/consumer-nlos-mobile-ui/README.md b/docs/screenshots/consumer-nlos-mobile-ui/README.md index 4ce42ddd..2462dc9e 100644 --- a/docs/screenshots/consumer-nlos-mobile-ui/README.md +++ b/docs/screenshots/consumer-nlos-mobile-ui/README.md @@ -4,7 +4,8 @@ These images are deterministic review captures from the production Expo web export at a 390 by 844 viewport. The Playwright flow navigates the real mobile application, exercises the synthetic replay control, verifies provenance and watermark requirements, opens the deterministic Three.js LiDAR point cloud, -and writes the PNG files in this directory. +verifies the reconstruction boundary, target lock, and confidence HUD, and +writes the PNG files in this directory. The captures contain only the disconnected state, governed setup copy, and the built in synthetic fixture. The point cloud is generated locally from gated diff --git a/docs/screenshots/consumer-nlos-mobile-ui/point-cloud-390x844.png b/docs/screenshots/consumer-nlos-mobile-ui/point-cloud-390x844.png index e46ea355..46175f87 100644 Binary files a/docs/screenshots/consumer-nlos-mobile-ui/point-cloud-390x844.png and b/docs/screenshots/consumer-nlos-mobile-ui/point-cloud-390x844.png differ diff --git a/docs/screenshots/consumer-nlos-mobile-ui/synthetic-390x844.png b/docs/screenshots/consumer-nlos-mobile-ui/synthetic-390x844.png index 76454274..62d192bd 100644 Binary files a/docs/screenshots/consumer-nlos-mobile-ui/synthetic-390x844.png and b/docs/screenshots/consumer-nlos-mobile-ui/synthetic-390x844.png differ diff --git a/ui/ios-nlos/App/TrackCanvas.swift b/ui/ios-nlos/App/TrackCanvas.swift index 7f6a26c1..19016995 100644 --- a/ui/ios-nlos/App/TrackCanvas.swift +++ b/ui/ios-nlos/App/TrackCanvas.swift @@ -112,14 +112,48 @@ struct TrackCanvas: View { with: .color(color.opacity(0.82)) ) } + + var lockLine = Path() + lockLine.move(to: CGPoint(x: centerX, y: centerY - radiusY - 10)) + lockLine.addLine(to: CGPoint(x: centerX, y: centerY + radiusY + 10)) + context.stroke(lockLine, with: .color(color.opacity(0.55)), lineWidth: 1) + context.stroke( + Path(ellipseIn: CGRect( + x: centerX - radiusX - 5, + y: centerY - radiusY - 5, + width: (radiusX + 5) * 2, + height: (radiusY + 5) * 2 + )), + with: .color(color.opacity(0.86)), + style: StrokeStyle(lineWidth: 1.2, dash: [4, 3]) + ) + context.fill( + Path(ellipseIn: CGRect(x: centerX - 3, y: centerY - 3, width: 6, height: 6)), + with: .color(color) + ) + context.draw( + Text("\(String(track.trackId.prefix(12)).uppercased()) \(Int((track.confidence * 100).rounded()))%") + .font(.caption2.bold().monospaced()) + .foregroundColor(color), + at: CGPoint( + x: min(size.width - 62, centerX + radiusX + 34), + y: max(44, centerY - radiusY - 8) + ) + ) } context.draw( - Text("LIDAR POINT CLOUD / NATIVE SWIFTUI CANVAS") + Text("LIDAR POINT CLOUD / RECONSTRUCTION") .font(.caption2.bold().monospaced()) .foregroundColor(cyan), at: CGPoint(x: size.width / 2, y: 18) ) + context.draw( + Text("NOT RAW IPHONE LIDAR") + .font(.caption2.bold().monospaced()) + .foregroundColor(orange.opacity(0.82)), + at: CGPoint(x: size.width / 2, y: 34) + ) context.draw( Text("\(tracks.count * 72) GATED TARGET RETURNS") .font(.caption2.bold().monospaced()) diff --git a/ui/mobile/e2e/web/nlos.mobile.spec.ts b/ui/mobile/e2e/web/nlos.mobile.spec.ts index 4aa4a039..9d058d60 100644 --- a/ui/mobile/e2e/web/nlos.mobile.spec.ts +++ b/ui/mobile/e2e/web/nlos.mobile.spec.ts @@ -119,7 +119,10 @@ test.describe('RuView NLOS mobile instrument UI', () => { await expect(cloud).toBeVisible(); await expect(page.getByTestId('nlos-lidar-point-cloud-canvas')).toHaveAttribute('data-ready', 'true'); await expect(page.getByTestId('nlos-cloud-target-count')).toHaveText('96'); + await expect(page.getByTestId('nlos-cloud-boundary-label')).toHaveText('RECONSTRUCTION / NOT RAW SCAN'); await expect(page.getByText('THREE.JS / WEBGL')).toBeVisible(); + await expect(page.getByText('01 TRACK LOCK')).toBeVisible(); + await expect(page.getByText('72% CONFIDENCE')).toBeVisible(); await expect(page.getByTestId('nlos-synthetic-watermark')).toBeVisible(); const dimensions = await cloud.evaluate((element) => ({ diff --git a/ui/mobile/src/__tests__/screens/NLOSScreen.test.tsx b/ui/mobile/src/__tests__/screens/NLOSScreen.test.tsx index 071b67b0..2b6ff1ef 100644 --- a/ui/mobile/src/__tests__/screens/NLOSScreen.test.tsx +++ b/ui/mobile/src/__tests__/screens/NLOSScreen.test.tsx @@ -14,6 +14,7 @@ import { buildLidarPointCloud, LIDAR_POINTS_PER_TRACK, LIDAR_RELAY_POINT_COUNT, + resolveLidarTrackCenter, } from '@/screens/NLOSScreen/lidarPointCloud'; const mockSafeAreaInsets = { top: 0, right: 0, bottom: 0, left: 0 }; @@ -310,6 +311,7 @@ describe('NLOSScreen', () => { expect(Array.from(first.positions)).toEqual(Array.from(second.positions)); expect(Array.from(first.colors)).toEqual(Array.from(second.colors)); expect(Array.from(first.positions).every(Number.isFinite)).toBe(true); + expect(resolveLidarTrackCenter(syntheticFrame.tracks[0]).every(Number.isFinite)).toBe(true); }); it('keeps privacy, setup, explainer, and feedback controls visible in the screen tree', () => { @@ -333,6 +335,7 @@ describe('NLOSScreen', () => { fireEvent.press(screen.getByTestId('nlos-view-cloud')); expect(screen.getByTestId('nlos-view-cloud').props.accessibilityState.selected).toBe(true); expect(screen.getByTestId('nlos-lidar-point-cloud')).toBeTruthy(); + expect(screen.getByText('RECONSTRUCTION / NOT RAW SCAN')).toBeTruthy(); expect(screen.getByTestId('nlos-cloud-target-count').props.children).toBe( syntheticFrame.tracks.length * LIDAR_POINTS_PER_TRACK, ); diff --git a/ui/mobile/src/screens/NLOSScreen/LidarPointCloud.tsx b/ui/mobile/src/screens/NLOSScreen/LidarPointCloud.tsx index 02593292..a8b2c337 100644 --- a/ui/mobile/src/screens/NLOSScreen/LidarPointCloud.tsx +++ b/ui/mobile/src/screens/NLOSScreen/LidarPointCloud.tsx @@ -1,10 +1,14 @@ -import { memo, useMemo } from 'react'; +import { Fragment, memo, useMemo } from 'react'; import { StyleSheet, View } from 'react-native'; import Svg, { Circle, Line, Rect, Text as SvgText } from 'react-native-svg'; import { instrumentColors } from '@/components/InstrumentPanel'; import { ThemedText } from '@/components/ThemedText'; import type { NlosFreshness, NlosTrack } from '@/types/nlos'; -import { buildLidarPointCloud, LIDAR_RELAY_POINT_COUNT } from './lidarPointCloud'; +import { + buildLidarPointCloud, + LIDAR_RELAY_POINT_COUNT, + resolveLidarTrackCenter, +} from './lidarPointCloud'; interface LidarPointCloudProps { tracks: NlosTrack[]; @@ -47,6 +51,16 @@ export const LidarPointCloud = memo(({ tracks, freshness, width }: LidarPointClo return projected; }, [cloud]); const displayWidth = Math.max(260, Math.min(width, 560)); + const markers = useMemo(() => tracks.slice(0, 16).map((track) => { + const [x, y, z] = resolveLidarTrackCenter(track); + return { + color: track.state === 'degraded' ? instrumentColors.warning : instrumentColors.green, + confidence: Math.round(track.confidence * 100), + id: track.trackId.toUpperCase(), + x: 180 + x * 34 - z * 8, + y: 220 - y * 46 + z * 7, + }; + }), [tracks]); return ( ))} + {markers.map((marker) => ( + + + + + {marker.id} + {marker.confidence}% CONF. + + ))} LIDAR CLOUD / NATIVE PROJECTION + RECONSTRUCTION / NOT RAW SCAN + PROJECTED diff --git a/ui/mobile/src/screens/NLOSScreen/LidarPointCloud.web.tsx b/ui/mobile/src/screens/NLOSScreen/LidarPointCloud.web.tsx index 599ce4fc..c462064c 100644 --- a/ui/mobile/src/screens/NLOSScreen/LidarPointCloud.web.tsx +++ b/ui/mobile/src/screens/NLOSScreen/LidarPointCloud.web.tsx @@ -4,7 +4,11 @@ import * as THREE from 'three'; import { instrumentColors } from '@/components/InstrumentPanel'; import { ThemedText } from '@/components/ThemedText'; import type { NlosFreshness, NlosTrack } from '@/types/nlos'; -import { buildLidarPointCloud } from './lidarPointCloud'; +import { + buildLidarPointCloud, + LIDAR_MAX_TRACKS, + resolveLidarTrackCenter, +} from './lidarPointCloud'; interface LidarPointCloudProps { tracks: NlosTrack[]; @@ -19,6 +23,8 @@ const CANVAS_ASPECT_RATIO = 360 / 260; interface WebSceneState { geometry: THREE.BufferGeometry; material: THREE.PointsMaterial; + markerGroup: THREE.Group; + markerSignature: string; points: THREE.Points; render: () => void; } @@ -28,6 +34,16 @@ const disposeMaterial = (material: THREE.Material | THREE.Material[]) => { else material.dispose(); }; +const clearGroup = (group: THREE.Group) => { + group.traverse((entry) => { + if (entry instanceof THREE.Mesh || entry instanceof THREE.Line) { + entry.geometry.dispose(); + disposeMaterial(entry.material); + } + }); + group.clear(); +}; + const rgb = (red: number, green: number, blue: number) => ( `rgb(${Math.round(red * 255)}, ${Math.round(green * 255)}, ${Math.round(blue * 255)})` ); @@ -114,6 +130,9 @@ export const LidarPointCloud = memo(({ tracks, freshness, width }: LidarPointClo const points = new THREE.Points(geometry, material); scene.add(points); + const markerGroup = new THREE.Group(); + scene.add(markerGroup); + const floorGrid = new THREE.GridHelper(8, 16, 0x174957, 0x102934); floorGrid.position.y = -0.04; floorGrid.position.z = -1.25; @@ -160,7 +179,14 @@ export const LidarPointCloud = memo(({ tracks, freshness, width }: LidarPointClo renderer.render(scene, camera); }; - sceneRef.current = { geometry, material, points, render: renderScene }; + sceneRef.current = { + geometry, + markerGroup, + markerSignature: '', + material, + points, + render: renderScene, + }; const onPointerDown = (event: PointerEvent) => { dragging = true; @@ -245,7 +271,8 @@ export const LidarPointCloud = memo(({ tracks, freshness, width }: LidarPointClo window.cancelAnimationFrame(animationFrame); resizeObserver?.disconnect(); listeners.forEach((remove) => remove()); - scene.remove(points, floorGrid, relayPlane, sensor, ray); + scene.remove(points, markerGroup, floorGrid, relayPlane, sensor, ray); + clearGroup(markerGroup); geometry.dispose(); material.dispose(); floorGrid.geometry.dispose(); @@ -296,8 +323,63 @@ export const LidarPointCloud = memo(({ tracks, freshness, width }: LidarPointClo state.geometry.computeBoundingSphere(); state.material.opacity = freshness === 'fresh' ? 0.92 : 0.34; state.material.needsUpdate = true; + + const markerTracks = tracks.slice(0, LIDAR_MAX_TRACKS); + const markerSignature = markerTracks.map((track) => track.state).join('|'); + if (markerSignature !== state.markerSignature) { + clearGroup(state.markerGroup); + state.markerSignature = markerSignature; + markerTracks.forEach((track) => { + const degraded = track.state === 'degraded'; + const color = degraded ? 0xffb65c : 0x58f28b; + const opacity = freshness === 'fresh' ? 0.82 : 0.28; + const horizontalRing = new THREE.Mesh( + new THREE.TorusGeometry(0.31, 0.016, 5, 40), + new THREE.MeshBasicMaterial({ color, opacity, transparent: true }), + ); + horizontalRing.rotation.x = Math.PI / 2; + state.markerGroup.add(horizontalRing); + + const core = new THREE.Mesh( + new THREE.OctahedronGeometry(0.08, 0), + new THREE.MeshBasicMaterial({ color, wireframe: true }), + ); + state.markerGroup.add(core); + + const pillarGeometry = new THREE.BufferGeometry(); + pillarGeometry.setAttribute('position', new THREE.BufferAttribute(new Float32Array(6), 3)); + const pillar = new THREE.Line( + pillarGeometry, + new THREE.LineBasicMaterial({ color, opacity: opacity * 0.62, transparent: true }), + ); + state.markerGroup.add(pillar); + }); + } + + markerTracks.forEach((track, index) => { + const [x, y, z] = resolveLidarTrackCenter(track); + const degraded = track.state === 'degraded'; + const opacity = freshness === 'fresh' ? 0.82 : 0.28; + const horizontalRing = state.markerGroup.children[index * 3] as THREE.Mesh; + horizontalRing.position.set(x, y, z); + (horizontalRing.material as THREE.MeshBasicMaterial).opacity = opacity; + + const core = state.markerGroup.children[index * 3 + 1] as THREE.Mesh; + core.position.set(x, y, z); + (core.material as THREE.MeshBasicMaterial).opacity = degraded ? 0.82 : 1; + (core.material as THREE.MeshBasicMaterial).transparent = degraded; + + const pillar = state.markerGroup.children[index * 3 + 2] as THREE.Line; + const pillarPosition = pillar.geometry.getAttribute('position') as THREE.BufferAttribute; + (pillarPosition.array as Float32Array).set([ + x, Math.max(0.02, y - 0.52), z, + x, y + 0.52, z, + ]); + pillarPosition.needsUpdate = true; + (pillar.material as THREE.LineBasicMaterial).opacity = opacity * 0.62; + }); state.render(); - }, [cloud, freshness]); + }, [cloud, freshness, tracks]); return ( ) : null} - LIDAR POINT CLOUD - - {rendererState === 'fallback' ? 'STATIC FALLBACK' : 'THREE.JS / WEBGL'} - + + LIDAR POINT CLOUD + + RECONSTRUCTION / NOT RAW SCAN + + + + + {rendererState === 'fallback' ? 'STATIC FALLBACK' : 'THREE.JS / WEBGL'} + + + {tracks.length ? `${String(tracks.length).padStart(2, '0')} TRACK LOCK` : 'NO TARGET LOCK'} + + + + RELAY + + HIDDEN + + {tracks[0] ? ( + + + + + {tracks[0].trackId.toUpperCase()} + + + {Math.round(tracks[0].confidence * 100)}% CONFIDENCE + + + + ) : null} @@ -370,12 +480,46 @@ const styles = StyleSheet.create({ right: 12, zIndex: 2, flexDirection: 'row', - alignItems: 'center', + alignItems: 'flex-start', justifyContent: 'space-between', gap: 8, }, + titleStack: { gap: 3 }, hudTitle: { color: instrumentColors.cyan, fontSize: 9, letterSpacing: 1.15 }, + boundaryLabel: { color: instrumentColors.warning, fontSize: 6.5, letterSpacing: 0.7 }, + rendererStack: { alignItems: 'flex-end', gap: 3 }, rendererLabel: { color: instrumentColors.textSecondary, fontSize: 8, letterSpacing: 0.8 }, + lockLabel: { color: instrumentColors.green, fontSize: 7, letterSpacing: 0.7 }, + depthScale: { + position: 'absolute', + left: 12, + top: 70, + bottom: 54, + zIndex: 2, + alignItems: 'center', + justifyContent: 'space-between', + }, + depthLine: { flex: 1, width: 1, marginVertical: 5, backgroundColor: instrumentColors.cyanDim }, + depthLabel: { color: instrumentColors.textSecondary, fontSize: 6, letterSpacing: 0.55 }, + trackChip: { + position: 'absolute', + top: 54, + right: 12, + zIndex: 2, + minHeight: 34, + flexDirection: 'row', + alignItems: 'center', + gap: 7, + paddingHorizontal: 9, + paddingVertical: 6, + borderRadius: 8, + borderColor: instrumentColors.greenDim, + borderWidth: 1, + backgroundColor: 'rgba(5, 14, 18, 0.76)', + }, + trackChipDot: { width: 6, height: 6, borderRadius: 3, backgroundColor: instrumentColors.green }, + trackChipId: { color: instrumentColors.text, fontSize: 7, letterSpacing: 0.65 }, + trackChipMeta: { color: instrumentColors.green, fontSize: 6.5, letterSpacing: 0.5 }, bottomHud: { position: 'absolute', left: 12, diff --git a/ui/mobile/src/screens/NLOSScreen/index.tsx b/ui/mobile/src/screens/NLOSScreen/index.tsx index 886841a8..f246f724 100644 --- a/ui/mobile/src/screens/NLOSScreen/index.tsx +++ b/ui/mobile/src/screens/NLOSScreen/index.tsx @@ -30,10 +30,10 @@ const ViewModePicker = ({ }) => ( {([ - ['plan', '2D PLAN'], - ['perspective', '3D VIEW'], - ['cloud', 'LIDAR CLOUD'], - ] as const).map(([option, label]) => { + ['plan', '2D', 'PLAN'], + ['perspective', '3D', 'SCENE'], + ['cloud', 'LIDAR', 'CLOUD'], + ] as const).map(([option, eyebrow, label]) => { const selected = option === value; return ( onChange(option)} style={[styles.pickerButton, selected && styles.pickerButtonSelected]} > + + {eyebrow} + [ + clamp(track.positionM.x, -6, 6) * 0.55, + 0.45 + clamp(track.positionM.y, 0, 4) * 0.5, + 0.35 - clamp(track.positionM.z, 0, 8) * 0.58, +]; + export const buildLidarPointCloud = (tracks: readonly NlosTrack[]): LidarPointCloudData => { const visibleTracks = tracks.slice(0, LIDAR_MAX_TRACKS); const targetPointCount = visibleTracks.length * LIDAR_POINTS_PER_TRACK; @@ -79,9 +85,7 @@ export const buildLidarPointCloud = (tracks: readonly NlosTrack[]): LidarPointCl const uncertaintyX = clamp(Math.sqrt(track.covarianceDiagonalM2.x), 0.08, 0.85); const uncertaintyY = clamp(Math.sqrt(track.covarianceDiagonalM2.y), 0.08, 0.85); const uncertaintyZ = clamp(Math.sqrt(track.covarianceDiagonalM2.z), 0.08, 0.85); - const centerX = clamp(track.positionM.x, -6, 6) * 0.55; - const centerY = 0.45 + clamp(track.positionM.y, 0, 4) * 0.5; - const centerZ = 0.35 - clamp(track.positionM.z, 0, 8) * 0.58; + const [centerX, centerY, centerZ] = resolveLidarTrackCenter(track); const degraded = track.state === 'degraded'; const intensity = 0.58 + clamp(track.confidence, 0, 1) * 0.42;