feat(mobile): refine spatial target UI

This commit is contained in:
rUv
2026-08-23 21:09:38 -04:00
parent 03b1f2459e
commit 10ec346db1
11 changed files with 254 additions and 22 deletions

View File

@@ -337,6 +337,13 @@ consume only the existing fail-closed displayable track list. Do not add
`expo-gl`, React Three Fiber, a WebView, a raw point-cloud schema, or a new sensor
adapter in this UI-only decision.
Render explicit target-lock reticles, a relay-to-hidden depth scale, confidence,
and a persistent `RECONSTRUCTION / NOT RAW SCAN` boundary in the cloud surface.
The reticle geometry reuses bounded marker objects and updates their positions in
place so frame changes do not create a new WebGL context or allocate new marker
meshes. These cues improve operator target acquisition without promoting the
schematic cloud into physical sensor evidence.
## Alternatives with quantified tradeoffs
Alternatives are scored from 1, poor, to 5, strong. Weighted score is out of

View File

@@ -4,7 +4,8 @@ These images are deterministic review captures from the production Expo web
export at a 390 by 844 viewport. The Playwright flow navigates the real mobile
application, exercises the synthetic replay control, verifies provenance and
watermark requirements, opens the deterministic Three.js LiDAR point cloud,
and writes the PNG files in this directory.
verifies the reconstruction boundary, target lock, and confidence HUD, and
writes the PNG files in this directory.
The captures contain only the disconnected state, governed setup copy, and the
built in synthetic fixture. The point cloud is generated locally from gated

Binary file not shown.

Before

Width:  |  Height:  |  Size: 69 KiB

After

Width:  |  Height:  |  Size: 74 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 53 KiB

After

Width:  |  Height:  |  Size: 52 KiB

View File

@@ -112,14 +112,48 @@ struct TrackCanvas: View {
with: .color(color.opacity(0.82))
)
}
var lockLine = Path()
lockLine.move(to: CGPoint(x: centerX, y: centerY - radiusY - 10))
lockLine.addLine(to: CGPoint(x: centerX, y: centerY + radiusY + 10))
context.stroke(lockLine, with: .color(color.opacity(0.55)), lineWidth: 1)
context.stroke(
Path(ellipseIn: CGRect(
x: centerX - radiusX - 5,
y: centerY - radiusY - 5,
width: (radiusX + 5) * 2,
height: (radiusY + 5) * 2
)),
with: .color(color.opacity(0.86)),
style: StrokeStyle(lineWidth: 1.2, dash: [4, 3])
)
context.fill(
Path(ellipseIn: CGRect(x: centerX - 3, y: centerY - 3, width: 6, height: 6)),
with: .color(color)
)
context.draw(
Text("\(String(track.trackId.prefix(12)).uppercased()) \(Int((track.confidence * 100).rounded()))%")
.font(.caption2.bold().monospaced())
.foregroundColor(color),
at: CGPoint(
x: min(size.width - 62, centerX + radiusX + 34),
y: max(44, centerY - radiusY - 8)
)
)
}
context.draw(
Text("LIDAR POINT CLOUD / NATIVE SWIFTUI CANVAS")
Text("LIDAR POINT CLOUD / RECONSTRUCTION")
.font(.caption2.bold().monospaced())
.foregroundColor(cyan),
at: CGPoint(x: size.width / 2, y: 18)
)
context.draw(
Text("NOT RAW IPHONE LIDAR")
.font(.caption2.bold().monospaced())
.foregroundColor(orange.opacity(0.82)),
at: CGPoint(x: size.width / 2, y: 34)
)
context.draw(
Text("\(tracks.count * 72) GATED TARGET RETURNS")
.font(.caption2.bold().monospaced())

View File

@@ -119,7 +119,10 @@ test.describe('RuView NLOS mobile instrument UI', () => {
await expect(cloud).toBeVisible();
await expect(page.getByTestId('nlos-lidar-point-cloud-canvas')).toHaveAttribute('data-ready', 'true');
await expect(page.getByTestId('nlos-cloud-target-count')).toHaveText('96');
await expect(page.getByTestId('nlos-cloud-boundary-label')).toHaveText('RECONSTRUCTION / NOT RAW SCAN');
await expect(page.getByText('THREE.JS / WEBGL')).toBeVisible();
await expect(page.getByText('01 TRACK LOCK')).toBeVisible();
await expect(page.getByText('72% CONFIDENCE')).toBeVisible();
await expect(page.getByTestId('nlos-synthetic-watermark')).toBeVisible();
const dimensions = await cloud.evaluate((element) => ({

View File

@@ -14,6 +14,7 @@ import {
buildLidarPointCloud,
LIDAR_POINTS_PER_TRACK,
LIDAR_RELAY_POINT_COUNT,
resolveLidarTrackCenter,
} from '@/screens/NLOSScreen/lidarPointCloud';
const mockSafeAreaInsets = { top: 0, right: 0, bottom: 0, left: 0 };
@@ -310,6 +311,7 @@ describe('NLOSScreen', () => {
expect(Array.from(first.positions)).toEqual(Array.from(second.positions));
expect(Array.from(first.colors)).toEqual(Array.from(second.colors));
expect(Array.from(first.positions).every(Number.isFinite)).toBe(true);
expect(resolveLidarTrackCenter(syntheticFrame.tracks[0]).every(Number.isFinite)).toBe(true);
});
it('keeps privacy, setup, explainer, and feedback controls visible in the screen tree', () => {
@@ -333,6 +335,7 @@ describe('NLOSScreen', () => {
fireEvent.press(screen.getByTestId('nlos-view-cloud'));
expect(screen.getByTestId('nlos-view-cloud').props.accessibilityState.selected).toBe(true);
expect(screen.getByTestId('nlos-lidar-point-cloud')).toBeTruthy();
expect(screen.getByText('RECONSTRUCTION / NOT RAW SCAN')).toBeTruthy();
expect(screen.getByTestId('nlos-cloud-target-count').props.children).toBe(
syntheticFrame.tracks.length * LIDAR_POINTS_PER_TRACK,
);

View File

@@ -1,10 +1,14 @@
import { memo, useMemo } from 'react';
import { Fragment, memo, useMemo } from 'react';
import { StyleSheet, View } from 'react-native';
import Svg, { Circle, Line, Rect, Text as SvgText } from 'react-native-svg';
import { instrumentColors } from '@/components/InstrumentPanel';
import { ThemedText } from '@/components/ThemedText';
import type { NlosFreshness, NlosTrack } from '@/types/nlos';
import { buildLidarPointCloud, LIDAR_RELAY_POINT_COUNT } from './lidarPointCloud';
import {
buildLidarPointCloud,
LIDAR_RELAY_POINT_COUNT,
resolveLidarTrackCenter,
} from './lidarPointCloud';
interface LidarPointCloudProps {
tracks: NlosTrack[];
@@ -47,6 +51,16 @@ export const LidarPointCloud = memo(({ tracks, freshness, width }: LidarPointClo
return projected;
}, [cloud]);
const displayWidth = Math.max(260, Math.min(width, 560));
const markers = useMemo(() => tracks.slice(0, 16).map((track) => {
const [x, y, z] = resolveLidarTrackCenter(track);
return {
color: track.state === 'degraded' ? instrumentColors.warning : instrumentColors.green,
confidence: Math.round(track.confidence * 100),
id: track.trackId.toUpperCase(),
x: 180 + x * 34 - z * 8,
y: 220 - y * 46 + z * 7,
};
}), [tracks]);
return (
<View
@@ -70,7 +84,18 @@ export const LidarPointCloud = memo(({ tracks, freshness, width }: LidarPointClo
opacity={freshness === 'fresh' ? 0.88 : 0.35}
/>
))}
{markers.map((marker) => (
<Fragment key={`cloud-marker-${marker.id}`}>
<Line x1={marker.x} y1={marker.y - 20} x2={marker.x} y2={marker.y + 20} stroke={marker.color} opacity={0.55} />
<Circle cx={marker.x} cy={marker.y} r={11} fill="none" stroke={marker.color} strokeWidth={1.4} />
<Circle cx={marker.x} cy={marker.y} r={3} fill={marker.color} />
<SvgText x={marker.x + 15} y={marker.y - 5} fill={instrumentColors.text} fontSize={7}>{marker.id}</SvgText>
<SvgText x={marker.x + 15} y={marker.y + 6} fill={marker.color} fontSize={6}>{marker.confidence}% CONF.</SvgText>
</Fragment>
))}
<SvgText x={24} y={34} fill={instrumentColors.cyan} fontSize={9} letterSpacing={1.1}>LIDAR CLOUD / NATIVE PROJECTION</SvgText>
<SvgText x={24} y={45} fill={instrumentColors.warning} fontSize={6.5} letterSpacing={0.7}>RECONSTRUCTION / NOT RAW SCAN</SvgText>
<SvgText x={276} y={34} fill={instrumentColors.textSecondary} fontSize={7} letterSpacing={0.7}>PROJECTED</SvgText>
</Svg>
<View pointerEvents="none" style={styles.metricRow}>
<ThemedText testID="nlos-cloud-target-count" preset="labelLg" style={styles.metricValue}>

View File

@@ -4,7 +4,11 @@ import * as THREE from 'three';
import { instrumentColors } from '@/components/InstrumentPanel';
import { ThemedText } from '@/components/ThemedText';
import type { NlosFreshness, NlosTrack } from '@/types/nlos';
import { buildLidarPointCloud } from './lidarPointCloud';
import {
buildLidarPointCloud,
LIDAR_MAX_TRACKS,
resolveLidarTrackCenter,
} from './lidarPointCloud';
interface LidarPointCloudProps {
tracks: NlosTrack[];
@@ -19,6 +23,8 @@ const CANVAS_ASPECT_RATIO = 360 / 260;
interface WebSceneState {
geometry: THREE.BufferGeometry;
material: THREE.PointsMaterial;
markerGroup: THREE.Group;
markerSignature: string;
points: THREE.Points;
render: () => void;
}
@@ -28,6 +34,16 @@ const disposeMaterial = (material: THREE.Material | THREE.Material[]) => {
else material.dispose();
};
const clearGroup = (group: THREE.Group) => {
group.traverse((entry) => {
if (entry instanceof THREE.Mesh || entry instanceof THREE.Line) {
entry.geometry.dispose();
disposeMaterial(entry.material);
}
});
group.clear();
};
const rgb = (red: number, green: number, blue: number) => (
`rgb(${Math.round(red * 255)}, ${Math.round(green * 255)}, ${Math.round(blue * 255)})`
);
@@ -114,6 +130,9 @@ export const LidarPointCloud = memo(({ tracks, freshness, width }: LidarPointClo
const points = new THREE.Points(geometry, material);
scene.add(points);
const markerGroup = new THREE.Group();
scene.add(markerGroup);
const floorGrid = new THREE.GridHelper(8, 16, 0x174957, 0x102934);
floorGrid.position.y = -0.04;
floorGrid.position.z = -1.25;
@@ -160,7 +179,14 @@ export const LidarPointCloud = memo(({ tracks, freshness, width }: LidarPointClo
renderer.render(scene, camera);
};
sceneRef.current = { geometry, material, points, render: renderScene };
sceneRef.current = {
geometry,
markerGroup,
markerSignature: '',
material,
points,
render: renderScene,
};
const onPointerDown = (event: PointerEvent) => {
dragging = true;
@@ -245,7 +271,8 @@ export const LidarPointCloud = memo(({ tracks, freshness, width }: LidarPointClo
window.cancelAnimationFrame(animationFrame);
resizeObserver?.disconnect();
listeners.forEach((remove) => remove());
scene.remove(points, floorGrid, relayPlane, sensor, ray);
scene.remove(points, markerGroup, floorGrid, relayPlane, sensor, ray);
clearGroup(markerGroup);
geometry.dispose();
material.dispose();
floorGrid.geometry.dispose();
@@ -296,8 +323,63 @@ export const LidarPointCloud = memo(({ tracks, freshness, width }: LidarPointClo
state.geometry.computeBoundingSphere();
state.material.opacity = freshness === 'fresh' ? 0.92 : 0.34;
state.material.needsUpdate = true;
const markerTracks = tracks.slice(0, LIDAR_MAX_TRACKS);
const markerSignature = markerTracks.map((track) => track.state).join('|');
if (markerSignature !== state.markerSignature) {
clearGroup(state.markerGroup);
state.markerSignature = markerSignature;
markerTracks.forEach((track) => {
const degraded = track.state === 'degraded';
const color = degraded ? 0xffb65c : 0x58f28b;
const opacity = freshness === 'fresh' ? 0.82 : 0.28;
const horizontalRing = new THREE.Mesh(
new THREE.TorusGeometry(0.31, 0.016, 5, 40),
new THREE.MeshBasicMaterial({ color, opacity, transparent: true }),
);
horizontalRing.rotation.x = Math.PI / 2;
state.markerGroup.add(horizontalRing);
const core = new THREE.Mesh(
new THREE.OctahedronGeometry(0.08, 0),
new THREE.MeshBasicMaterial({ color, wireframe: true }),
);
state.markerGroup.add(core);
const pillarGeometry = new THREE.BufferGeometry();
pillarGeometry.setAttribute('position', new THREE.BufferAttribute(new Float32Array(6), 3));
const pillar = new THREE.Line(
pillarGeometry,
new THREE.LineBasicMaterial({ color, opacity: opacity * 0.62, transparent: true }),
);
state.markerGroup.add(pillar);
});
}
markerTracks.forEach((track, index) => {
const [x, y, z] = resolveLidarTrackCenter(track);
const degraded = track.state === 'degraded';
const opacity = freshness === 'fresh' ? 0.82 : 0.28;
const horizontalRing = state.markerGroup.children[index * 3] as THREE.Mesh;
horizontalRing.position.set(x, y, z);
(horizontalRing.material as THREE.MeshBasicMaterial).opacity = opacity;
const core = state.markerGroup.children[index * 3 + 1] as THREE.Mesh;
core.position.set(x, y, z);
(core.material as THREE.MeshBasicMaterial).opacity = degraded ? 0.82 : 1;
(core.material as THREE.MeshBasicMaterial).transparent = degraded;
const pillar = state.markerGroup.children[index * 3 + 2] as THREE.Line;
const pillarPosition = pillar.geometry.getAttribute('position') as THREE.BufferAttribute;
(pillarPosition.array as Float32Array).set([
x, Math.max(0.02, y - 0.52), z,
x, y + 0.52, z,
]);
pillarPosition.needsUpdate = true;
(pillar.material as THREE.LineBasicMaterial).opacity = opacity * 0.62;
});
state.render();
}, [cloud, freshness]);
}, [cloud, freshness, tracks]);
return (
<View
@@ -321,11 +403,39 @@ export const LidarPointCloud = memo(({ tracks, freshness, width }: LidarPointClo
</View>
) : null}
<View pointerEvents="none" style={styles.topHud}>
<ThemedText preset="mono" style={styles.hudTitle}>LIDAR POINT CLOUD</ThemedText>
<ThemedText preset="mono" style={styles.rendererLabel}>
{rendererState === 'fallback' ? 'STATIC FALLBACK' : 'THREE.JS / WEBGL'}
</ThemedText>
<View style={styles.titleStack}>
<ThemedText preset="mono" style={styles.hudTitle}>LIDAR POINT CLOUD</ThemedText>
<ThemedText testID="nlos-cloud-boundary-label" preset="mono" style={styles.boundaryLabel}>
RECONSTRUCTION / NOT RAW SCAN
</ThemedText>
</View>
<View style={styles.rendererStack}>
<ThemedText preset="mono" style={styles.rendererLabel}>
{rendererState === 'fallback' ? 'STATIC FALLBACK' : 'THREE.JS / WEBGL'}
</ThemedText>
<ThemedText preset="mono" style={styles.lockLabel}>
{tracks.length ? `${String(tracks.length).padStart(2, '0')} TRACK LOCK` : 'NO TARGET LOCK'}
</ThemedText>
</View>
</View>
<View pointerEvents="none" style={styles.depthScale}>
<ThemedText preset="mono" style={styles.depthLabel}>RELAY</ThemedText>
<View style={styles.depthLine} />
<ThemedText preset="mono" style={styles.depthLabel}>HIDDEN</ThemedText>
</View>
{tracks[0] ? (
<View pointerEvents="none" style={styles.trackChip}>
<View style={styles.trackChipDot} />
<View>
<ThemedText preset="mono" style={styles.trackChipId}>
{tracks[0].trackId.toUpperCase()}
</ThemedText>
<ThemedText preset="mono" style={styles.trackChipMeta}>
{Math.round(tracks[0].confidence * 100)}% CONFIDENCE
</ThemedText>
</View>
</View>
) : null}
<View pointerEvents="none" style={styles.bottomHud}>
<View>
<ThemedText testID="nlos-cloud-target-count" preset="labelLg" style={styles.metricValue}>
@@ -370,12 +480,46 @@ const styles = StyleSheet.create({
right: 12,
zIndex: 2,
flexDirection: 'row',
alignItems: 'center',
alignItems: 'flex-start',
justifyContent: 'space-between',
gap: 8,
},
titleStack: { gap: 3 },
hudTitle: { color: instrumentColors.cyan, fontSize: 9, letterSpacing: 1.15 },
boundaryLabel: { color: instrumentColors.warning, fontSize: 6.5, letterSpacing: 0.7 },
rendererStack: { alignItems: 'flex-end', gap: 3 },
rendererLabel: { color: instrumentColors.textSecondary, fontSize: 8, letterSpacing: 0.8 },
lockLabel: { color: instrumentColors.green, fontSize: 7, letterSpacing: 0.7 },
depthScale: {
position: 'absolute',
left: 12,
top: 70,
bottom: 54,
zIndex: 2,
alignItems: 'center',
justifyContent: 'space-between',
},
depthLine: { flex: 1, width: 1, marginVertical: 5, backgroundColor: instrumentColors.cyanDim },
depthLabel: { color: instrumentColors.textSecondary, fontSize: 6, letterSpacing: 0.55 },
trackChip: {
position: 'absolute',
top: 54,
right: 12,
zIndex: 2,
minHeight: 34,
flexDirection: 'row',
alignItems: 'center',
gap: 7,
paddingHorizontal: 9,
paddingVertical: 6,
borderRadius: 8,
borderColor: instrumentColors.greenDim,
borderWidth: 1,
backgroundColor: 'rgba(5, 14, 18, 0.76)',
},
trackChipDot: { width: 6, height: 6, borderRadius: 3, backgroundColor: instrumentColors.green },
trackChipId: { color: instrumentColors.text, fontSize: 7, letterSpacing: 0.65 },
trackChipMeta: { color: instrumentColors.green, fontSize: 6.5, letterSpacing: 0.5 },
bottomHud: {
position: 'absolute',
left: 12,

View File

@@ -30,10 +30,10 @@ const ViewModePicker = ({
}) => (
<View accessibilityRole="tablist" style={styles.picker}>
{([
['plan', '2D PLAN'],
['perspective', '3D VIEW'],
['cloud', 'LIDAR CLOUD'],
] as const).map(([option, label]) => {
['plan', '2D', 'PLAN'],
['perspective', '3D', 'SCENE'],
['cloud', 'LIDAR', 'CLOUD'],
] as const).map(([option, eyebrow, label]) => {
const selected = option === value;
return (
<Pressable
@@ -44,6 +44,15 @@ const ViewModePicker = ({
onPress={() => onChange(option)}
style={[styles.pickerButton, selected && styles.pickerButtonSelected]}
>
<ThemedText
preset="mono"
style={[
styles.pickerEyebrow,
{ color: selected ? instrumentColors.green : instrumentColors.textSecondary },
]}
>
{eyebrow}
</ThemedText>
<ThemedText
preset="mono"
style={[
@@ -448,6 +457,7 @@ const styles = StyleSheet.create({
flex: 1,
alignItems: 'center',
justifyContent: 'center',
gap: 2,
borderRadius: 9,
},
pickerButtonSelected: {
@@ -455,8 +465,9 @@ const styles = StyleSheet.create({
borderColor: instrumentColors.cyanDim,
borderWidth: 1,
},
pickerLabel: { fontSize: 9, letterSpacing: 0.55, textAlign: 'center' },
visualizationStage: { position: 'relative', overflow: 'hidden' },
pickerEyebrow: { fontSize: 7, letterSpacing: 0.8, textAlign: 'center' },
pickerLabel: { fontSize: 10, letterSpacing: 0.7, textAlign: 'center' },
visualizationStage: { position: 'relative', overflow: 'hidden', paddingTop: spacing.xs },
watermark: {
...StyleSheet.absoluteFillObject,
alignItems: 'center',

View File

@@ -34,6 +34,12 @@ const writePoint = (
colors[offset + 2] = color[2];
};
export const resolveLidarTrackCenter = (track: NlosTrack): readonly [number, number, number] => [
clamp(track.positionM.x, -6, 6) * 0.55,
0.45 + clamp(track.positionM.y, 0, 4) * 0.5,
0.35 - clamp(track.positionM.z, 0, 8) * 0.58,
];
export const buildLidarPointCloud = (tracks: readonly NlosTrack[]): LidarPointCloudData => {
const visibleTracks = tracks.slice(0, LIDAR_MAX_TRACKS);
const targetPointCount = visibleTracks.length * LIDAR_POINTS_PER_TRACK;
@@ -79,9 +85,7 @@ export const buildLidarPointCloud = (tracks: readonly NlosTrack[]): LidarPointCl
const uncertaintyX = clamp(Math.sqrt(track.covarianceDiagonalM2.x), 0.08, 0.85);
const uncertaintyY = clamp(Math.sqrt(track.covarianceDiagonalM2.y), 0.08, 0.85);
const uncertaintyZ = clamp(Math.sqrt(track.covarianceDiagonalM2.z), 0.08, 0.85);
const centerX = clamp(track.positionM.x, -6, 6) * 0.55;
const centerY = 0.45 + clamp(track.positionM.y, 0, 4) * 0.5;
const centerZ = 0.35 - clamp(track.positionM.z, 0, 8) * 0.58;
const [centerX, centerY, centerZ] = resolveLidarTrackCenter(track);
const degraded = track.state === 'degraded';
const intensity = 0.58 + clamp(track.confidence, 0, 1) * 0.42;