feat: add native iPhone LiDAR sensor and web viewer (#1684)

* feat(ios): add RuView LiDAR frame protocol

* feat(ios): capture ARKit scene depth for RuView

* feat(ios): stream compact LiDAR frames over websocket

* feat(ios): add native LiDAR capture UI

* feat(ios): add RuView LiDAR app entrypoint

* feat(web): add LiDAR bridge web package

* feat(web): decode RuView LiDAR wire frames

* feat(web): add local LiDAR websocket relay

* feat(web): add LiDAR browser viewer

* feat(web): render live LiDAR point cloud

* fix(ios): use wall clock time for LiDAR provenance

* feat(web): style LiDAR viewer

* test(web): add LiDAR codec tests

* docs: add iPhone LiDAR integration guide

* docs(adr): define iPhone LiDAR sensor bridge

* fix(ios): harden and validate LiDAR bridge

* fix(ios): qualify LiDAR wire depth type
This commit is contained in:
rUv
2026-08-22 18:06:16 -04:00
committed by GitHub
parent bd110e0eac
commit 0df48df7b2
17 changed files with 1316 additions and 0 deletions

View File

@@ -0,0 +1,108 @@
import { decodeLiDARPacket, depthToPointCloud } from './codec.mjs';
const canvas = document.querySelector('#view');
const ctx = canvas.getContext('2d');
const status = document.querySelector('#status');
const fpsEl = document.querySelector('#fps');
const pointsEl = document.querySelector('#points');
const seqEl = document.querySelector('#seq');
const latencyEl = document.querySelector('#latency');
const sensorEl = document.querySelector('#sensor');
let lastFrameAt = performance.now();
let yaw = 0.3;
let pitch = -0.15;
let scale = 120;
function connect() {
const token = new URLSearchParams(location.search).get('token');
if (!token) {
status.textContent = 'TOKEN REQUIRED';
status.dataset.state = 'warn';
return;
}
const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:';
const socket = new WebSocket(`${protocol}//${location.host}/ws/lidar?token=${encodeURIComponent(token)}`);
socket.addEventListener('open', () => {
status.textContent = 'LIVE';
status.dataset.state = 'live';
});
socket.addEventListener('close', () => {
status.textContent = 'RECONNECTING';
status.dataset.state = 'warn';
setTimeout(connect, 1000);
});
socket.addEventListener('message', (event) => {
try {
const raw = JSON.parse(event.data);
const frame = decodeLiDARPacket(raw);
const points = depthToPointCloud(frame, 1);
render(points);
const now = performance.now();
const delta = now - lastFrameAt;
lastFrameAt = now;
fpsEl.textContent = delta > 0 ? (1000 / delta).toFixed(1) : '0.0';
pointsEl.textContent = points.length.toLocaleString();
seqEl.textContent = frame.provenance.sequence;
latencyEl.textContent = Math.max(0, Date.now() - Number(frame.provenance.timestampNs / 1_000_000)).toFixed(0);
sensorEl.textContent = frame.provenance.sensor;
} catch (error) {
console.error(error);
status.textContent = 'FRAME ERROR';
status.dataset.state = 'warn';
}
});
}
function resize() {
const dpr = Math.min(devicePixelRatio || 1, 2);
const rect = canvas.getBoundingClientRect();
canvas.width = Math.max(1, Math.floor(rect.width * dpr));
canvas.height = Math.max(1, Math.floor(rect.height * dpr));
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
}
function render(points) {
resize();
const w = canvas.clientWidth;
const h = canvas.clientHeight;
ctx.clearRect(0, 0, w, h);
ctx.fillStyle = '#091018';
ctx.fillRect(0, 0, w, h);
const cy = Math.cos(yaw);
const sy = Math.sin(yaw);
const cp = Math.cos(pitch);
const sp = Math.sin(pitch);
ctx.fillStyle = '#58e0d2';
for (let i = 0; i < points.length; i += 1) {
let [x, y, z] = points[i];
const rx = x * cy - z * sy;
const rz = x * sy + z * cy;
const ry = y * cp - rz * sp;
const rz2 = y * sp + rz * cp;
const perspective = 1 / Math.max(0.35, 1.8 - rz2 * 0.12);
const px = w / 2 + rx * scale * perspective;
const py = h / 2 + ry * scale * perspective;
if (px >= 0 && px < w && py >= 0 && py < h) ctx.fillRect(px, py, 1.4, 1.4);
}
}
canvas.addEventListener('pointermove', (event) => {
if (!event.buttons) return;
yaw += event.movementX * 0.006;
pitch += event.movementY * 0.006;
});
canvas.addEventListener('wheel', (event) => {
event.preventDefault();
scale = Math.max(40, Math.min(400, scale - event.deltaY * 0.2));
}, { passive: false });
connect();

View File

@@ -0,0 +1,121 @@
export function decodeLiDARPacket(packet) {
if (!packet || packet.type !== 'ruview.lidar.depth.v1') {
throw new Error('Unsupported LiDAR packet type');
}
const { depth } = packet;
if (!depth || depth.encoding !== 'u16le-mm+u8-confidence') {
throw new Error('Unsupported depth encoding');
}
assertPositiveInteger(depth.width, 'depth.width');
assertPositiveInteger(depth.height, 'depth.height');
assertIntrinsics(packet.intrinsics);
if (!packet.pose || !Array.isArray(packet.pose.matrix) || packet.pose.matrix.length !== 16
|| packet.pose.matrix.some((value) => !Number.isFinite(value))) {
throw new Error('Invalid camera pose');
}
const mmBytes = base64ToBytes(depth.millimetersBase64, 'millimetersBase64');
const confidence = base64ToBytes(depth.confidenceBase64, 'confidenceBase64');
const expectedPixels = depth.width * depth.height;
if (!Number.isSafeInteger(expectedPixels) || expectedPixels > 1_000_000) {
throw new Error('Depth dimensions exceed the supported pixel limit');
}
if (mmBytes.byteLength !== expectedPixels * 2) {
throw new Error(`Depth payload length mismatch: expected ${expectedPixels * 2}, got ${mmBytes.byteLength}`);
}
if (confidence.byteLength !== expectedPixels) {
throw new Error(`Confidence payload length mismatch: expected ${expectedPixels}, got ${confidence.byteLength}`);
}
const view = new DataView(mmBytes.buffer, mmBytes.byteOffset, mmBytes.byteLength);
const meters = new Float32Array(expectedPixels);
for (let i = 0; i < expectedPixels; i += 1) {
meters[i] = view.getUint16(i * 2, true) / 1000;
}
return {
...packet,
depth: {
width: depth.width,
height: depth.height,
meters,
confidence,
},
};
}
export function depthToPointCloud(frame, confidenceThreshold = 1) {
const { width, height, meters, confidence } = frame.depth;
const { fx, fy, cx, cy, imageWidth, imageHeight } = frame.intrinsics;
assertPositiveInteger(width, 'depth.width');
assertPositiveInteger(height, 'depth.height');
assertIntrinsics(frame.intrinsics);
if (meters.length !== width * height || confidence.length !== width * height) {
throw new Error('Decoded depth array length mismatch');
}
if (!Number.isFinite(confidenceThreshold) || confidenceThreshold < 0 || confidenceThreshold > 255) {
throw new Error('Invalid confidence threshold');
}
const sx = width / imageWidth;
const sy = height / imageHeight;
const scaledFx = fx * sx;
const scaledFy = fy * sy;
const scaledCx = cx * sx;
const scaledCy = cy * sy;
const points = [];
for (let v = 0; v < height; v += 1) {
for (let u = 0; u < width; u += 1) {
const index = v * width + u;
const z = meters[index];
if (!Number.isFinite(z) || z <= 0 || confidence[index] < confidenceThreshold) continue;
const x = ((u - scaledCx) / scaledFx) * z;
const y = ((v - scaledCy) / scaledFy) * z;
points.push([x, y === 0 ? 0 : -y, -z]);
}
}
return points;
}
function assertPositiveInteger(value, name) {
if (!Number.isSafeInteger(value) || value <= 0) {
throw new Error(`${name} must be a positive integer`);
}
}
function assertIntrinsics(intrinsics) {
if (!intrinsics
|| !Number.isFinite(intrinsics.fx) || intrinsics.fx <= 0
|| !Number.isFinite(intrinsics.fy) || intrinsics.fy <= 0
|| !Number.isFinite(intrinsics.cx)
|| !Number.isFinite(intrinsics.cy)) {
throw new Error('Invalid camera intrinsics');
}
assertPositiveInteger(intrinsics.imageWidth, 'intrinsics.imageWidth');
assertPositiveInteger(intrinsics.imageHeight, 'intrinsics.imageHeight');
}
function base64ToBytes(value, name) {
if (typeof value !== 'string' || value.length === 0 || value.length % 4 !== 0
|| !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) {
throw new Error(`${name} must be canonical base64`);
}
if (typeof Buffer !== 'undefined') {
return Uint8Array.from(Buffer.from(value, 'base64'));
}
const binary = atob(value);
const bytes = new Uint8Array(binary.length);
for (let i = 0; i < binary.length; i += 1) {
bytes[i] = binary.charCodeAt(i);
}
return bytes;
}

View File

@@ -0,0 +1,69 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { decodeLiDARPacket, depthToPointCloud } from './codec.mjs';
function b64(bytes) {
return Buffer.from(bytes).toString('base64');
}
test('decodes u16 millimeter depth and confidence', () => {
const packet = {
type: 'ruview.lidar.depth.v1',
intrinsics: { fx: 100, fy: 100, cx: 1, cy: 1, imageWidth: 2, imageHeight: 2 },
pose: { matrix: Array(16).fill(0) },
depth: {
width: 2,
height: 2,
encoding: 'u16le-mm+u8-confidence',
millimetersBase64: b64([0xe8,0x03,0xd0,0x07,0xb8,0x0b,0xa0,0x0f]),
confidenceBase64: b64([2,2,1,0]),
},
provenance: { sensor: 'test', source: 'live', privacyClass: 'geometry-only', sequence: 1, timestampNs: 1, schema: 'ruview.lidar.depth.v1' },
};
const frame = decodeLiDARPacket(packet);
assert.deepEqual(Array.from(frame.depth.meters), [1,2,3,4]);
assert.deepEqual(Array.from(frame.depth.confidence), [2,2,1,0]);
});
test('rejects malformed payload length', () => {
assert.throws(() => decodeLiDARPacket({
type: 'ruview.lidar.depth.v1',
intrinsics: { fx: 100, fy: 100, cx: 1, cy: 1, imageWidth: 2, imageHeight: 2 },
pose: { matrix: Array(16).fill(0) },
depth: { width: 2, height: 2, encoding: 'u16le-mm+u8-confidence', millimetersBase64: b64([1,2]), confidenceBase64: b64([1,1,1,1]) },
}), /length mismatch/);
});
test('rejects invalid dimensions, intrinsics, pose, and base64', () => {
const valid = {
type: 'ruview.lidar.depth.v1',
intrinsics: { fx: 100, fy: 100, cx: 0, cy: 0, imageWidth: 1, imageHeight: 1 },
pose: { matrix: Array(16).fill(0) },
depth: {
width: 1,
height: 1,
encoding: 'u16le-mm+u8-confidence',
millimetersBase64: b64([0xe8, 0x03]),
confidenceBase64: b64([2]),
},
};
assert.throws(() => decodeLiDARPacket({ ...valid, depth: { ...valid.depth, width: 0 } }), /positive integer/);
assert.throws(() => decodeLiDARPacket({ ...valid, intrinsics: { ...valid.intrinsics, fx: 0 } }), /intrinsics/);
assert.throws(() => decodeLiDARPacket({ ...valid, pose: { matrix: [1] } }), /pose/);
assert.throws(() => decodeLiDARPacket({
...valid,
depth: { ...valid.depth, millimetersBase64: '!!!!' },
}), /canonical base64/);
});
test('projects depth into a point cloud and honors confidence', () => {
const frame = {
intrinsics: { fx: 100, fy: 100, cx: 0, cy: 0, imageWidth: 2, imageHeight: 2 },
depth: { width: 2, height: 2, meters: Float32Array.from([1,1,1,1]), confidence: Uint8Array.from([2,0,2,0]) },
};
const points = depthToPointCloud(frame, 1);
assert.equal(points.length, 2);
assert.deepEqual(points[0], [0, 0, -1]);
});

View File

@@ -0,0 +1,36 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width,initial-scale=1,viewport-fit=cover" />
<title>RuView iPhone LiDAR</title>
<link rel="stylesheet" href="./styles.css" />
</head>
<body>
<main>
<header>
<div>
<p class="eyebrow">RUVIEW SENSOR BRIDGE</p>
<h1>iPhone LiDAR</h1>
</div>
<span id="status">CONNECTING</span>
</header>
<section class="metrics">
<div><strong id="fps">0.0</strong><span>FPS</span></div>
<div><strong id="points">0</strong><span>POINTS</span></div>
<div><strong id="seq">0</strong><span>SEQ</span></div>
<div><strong id="latency">0</strong><span>MS</span></div>
</section>
<canvas id="view"></canvas>
<footer>
<span>Geometry only</span>
<span>No RGB upload</span>
<span id="sensor">Waiting for sensor</span>
</footer>
</main>
<script type="module" src="./app.mjs"></script>
</body>
</html>

View File

@@ -0,0 +1,39 @@
{
"name": "@ruview/iphone-lidar-web",
"version": "0.1.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@ruview/iphone-lidar-web",
"version": "0.1.0",
"dependencies": {
"ws": "^8.18.3"
},
"engines": {
"node": ">=20"
}
},
"node_modules/ws": {
"version": "8.21.3",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz",
"integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==",
"license": "MIT",
"engines": {
"node": ">=10.0.0"
},
"peerDependencies": {
"bufferutil": "^4.0.1",
"utf-8-validate": ">=5.0.2"
},
"peerDependenciesMeta": {
"bufferutil": {
"optional": true
},
"utf-8-validate": {
"optional": true
}
}
}
}
}

View File

@@ -0,0 +1,16 @@
{
"name": "@ruview/iphone-lidar-web",
"private": true,
"version": "0.1.0",
"type": "module",
"scripts": {
"start": "node relay.mjs",
"test": "node --test"
},
"engines": {
"node": ">=20"
},
"dependencies": {
"ws": "^8.18.3"
}
}

View File

@@ -0,0 +1,166 @@
import { randomBytes, timingSafeEqual } from 'node:crypto';
import http from 'node:http';
import { readFile } from 'node:fs/promises';
import { extname, join } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { WebSocket, WebSocketServer } from 'ws';
const root = fileURLToPath(new URL('.', import.meta.url));
const staticFiles = new Set(['index.html', 'app.mjs', 'codec.mjs', 'styles.css']);
const maxPayloadBytes = 2_000_000;
function constantTimeEqual(left, right) {
const leftBytes = Buffer.from(left, 'utf8');
const rightBytes = Buffer.from(right, 'utf8');
return leftBytes.length === rightBytes.length && timingSafeEqual(leftBytes, rightBytes);
}
function rejectUpgrade(socket, status, message) {
const body = `${message}\n`;
socket.end(
`HTTP/1.1 ${status}\r\nConnection: close\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: ${Buffer.byteLength(body)}\r\n\r\n${body}`,
);
}
export function createLiDARRelay({ token, rootDirectory = root } = {}) {
const accessToken = token || randomBytes(24).toString('hex');
const clients = new Set();
const server = http.createServer(async (req, res) => {
if (req.method !== 'GET' && req.method !== 'HEAD') {
res.writeHead(405, { allow: 'GET, HEAD' }).end();
return;
}
let pathname;
try {
pathname = decodeURIComponent(new URL(req.url || '/', 'http://localhost').pathname);
} catch {
res.writeHead(400).end('bad path');
return;
}
const filename = pathname === '/' ? 'index.html' : pathname.slice(1);
if (!staticFiles.has(filename)) {
res.writeHead(404).end('not found');
return;
}
try {
const data = await readFile(join(rootDirectory, filename));
const contentType = {
'.html': 'text/html; charset=utf-8',
'.mjs': 'text/javascript; charset=utf-8',
'.css': 'text/css; charset=utf-8',
}[extname(filename)] || 'application/octet-stream';
res.writeHead(200, {
'content-type': contentType,
'cache-control': 'no-store',
'content-security-policy': "default-src 'self'; connect-src 'self' ws: wss:; img-src 'self'; style-src 'self'; base-uri 'none'; frame-ancestors 'none'",
'referrer-policy': 'no-referrer',
'x-content-type-options': 'nosniff',
});
if (req.method === 'HEAD') res.end();
else res.end(data);
} catch {
res.writeHead(404).end('not found');
}
});
const wss = new WebSocketServer({ noServer: true, maxPayload: maxPayloadBytes });
server.on('upgrade', (req, socket, head) => {
let url;
try {
url = new URL(req.url || '/', 'http://localhost');
} catch {
rejectUpgrade(socket, '400 Bad Request', 'bad websocket URL');
return;
}
if (url.pathname !== '/ws/lidar') {
rejectUpgrade(socket, '404 Not Found', 'not found');
return;
}
const suppliedToken = url.searchParams.get('token') || '';
if (!constantTimeEqual(suppliedToken, accessToken)) {
rejectUpgrade(socket, '401 Unauthorized', 'valid LiDAR relay token required');
return;
}
wss.handleUpgrade(req, socket, head, (websocket) => {
wss.emit('connection', websocket, req);
});
});
wss.on('connection', (socket) => {
clients.add(socket);
socket.on('close', () => clients.delete(socket));
socket.on('error', () => socket.terminate());
socket.on('message', (data, isBinary) => {
if (isBinary) return;
let packet;
try {
packet = JSON.parse(data.toString());
} catch {
return;
}
if (packet?.type !== 'ruview.lidar.depth.v1') return;
for (const peer of clients) {
if (peer !== socket && peer.readyState === WebSocket.OPEN) {
peer.send(data.toString());
}
}
});
});
return {
accessToken,
server,
async listen(port = 0, host = '127.0.0.1') {
await new Promise((resolve, reject) => {
server.once('error', reject);
server.listen(port, host, () => {
server.off('error', reject);
resolve();
});
});
return server.address();
},
async close() {
for (const peer of clients) peer.terminate();
await new Promise((resolve) => wss.close(resolve));
if (server.listening) {
await new Promise((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
});
}
},
};
}
function configuredPort(value) {
const parsed = Number(value);
if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > 65_535) {
throw new Error(`PORT must be an integer from 1 to 65535; received ${value}`);
}
return parsed;
}
const directInvocation = process.argv[1]
&& pathToFileURL(process.argv[1]).href === import.meta.url;
if (directInvocation) {
const port = configuredPort(process.env.PORT || '8787');
const host = process.env.HOST || '0.0.0.0';
const relay = createLiDARRelay({ token: process.env.RUVIEW_LIDAR_TOKEN });
await relay.listen(port, host);
const encodedToken = encodeURIComponent(relay.accessToken);
console.log(`RuView iPhone LiDAR relay listening on ${host}:${port}`);
console.log(`Browser: http://<host>:${port}/?token=${encodedToken}`);
console.log(`Native endpoint: ws://<host>:${port}/ws/lidar?token=${encodedToken}`);
}

View File

@@ -0,0 +1,73 @@
import assert from 'node:assert/strict';
import http from 'node:http';
import test from 'node:test';
import { WebSocket } from 'ws';
import { createLiDARRelay } from './relay.mjs';
function connect(url) {
return new Promise((resolve, reject) => {
const socket = new WebSocket(url);
socket.once('open', () => resolve(socket));
socket.once('error', reject);
});
}
function request(url) {
return new Promise((resolve, reject) => {
http.get(url, (response) => {
response.resume();
response.once('end', () => resolve(response));
}).once('error', reject);
});
}
test('relay requires a token, limits static files, and forwards LiDAR frames', async (t) => {
const token = 'test-token-for-relay';
const relay = createLiDARRelay({ token });
const address = await relay.listen(0, '127.0.0.1');
const httpBase = `http://127.0.0.1:${address.port}`;
const wsBase = `ws://127.0.0.1:${address.port}/ws/lidar`;
const sockets = [];
t.after(async () => {
for (const socket of sockets) socket.terminate();
await relay.close();
});
const indexResponse = await request(`${httpBase}/?token=${token}`);
assert.equal(indexResponse.statusCode, 200);
assert.match(indexResponse.headers['content-security-policy'], /frame-ancestors 'none'/);
const sourceResponse = await request(`${httpBase}/relay.mjs`);
assert.equal(sourceResponse.statusCode, 404);
await new Promise((resolve, reject) => {
const unauthorized = new WebSocket(wsBase);
unauthorized.once('unexpected-response', (_request, response) => {
assert.equal(response.statusCode, 401);
response.resume();
resolve();
});
unauthorized.once('open', () => reject(new Error('unauthorized websocket opened')));
unauthorized.once('error', () => {});
});
const sender = await connect(`${wsBase}?token=${encodeURIComponent(token)}`);
const receiver = await connect(`${wsBase}?token=${encodeURIComponent(token)}`);
sockets.push(sender, receiver);
const received = new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('timed out waiting for relayed frame')), 2_000);
receiver.once('message', (data, isBinary) => {
clearTimeout(timer);
resolve({ data, isBinary });
});
});
const frame = { type: 'ruview.lidar.depth.v1', provenance: { sequence: 7 } };
sender.send(JSON.stringify(frame));
const message = await received;
assert.equal(message.isBinary, false);
assert.deepEqual(JSON.parse(message.data.toString()), frame);
});

View File

@@ -0,0 +1 @@
*{box-sizing:border-box}body{margin:0;background:#05080d;color:#e8f1f5;font-family:ui-monospace,SFMono-Regular,Menlo,monospace}main{min-height:100vh;padding:18px;display:grid;grid-template-rows:auto auto 1fr auto;gap:14px}header{display:flex;align-items:end;justify-content:space-between}h1{margin:0;font-size:clamp(28px,6vw,54px)}.eyebrow{margin:0 0 6px;color:#58e0d2;font-size:11px;letter-spacing:.16em}#status{border:1px solid #2a3946;border-radius:999px;padding:7px 11px;font-size:11px}#status[data-state=live]{color:#58e0d2;border-color:#58e0d2}.metrics{display:grid;grid-template-columns:repeat(4,1fr);gap:8px}.metrics div{background:#0b1219;border:1px solid #17222d;border-radius:10px;padding:10px}.metrics strong{display:block;font-size:18px}.metrics span,footer{font-size:10px;color:#8aa0af}canvas{width:100%;height:100%;min-height:55vh;border-radius:14px;border:1px solid #17222d;background:#091018;touch-action:none}footer{display:flex;gap:16px;flex-wrap:wrap}@media(max-width:640px){.metrics{grid-template-columns:repeat(2,1fr)}canvas{min-height:58vh}}