mirror of
https://github.com/ruvnet/RuView.git
synced 2026-09-02 05:25:52 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0df48df7b2 | ||
|
|
bd110e0eac | ||
|
|
f3c361efd1 | ||
|
|
a3b6e1d500 |
70
.github/workflows/iphone-lidar.yml
vendored
Normal file
70
.github/workflows/iphone-lidar.yml
vendored
Normal file
@@ -0,0 +1,70 @@
|
||||
name: iPhone LiDAR integration
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'integrations/iphone-lidar/**'
|
||||
- 'docs/adr/ADR-340-iphone-lidar-sensor-bridge.md'
|
||||
- '.github/workflows/iphone-lidar.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'integrations/iphone-lidar/**'
|
||||
- 'docs/adr/ADR-340-iphone-lidar-sensor-bridge.md'
|
||||
- '.github/workflows/iphone-lidar.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
web:
|
||||
name: Node relay and codec
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: integrations/iphone-lidar/web
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: integrations/iphone-lidar/web/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci --ignore-scripts
|
||||
|
||||
- name: Run tests
|
||||
run: npm test
|
||||
|
||||
- name: Audit runtime dependencies
|
||||
run: npm audit --omit=optional --audit-level=high
|
||||
|
||||
ios:
|
||||
name: iOS 17 compile
|
||||
runs-on: macos-15
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
|
||||
|
||||
- name: Compile native sources with strict concurrency
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sdk="$(xcrun --sdk iphoneos --show-sdk-path)"
|
||||
build_dir="$RUNNER_TEMP/ruview-lidar-build"
|
||||
mkdir -p "$build_dir"
|
||||
cd "$build_dir"
|
||||
xcrun swiftc \
|
||||
-parse-as-library \
|
||||
-target arm64-apple-ios17.0 \
|
||||
-sdk "$sdk" \
|
||||
-module-name RuViewLiDAR \
|
||||
-strict-concurrency=complete \
|
||||
-warnings-as-errors \
|
||||
-emit-module \
|
||||
-emit-module-path "$build_dir/RuViewLiDAR.swiftmodule" \
|
||||
-c "$GITHUB_WORKSPACE"/integrations/iphone-lidar/native/RuViewLiDAR/*.swift
|
||||
27
.github/workflows/security-scan.yml
vendored
27
.github/workflows/security-scan.yml
vendored
@@ -14,6 +14,33 @@ env:
|
||||
PYTHON_VERSION: '3.11'
|
||||
|
||||
jobs:
|
||||
# Rust dependency advisories are deterministic for the checked-in lockfile,
|
||||
# so this job gates the PR and retains the exact machine-readable report.
|
||||
rust-audit:
|
||||
name: Rust Dependency Audit
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
|
||||
- name: Install cargo-audit
|
||||
run: cargo install cargo-audit --locked --version 0.22.2
|
||||
|
||||
- name: Audit the checked-in Rust lockfile
|
||||
run: |
|
||||
set -o pipefail
|
||||
cargo audit --file v2/Cargo.lock --json | tee v2/cargo-audit.json
|
||||
|
||||
- name: Upload Rust advisory report
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
if: always()
|
||||
with:
|
||||
name: cargo-audit-report
|
||||
path: v2/cargo-audit.json
|
||||
if-no-files-found: error
|
||||
|
||||
# Static Application Security Testing (SAST)
|
||||
sast:
|
||||
name: Static Application Security Testing
|
||||
|
||||
1
.github/workflows/sensing-server-docker.yml
vendored
1
.github/workflows/sensing-server-docker.yml
vendored
@@ -28,6 +28,7 @@ on:
|
||||
- 'v2/crates/wifi-densepose-wifiscan/**'
|
||||
- 'v2/crates/wifi-densepose-bfld/**'
|
||||
- 'v2/crates/cog-ha-matter/**'
|
||||
- 'v2/crates/homecore*/**'
|
||||
- 'v2/Cargo.toml'
|
||||
- 'v2/Cargo.lock'
|
||||
- 'ui/**'
|
||||
|
||||
16
README.md
16
README.md
@@ -245,8 +245,8 @@ See the measured benchmarks, witness records, and one-command reproducibility ch
|
||||
|------|-------|---------|
|
||||
| **MM-Fi pose model (SOTA)** | [`ruvnet/wifi-densepose-mmfi-pose`](https://huggingface.co/ruvnet/wifi-densepose-mmfi-pose) | 82.69% torso-PCK@20 (single) · 83.59% (ensemble+TTA) · 75K-param micro variant 74.30% |
|
||||
| **AetherArena benchmark Space** | [`ruvnet/aether-arena`](https://huggingface.co/spaces/ruvnet/aether-arena) | self-correcting, auditable MM-Fi leaderboard |
|
||||
| **Full MM-Fi study (honest picture)** | [`docs/benchmarks/mmfi-wifi-sensing-study.md`](docs/benchmarks/mmfi-wifi-sensing-study.md) | pose + action; zero-shot cross-subject ~64%, +~30 s in-room calibration → 72.2% |
|
||||
| **Efficiency frontier** | [`docs/benchmarks/wifi-pose-efficiency-frontier.md`](docs/benchmarks/wifi-pose-efficiency-frontier.md) | SOTA-beating WiFi pose in a 20 KB int4 edge model |
|
||||
| **Full MM-Fi study (honest picture)** | [`docs/benchmarks/mmfi-wifi-sensing-study.md`](docs/benchmarks/mmfi-wifi-sensing-study.md) | pose + action; zero-shot cross-subject ~64%, labeled in-room calibration → 72.2% |
|
||||
| **Efficiency frontier** | [`docs/benchmarks/wifi-pose-efficiency-frontier.md`](docs/benchmarks/wifi-pose-efficiency-frontier.md) | SOTA-beating MM-Fi pose in a ~37 KB int4 model; live ESP32 compatibility not established |
|
||||
| **Pretrained encoder** | [`ruvnet/wifi-densepose-pretrained`](https://huggingface.co/ruvnet/wifi-densepose-pretrained) | 82.3% held-out temporal-triplet, 8 KB int4 |
|
||||
| **Reproducible proof (Trust Kill Switch)** | [`archive/v1/data/proof/verify.py`](archive/v1/data/proof/verify.py) + [`expected_features.sha256`](archive/v1/data/proof/expected_features.sha256) | one-command deterministic pipeline replay (SHA-256 of output vs published hash) |
|
||||
| **Benchmark-proof ADR** | [ADR-168](docs/adr/ADR-168-benchmark-proof.md) | how the numbers are produced and verified |
|
||||
@@ -488,12 +488,20 @@ Neural Network: processed signals → 17 body keypoints + vital signs + room mod
|
||||
Output: real-time pose, breathing, heart rate, room fingerprint, drift alerts
|
||||
```
|
||||
|
||||
No training cameras required — the [Self-Learning system (ADR-024)](docs/adr/ADR-024-contrastive-csi-embedding-model.md) bootstraps from raw WiFi data alone. [MERIDIAN (ADR-027)](docs/adr/ADR-027-cross-environment-domain-generalization.md) ensures the model works in any room, not just the one it trained in.
|
||||
The [Self-Learning system (ADR-024)](docs/adr/ADR-024-contrastive-csi-embedding-model.md) provides
|
||||
camera-free representation-learning components. Cross-room pose remains a separate, data-gated
|
||||
problem: [MERIDIAN (ADR-027)](docs/adr/ADR-027-cross-environment-domain-generalization.md) is
|
||||
**Proposed**, while the measured calibration reference requires labeled CSI/keypoint pairs and
|
||||
model-specific adapters. See the [model compatibility boundary](docs/user-guide.md#model-and-capture-compatibility).
|
||||
|
||||
---
|
||||
|
||||
## 🏢 Use Cases & Applications
|
||||
|
||||
> **Safety boundary:** these are research and prototype applications, not medical devices,
|
||||
> emergency systems, or safety-certified controls. Vital-sign and pose outputs require independent
|
||||
> validation on the exact hardware, room, subjects, and failure conditions before operational use.
|
||||
|
||||
WiFi sensing works anywhere WiFi exists. No new hardware in most cases — just software on existing access points or a $8 ESP32 add-on. Because there are no cameras, deployments avoid privacy regulations (GDPR video, HIPAA imaging) by design.
|
||||
|
||||
**Scaling:** Each AP distinguishes ~3-5 people (56 subcarriers). Multi-AP multiplies linearly — a 4-AP retail mesh covers ~15-20 occupants. No hard software limit; the practical ceiling is signal physics.
|
||||
@@ -528,7 +536,7 @@ WiFi sensing works anywhere WiFi exists. No new hardware in most cases — just
|
||||
| Use Case | What It Does | Hardware | Key Metric | Edge Module |
|
||||
|----------|-------------|----------|------------|-------------|
|
||||
| **Smart home automation** | Room-level presence triggers (lights, HVAC, music) that work through walls — no dead zones, no motion-sensor timeouts | 2-3 ESP32-S3 nodes ($24) | Through-wall range ~5m | [HVAC Presence](docs/edge-modules/building.md), [Lighting Zones](docs/edge-modules/building.md) |
|
||||
| **Fitness & sports** | Rep counting, posture correction, breathing cadence during exercise — no wearable, no camera in locker rooms | 3+ ESP32-S3 mesh | Pose: 17 keypoints | [Breathing Sync](docs/edge-modules/exotic.md), [Gait Analysis](docs/edge-modules/medical.md) |
|
||||
| **Fitness & sports research** | Explore motion and breathing cadence without a wearable or camera; reliable posture correction requires a validated compatible pose model | 3+ ESP32-S3 mesh + edge host | Prototype; no live S3 pose accuracy claim | [Breathing Sync](docs/edge-modules/exotic.md), [Gait Analysis](docs/edge-modules/medical.md) |
|
||||
| **Childcare & schools** | Naptime breathing monitoring, playground headcount, restricted-area alerts — privacy-safe for minors | 2-4 ESP32-S3 per zone | Breathing: ±1 BPM | [Sleep Apnea](docs/edge-modules/medical.md), [Perimeter Breach](docs/edge-modules/security.md) |
|
||||
| **Event venues & concerts** | Crowd density mapping, crush-risk detection via breathing compression, emergency evacuation flow tracking | Multi-AP mesh (4-8 APs) | Density per m² | [Customer Flow](docs/edge-modules/retail.md), [Panic Motion](docs/edge-modules/security.md) |
|
||||
| **Stadiums & arenas** | Section-level occupancy for dynamic pricing, concession staffing, emergency egress flow modeling | Enterprise AP grid | 15-20 per AP mesh | [Dwell Heatmap](docs/edge-modules/retail.md), [Queue Length](docs/edge-modules/retail.md) |
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
# RuView Calibration Service (reference implementation)
|
||||
|
||||
Turn a **shared WiFi-CSI pose base model** into a room-specific one with a **30-second labeled
|
||||
calibration** and a **~11 KB per-room LoRA adapter**. This is the deployable resolution of the
|
||||
cross-subject / cross-environment generalization problem (full study: [ADR-150 §3.3–3.6](../../docs/adr/ADR-150-rf-foundation-encoder.md)).
|
||||
Fit a room-specific **~11 KB LoRA adapter** for a shared WiFi-CSI pose base from a short **labeled
|
||||
capture**. This is a measured MM-Fi reference path for cross-subject / cross-environment adaptation
|
||||
(full study: [ADR-150 §3.3–3.6](../../docs/adr/ADR-150-rf-foundation-encoder.md)); it is not proof of
|
||||
plug-and-play adaptation from a live ESP32 stream.
|
||||
|
||||
> **Not the proposed MERIDIAN fast path.** Both producers below require paired CSI and keypoint
|
||||
> labels, and their tensor shapes and adapter files are model-specific. ADR-027's automatic,
|
||||
> unlabeled 10-second MERIDIAN calibration remains **Proposed** and is not implemented as an
|
||||
> end-to-end deployment command.
|
||||
|
||||
## Why
|
||||
|
||||
@@ -66,8 +72,8 @@ Adapters are **model-specific**. There are two calibration producers here:
|
||||
| `cog_calibrate.py` | cog **conv+MLP** (`pose_v1.safetensors`, 56×20) | `[N,56,20]` | `.safetensors` (`fc1.a`/`fc1.b`/`fc2.a`/`fc2.b`) | Rust `cog-pose-estimation run --adapter` |
|
||||
|
||||
```bash
|
||||
# Produce a cog-format per-room adapter for the deployed Rust pose engine:
|
||||
python cog_calibrate.py --base pose_v1.safetensors --data calib.npz --out room.safetensors
|
||||
# Produce a cog-format per-room adapter from X:[N,56,20], Y:[N,17,2]:
|
||||
python cog_calibrate.py --base pose_v1.safetensors --data cog-calib.npz --out room.safetensors
|
||||
# then in the cog runtime:
|
||||
cog-pose-estimation run --config <cfg> --adapter room.safetensors
|
||||
```
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# ADR-325: Cognitum Spaces activation and governed spatial exchange
|
||||
|
||||
- **Status**: Accepted — legacy read live; versioned hierarchy, local memory, and governed-action implementation validated on feature branches; deployment pending
|
||||
- **Status**: Accepted — legacy and versioned reads, OAuth activation, local spatial memory, governed-action policy, metaharness support, and npm distribution are implemented; HTTPS production evidence is complete
|
||||
- **Date**: 2026-08-17
|
||||
- **Deciders**: ruv
|
||||
- **Tags**: cognitum-spaces, oauth, spatial-state, privacy, ruvector, policy, autogenous
|
||||
@@ -423,41 +423,56 @@ Identity metadata deliberately advertises `spaces:read` for RuView but not
|
||||
publisher surface. RuView therefore has no OAuth write, command, policy-approval,
|
||||
or actuator capability.
|
||||
|
||||
This production evidence does not claim deployment of sites/buildings/floors/
|
||||
zones, entities, semantic event or alert resources, tenant-scoped RuVector
|
||||
spatial history, MQTT reconciliation, governed actions, commands, or actuators.
|
||||
The first three are implemented and locally validated in the 2026-08-19 feature
|
||||
branches described below, but remain non-production until their workflow and
|
||||
readback gates pass. Commands and actuators remain out of scope.
|
||||
That receipt was for the initial flat Space slice. The following production
|
||||
expansion supersedes only its hierarchy/event/alert deferral. MQTT, commands,
|
||||
actuators, real-hardware accuracy, and the long-duration operational trial
|
||||
remain outside the completed claim.
|
||||
|
||||
## Feature-branch implementation evidence (2026-08-19)
|
||||
## Completed implementation and production expansion (2026-08-19)
|
||||
|
||||
- Cognitum API ADR-101 implements all eight `/v1/spatial` collections, coherent
|
||||
transactional hierarchy checks, stable pagination, event/alert retention,
|
||||
strict P2/P3 admission, API-key-only writes, and OAuth/API-key reads. The
|
||||
Firestore emulator passed both the legacy and versioned verification scripts,
|
||||
including cross-resource message replay denial.
|
||||
- `ruview-cognitum-spaces` adds a strictly decoded, paged, read-only client for
|
||||
every versioned collection. `wifi-densepose spaces --resource ...` exposes it
|
||||
while preserving the legacy flat command. The focused client suite and the
|
||||
CLI no-default-features test gate pass.
|
||||
- The contributor metaharness now accepts `resource`, `limit`, and opaque
|
||||
`cursor` on `ruview_spaces_list`, fixes the API origin, strips API-key
|
||||
compatibility authority, and revalidates hierarchy/event/alert contracts.
|
||||
Full metaharness, security, brain, flywheel, manifest, audit, and pack gates
|
||||
pass for the `@ruvnet/ruview` 0.5.0 release candidate.
|
||||
- ADR-326 adds `ruview-spatial-memory`: one RuVector HNSW index per authenticated
|
||||
tenant/workspace, replay/derivation/retention gates, cascading erasure,
|
||||
bounded explanations, and XChaCha20-Poly1305 snapshots with reload-verified
|
||||
key rotation. Its focused crate suite passes with `SYNTHETIC` evidence.
|
||||
- ADR-327 extends `ruview-policy` with typed observe/recommend/execute intents,
|
||||
exact host grants, signed approvals, nonce/idempotency defense, the existing
|
||||
ADR-321 assurance matrix, and signed hash-chained receipts. `spaces:read` is
|
||||
explicitly denied as execution authority. Its focused crate suite passes.
|
||||
- The required whole-workspace Rust command was attempted twice on Windows:
|
||||
parallel compilation ended in a compiler-process stack-buffer-overrun and a
|
||||
single-job retry reached the configured timeout without a source/test
|
||||
diagnostic. This is not recorded as a green gate; Linux CI remains required.
|
||||
- Cognitum API PRs #211 and #212 shipped the eight `/v1/spatial` collections,
|
||||
transactional hierarchy integrity, stable pagination, event/alert retention,
|
||||
strict P2/P3 admission, API-key-only writes, OAuth/API-key reads, and the
|
||||
additive-only Firestore release authority. Function run `32279092861`
|
||||
promoted active Node 22 revision `spacesapi-00005-kaf`.
|
||||
- Edge PRs #214, #215, and #216 preserved canonical UUID routing, kept SQLi
|
||||
denial, and removed secret-valued API-key rate selection. Gateway run
|
||||
`32284410107` promoted the reviewed immutable digest to 100% production
|
||||
traffic. Every versioned collection returned HTTP 200 through the public
|
||||
edge; the hierarchy composite index is `READY` and both retention TTL fields
|
||||
are `ACTIVE`.
|
||||
- The dedicated RuView service credential was rotated to exactly
|
||||
`spaces:read` and `spaces:write`; its predecessor returns 401. A non-mutating
|
||||
invalid-body probe reached write validation without persisting customer data.
|
||||
Other potentially affected owner keys and residual log retention remain
|
||||
tracked in Cognitum API #217.
|
||||
- A live RuView Authorization Code + S256 PKCE consent requested exactly
|
||||
`sensing:read spaces:read`. Its in-memory token read versioned `sites` with
|
||||
HTTP 200 and schema `1.0`; the verifier then revoked the temporary refresh
|
||||
credential and persisted no token.
|
||||
- RuView PR #1650 merged `ruview-cognitum-spaces`,
|
||||
`ruview-spatial-memory`, the ADR-327 policy extension, CLI paging, and the
|
||||
guarded `ruview_spaces_list` metaharness surface. PR #1651 removed stale
|
||||
feature-branch guidance and refreshed the signed package manifest.
|
||||
- The contributor metaharness fixes the API origin, accepts bounded resource,
|
||||
limit, and opaque-cursor inputs, strips API-key compatibility authority over
|
||||
MCP, invokes only the hardened OAuth CLI, and rejects raw sensing or malformed
|
||||
hierarchy/event/alert output. Its test, security, reviewed-brain, flywheel,
|
||||
manifest, audit, exact-tarball, and claim-check gates pass.
|
||||
- Release run `32286297277` rebuilt and smoke-tested the exact package and
|
||||
provenance-published `@ruvnet/ruview` 0.5.0. The public npm registry resolves
|
||||
0.5.0 as `latest`; no workstation publish was used.
|
||||
- `ruview-spatial-memory` keeps one RuVector HNSW index per authenticated
|
||||
tenant/workspace with replay, derivation, retention, cascading-erasure,
|
||||
bounded-explanation, encrypted-snapshot, and reload-verified rotation gates.
|
||||
This is local `SYNTHETIC` evidence, not a production sensing claim.
|
||||
- `ruview-policy` keeps observe/recommend/execute intents distinct, requires
|
||||
exact host grants plus signed approval for consequence, rejects nonce replay,
|
||||
and emits signed hash-chained receipts. `spaces:read` is explicitly denied as
|
||||
execution authority.
|
||||
- Focused Rust gates and the Linux workspace/CLI/security lanes pass. Earlier
|
||||
Windows whole-workspace attempts ended in host compiler failure or timeout;
|
||||
those attempts are not reclassified as green evidence.
|
||||
- No OAuth write/action scope, actuator callback, MQTT deployment claim, sensing
|
||||
accuracy claim, or real-hardware claim is introduced.
|
||||
|
||||
@@ -478,8 +493,9 @@ readback gates pass. Commands and actuators remain out of scope.
|
||||
|
||||
- Two credential types coexist during migration and must stay visibly distinct.
|
||||
- OAuth depends on Identity JWKS availability and correct key rotation.
|
||||
- Production currently exposes the legacy spaces slice; the implemented full
|
||||
hierarchy/events/alerts model remains staged until deployment/readback.
|
||||
- Production exposes both the legacy Space twins and the versioned hierarchy,
|
||||
anonymous entities, semantic events, and alerts over HTTPS. MQTT remains a
|
||||
design contract without deployment evidence.
|
||||
- OAuth workspace IDs will return only documents populated with `workspaceId`;
|
||||
legacy owner-only documents require an explicit migration, never a broad query.
|
||||
- The RuView client exposes no write, command, or agent execution surface. The
|
||||
|
||||
59
docs/adr/ADR-340-iphone-lidar-sensor-bridge.md
Normal file
59
docs/adr/ADR-340-iphone-lidar-sensor-bridge.md
Normal file
@@ -0,0 +1,59 @@
|
||||
# ADR 340: iPhone LiDAR Sensor Bridge
|
||||
|
||||
Status: Proposed
|
||||
|
||||
## Context
|
||||
|
||||
RuView needs a low cost mobile geometry sensor that can contribute calibrated spatial observations without coupling the perception substrate to Apple frameworks.
|
||||
|
||||
ARKit exposes rear LiDAR scene depth through `ARFrame.sceneDepth` and `smoothedSceneDepth` on supported devices. Ordinary mobile web pages do not receive this ARKit depth surface directly, so native capture and web visualization must be separated.
|
||||
|
||||
## Decision
|
||||
|
||||
Use a two layer architecture.
|
||||
|
||||
1. Native Swift and ARKit perform acquisition.
|
||||
2. A modality neutral wire frame transports geometry into browser tools and, next, the RuView HAL.
|
||||
|
||||
The native client will capture depth, confidence, camera intrinsics, and world tracking pose. RGB imagery is excluded from the default transport.
|
||||
|
||||
The protocol identifier is `ruview.lidar.depth.v1`.
|
||||
|
||||
Depth samples are quantized to UInt16 millimeters for transport. Confidence remains UInt8. The default sender downsamples by two spatially and caps transmission at 15 FPS. Full fidelity depth remains available locally for future on device inference.
|
||||
|
||||
## RuView integration boundary
|
||||
|
||||
The transport must not become a second world model. The production receiver converts each packet into the canonical `ruview-hal::Observation`, then passes it through authenticated sensor identity, provenance, OOD gating, uncertainty aware fusion, spatial memory, and WorldGraph adapters.
|
||||
|
||||
Rules:
|
||||
|
||||
1. `source=live` is valid only for frames produced by an active ARKit session.
|
||||
2. Sequence numbers are monotonic per sensor session.
|
||||
3. Wall clock timestamp is separate from ARKit monotonic frame timing.
|
||||
4. RGB is off by default and requires an explicit higher privacy capability.
|
||||
5. Browser clients consume geometry but are not treated as authoritative sensors.
|
||||
6. Unsupported devices fail closed rather than substituting simulated depth.
|
||||
|
||||
## Performance target
|
||||
|
||||
`[SYNTHETIC]` A 256 x 192 Float32 depth map is about 196 KB before confidence and metadata. Downsampling to 128 x 96 and encoding each sample as two byte depth plus one byte confidence yields about 36.9 KB raw. At 15 FPS the raw sensor payload is about 553 KB/s. Base64 raises this to roughly 737 KB/s before JSON metadata. These values are arithmetic sizing estimates, not device measurements.
|
||||
|
||||
The `[CLAIMED target]` for local network latency is below 150 ms p95. A later binary WebSocket or QUIC transport can remove base64 overhead; the exact end-to-end reduction must be measured before it is claimed.
|
||||
|
||||
## Security
|
||||
|
||||
The development relay is LAN-facing, requires a random per-run bearer token, bounds message size, and restricts the files it serves. Its default `ws://` transport is not encrypted, so it is not a production trust boundary.
|
||||
|
||||
Production requires WSS, authenticated sensor identity, replay protection, message size limits, per tenant authorization, provenance receipts, and explicit retention policy before persistence.
|
||||
|
||||
## Consequences
|
||||
|
||||
Benefits include commodity hardware, metric depth, tracked camera pose, rapid room scanning, calibration support for RF sensing, and a practical ground truth source for RuView experiments.
|
||||
|
||||
The main limitation is that Apple provides processed scene depth rather than the underlying raw transient LiDAR waveform. Therefore this implementation supports direct geometry and sensor fusion now, but does not reproduce research systems that require raw multipath time of flight transients for non line of sight reconstruction.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
A physical LiDAR capable iPhone must stream live geometry to the browser viewer with monotonically increasing sequence numbers, no RGB payload, valid confidence maps, and below 150 ms p95 local network latency over a 60 second run.
|
||||
|
||||
CI type-checking and simulator runs do not satisfy this criterion. Until a captured physical-device run records the environment and results, the hardware behavior and latency remain unverified.
|
||||
@@ -5,10 +5,15 @@ PCK@20 (MultiFormer Table VII metric: `‖pred−gt‖ ≤ 0.2·‖R-shoulder
|
||||
|
||||
The flagship [`ruvnet/wifi-densepose-mmfi-pose`](https://huggingface.co/ruvnet/wifi-densepose-mmfi-pose)
|
||||
reaches **83.59%** torso-PCK@20 (vs MultiFormer 72.25%, CSI2Pose 68.41%). But the headline number
|
||||
isn't the whole story for **edge deployment** — on a Raspberry Pi / ESP32-class target, *params and
|
||||
isn't the whole story for **edge deployment** — on a Raspberry Pi-class edge host, *params and
|
||||
latency* matter as much as accuracy. So we swept model size to map the **accuracy-per-parameter
|
||||
frontier**: how small can a WiFi-CSI pose model be and still beat the prior published SOTA?
|
||||
|
||||
> **Hardware compatibility boundary.** These models consume MM-Fi tensors shaped
|
||||
> `[3,114,10]`. Parameter size alone does not make that input, model architecture, or runtime
|
||||
> compatible with an ESP32-S3/C6 capture node. The measurements below are dataset and x86/GPU
|
||||
> measurements; no ESP32 inference latency or live ESP32-to-MM-Fi adapter is claimed.
|
||||
|
||||
## The frontier
|
||||
|
||||
| Model | Params | Latency (batch=1) | torso-PCK@20 | vs SOTA (72.25%) |
|
||||
@@ -38,8 +43,10 @@ Size alone isn't the claim — what matters is **accuracy at the deployed precis
|
||||
|
||||
**The honest edge result:** `micro` is **lossless at int8 (73.5 KB, 74.70%)**, and at **int4 (36.7 KB)
|
||||
naïve post-training quantization falls below SOTA (70.21%) — but quantization-aware training fully
|
||||
recovers it to 74.46%**, still beating MultiFormer. So a **SOTA-beating WiFi-pose model genuinely runs
|
||||
in ~37 KB int4** (with QAT) or **~73 KB int8** (no retraining) — deployable on the sensing node itself.
|
||||
recovers it to 74.46%**, still beating MultiFormer. So a **SOTA-beating WiFi-pose model fits in
|
||||
~37 KB int4** (with QAT) or **~73 KB int8** (no retraining). That is a model-footprint result, not
|
||||
evidence that it runs on an ESP32 sensing node; a compatible capture adapter and embedded runtime
|
||||
still need to be implemented and measured.
|
||||
`nano` (40K params) sits at the SOTA line in fp32 and is best treated as int8.
|
||||
|
||||
(We also tested flagship→tiny **knowledge distillation**: it did *not* help — the tiny students reach
|
||||
|
||||
@@ -75,7 +75,7 @@ WiFi DensePose turns commodity WiFi signals into real-time human pose estimation
|
||||
|
||||
| Option | Cost | Capabilities |
|
||||
|--------|------|-------------|
|
||||
| ESP32-S3 mesh (3-6 boards) | ~$54 | Full CSI: pose, breathing, heartbeat, presence |
|
||||
| ESP32-S3 mesh (3-6 boards) | ~$54 | CSI capture for presence, motion, and vital-sign heuristics; live 17-keypoint pose remains below-target and is not a validated capability |
|
||||
| Intel 5300 / Atheros AR9580 | $50-100 | Full CSI with 3x3 MIMO (Linux only) |
|
||||
| Any WiFi laptop | $0 | RSSI-only: coarse presence and motion detection |
|
||||
|
||||
@@ -1214,6 +1214,22 @@ levels. Read the label, not the headline ([ADR-187](adr/ADR-187-archive-v1-depre
|
||||
|
||||
**Does it actually run, and can a single ESP32 do pose? ([#509](https://github.com/ruvnet/RuView/issues/509), [#1125](https://github.com/ruvnet/RuView/issues/1125))** Yes, it runs, and the results are reproducible: the deterministic signal-pipeline proof (`python archive/v1/data/proof/verify.py`, must print `VERDICT: PASS`), the committed pose training dump (`v2/crates/cog-pose-estimation/cog/artifacts/train_results.json`), and the auditable MM-Fi arena all back specific numbers. But a single-antenna, 56-subcarrier CSI stream at a 20-frame window does *not* carry the fine-grained spatial information the multi-antenna NIC research relies on — so the shippable pose accuracy the project stands behind today is the **MM-Fi benchmark number**, not a live single-ESP32 number. The path to a first reproducible on-device baseline (PCK@20 ≥ 35%) is tracked in [ADR-079](adr/ADR-079-camera-ground-truth-training.md) / [#645](https://github.com/ruvnet/RuView/issues/645).
|
||||
|
||||
### Model and capture compatibility
|
||||
|
||||
These artifacts share a pose objective, but not an input contract or adapter format. A checkpoint
|
||||
is usable only when capture preprocessing, tensor shape, architecture, and runtime all match.
|
||||
|
||||
| Artifact/path | Required input | Measured status | Live ESP32 compatibility |
|
||||
|---------------|----------------|-----------------|--------------------------|
|
||||
| MM-Fi flagship and `micro` transformer | `[N,3,114,10]` amplitude | **MEASURED** on MM-Fi `random_split`; calibration reference is Python `.npz` LoRA | No direct S3/C6 path is validated; resampling a 56-tone SISO stream does not recreate three-antenna MM-Fi input |
|
||||
| Cog `pose_v1.safetensors` | `[N,56,20]` amplitude | **MEASURED** PCK@20 = 3.0%, below the ≥35% target; cog-format LoRA is `.safetensors` | Shape matches the canonical window, but the documented live runtime remains below-target/stub and is not a reliable pose claim |
|
||||
| Viewer `heuristic_pose_from_amplitude` | live canonical amplitude | Skeleton-layout placeholder, not a trained pose model | Renders a demonstrator skeleton only; it is not pose accuracy evidence |
|
||||
| MERIDIAN automatic unlabeled calibration | proposed ~200-frame target-room capture | **PROPOSED** in ADR-027; no validated end-to-end command | Not available. The current calibration tools require paired CSI/keypoint labels and model-specific inputs |
|
||||
|
||||
Calibration files are not interchangeable: `calibrate.py` targets the MM-Fi transformer, while
|
||||
`cog_calibrate.py` targets the cog conv+MLP. See the
|
||||
[calibration reference](../aether-arena/calibration/README.md) for their exact schemas.
|
||||
|
||||
### Download
|
||||
|
||||
```bash
|
||||
@@ -1458,7 +1474,7 @@ The pipeline runs 10 phases:
|
||||
3. Subcarrier resampling (114->56 or 30->56 via Catmull-Rom interpolation)
|
||||
4. Graph transformer construction (17 COCO keypoints, 16 bone edges)
|
||||
5. Cross-attention training (CSI features -> body pose)
|
||||
6. **Domain-adversarial training** (MERIDIAN: gradient reversal + virtual domain augmentation)
|
||||
6. Experimental domain-adversarial components (MERIDIAN research modules; not a validated automatic deployment path)
|
||||
7. Composite loss optimization (MSE + CE + UV + temporal + bone + symmetry)
|
||||
8. SONA adaptation (micro-LoRA + EWC++)
|
||||
9. Sparse inference optimization (hot/cold neuron partitioning)
|
||||
@@ -1474,14 +1490,18 @@ Progressive loading enables instant startup (Layer A loads in <5ms with basic in
|
||||
|
||||
### Cross-Environment Adaptation (MERIDIAN)
|
||||
|
||||
Models trained in one room typically lose 40-70% accuracy in a new room due to different WiFi multipath patterns. The MERIDIAN system (ADR-027) solves this with a 10-second automatic calibration:
|
||||
Models trained in one room can lose substantial accuracy in a new room because the multipath
|
||||
distribution changes. ADR-027 proposes an automatic adaptation design, and the Rust tree contains
|
||||
individual research components, but RuView does **not** currently provide a validated command that
|
||||
turns ~200 unlabeled frames into a working room adapter.
|
||||
|
||||
1. **Deploy** the trained model in a new room
|
||||
2. **Collect** ~200 unlabeled CSI frames (10 seconds at 20 Hz)
|
||||
3. The system automatically generates environment-specific LoRA weights via contrastive test-time training
|
||||
4. No labels, no retraining, no user intervention
|
||||
Current, testable calibration is the separate **labeled** reference in
|
||||
`aether-arena/calibration/`: collect paired CSI/keypoint samples, then fit a model-specific LoRA
|
||||
adapter. The MM-Fi transformer expects `[N,3,114,10]`; the cog expects `[N,56,20]`. Neither adapter
|
||||
loads into the other model, and neither result establishes live ESP32 pose accuracy without a
|
||||
leakage-free held-out capture and mean-pose baseline.
|
||||
|
||||
MERIDIAN components (all pure Rust, +12K parameters):
|
||||
ADR-027 research components:
|
||||
|
||||
| Component | What it does |
|
||||
|-----------|-------------|
|
||||
@@ -1489,7 +1509,7 @@ MERIDIAN components (all pure Rust, +12K parameters):
|
||||
| Domain Factorizer | Separates pose-relevant from room-specific features |
|
||||
| Geometry Encoder | Encodes AP positions (FiLM conditioning with DeepSets) |
|
||||
| Virtual Augmentor | Generates synthetic environments for robust training |
|
||||
| Rapid Adaptation | 10-second unsupervised calibration via contrastive TTT |
|
||||
| Rapid Adaptation | Proposed unlabeled contrastive TTT; not wired as a validated deployment workflow |
|
||||
|
||||
See [ADR-027](adr/ADR-027-cross-environment-domain-generalization.md) for the full design.
|
||||
|
||||
@@ -1647,7 +1667,7 @@ Not every ADR-295–296 remediation item is preview-only. Three are live now:
|
||||
|
||||
| Target | Use case | Source target flag | Notes |
|
||||
|---|---|---|---|
|
||||
| **ESP32-S3** (default) | Production CSI mesh, 17-keypoint pose | `idf.py set-target esp32s3` | Dual-core 240 MHz, PSRAM, native USB-OTG, DVP camera path |
|
||||
| **ESP32-S3** (default) | Production CSI capture mesh; presence/motion/vital heuristics | `idf.py set-target esp32s3` | Dual-core 240 MHz, PSRAM, native USB-OTG, DVP camera path; live 17-keypoint pose is not validated |
|
||||
| **ESP32-C6** ([ADR-110](adr/ADR-110-esp32-c6-firmware-extension.md)) | Wi-Fi 6 / 802.15.4 research, battery seed nodes | `idf.py set-target esp32c6` | Single-core 160 MHz, no PSRAM, 802.11ax HE PHY, 802.15.4 (Thread/Zigbee), LP-core hibernation ~5 µA |
|
||||
|
||||
The same `firmware/esp32-csi-node` source tree builds for both. ESP-IDF picks up `sdkconfig.defaults.esp32c6` automatically when the target is set to `esp32c6`; otherwise it uses `sdkconfig.defaults` (S3). All C6-only modules are `#ifdef`-gated, so the S3 build is byte-identical to today.
|
||||
@@ -2071,7 +2091,11 @@ Pre-trained models are available on HuggingFace:
|
||||
- **SOTA MM-Fi pose model** (82.69% torso-PCK@20) — https://huggingface.co/ruvnet/wifi-densepose-mmfi-pose
|
||||
- **AetherArena leaderboard Space** — https://huggingface.co/spaces/ruvnet/aether-arena
|
||||
|
||||
Download and start sensing immediately — no datasets, no GPU, no training needed. Results are reproducible via `python archive/v1/data/proof/verify.py` (deterministic SHA-256 proof) — see [ADR-168](adr/ADR-168-benchmark-proof.md).
|
||||
The encoder artifact can be downloaded for its documented inference path without retraining. The
|
||||
MM-Fi pose checkpoint is benchmark evidence, not a drop-in live ESP32 model; it requires its exact
|
||||
input contract and runtime described in [Model and capture compatibility](#model-and-capture-compatibility).
|
||||
The deterministic signal-pipeline proof is reproducible via `python archive/v1/data/proof/verify.py`
|
||||
(see [ADR-168](adr/ADR-168-benchmark-proof.md)), but that proof does not validate pose accuracy.
|
||||
|
||||
### Quick Start with Pre-Trained Models
|
||||
|
||||
@@ -2636,7 +2660,12 @@ No. Run `docker run -p 3000:3000 ruvnet/wifi-densepose:latest` and open `http://
|
||||
No. Consumer WiFi exposes only RSSI (one number per access point), not CSI (56+ complex subcarrier values per frame). RSSI supports coarse presence and motion detection. Full pose estimation requires CSI-capable hardware like an ESP32-S3 ($8) or a research NIC.
|
||||
|
||||
**Q: How accurate is the pose estimation?**
|
||||
Accuracy depends on hardware and environment. With a 3-node ESP32 mesh in a single room, the system tracks 17 COCO keypoints. The core algorithm follows the CMU "DensePose From WiFi" paper ([arXiv:2301.00250](https://arxiv.org/abs/2301.00250)). The MERIDIAN domain generalization system (ADR-027) reduces cross-environment accuracy loss from 40-70% to under 15% via 10-second automatic calibration.
|
||||
The strongest published RuView result is the MM-Fi transformer benchmark (82.69% torso-PCK@20 on
|
||||
the matched `random_split` protocol), not a live ESP32 result. The committed cog model measured
|
||||
3.0% PCK@20 on its holdout, below the ≥35% target, and the viewer's live skeleton is a heuristic
|
||||
placeholder. No measured claim currently shows a 3-node ESP32 mesh reliably tracking 17 keypoints.
|
||||
ADR-027's 10-second unlabeled MERIDIAN adaptation is Proposed; the available measured calibration
|
||||
reference uses labeled, model-specific samples.
|
||||
|
||||
**Q: Does it work through walls?**
|
||||
Yes. WiFi signals penetrate non-metallic materials (drywall, wood, concrete up to ~30cm). Metal walls/doors significantly attenuate the signal. With a single AP the effective through-wall range is approximately 5 meters. With a 3-6 node multistatic mesh (ADR-029), attention-weighted cross-viewpoint fusion extends the effective range to ~8 meters through standard residential walls.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"generator": "RuView metaharness provenance v2",
|
||||
"template": "vertical:ruview",
|
||||
"name": "@ruvnet/ruview",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.1",
|
||||
"hosts": [
|
||||
"claude-code",
|
||||
"codex"
|
||||
@@ -31,7 +31,7 @@
|
||||
"flywheel/genome.json": "75db44a3cab70d9459fc8c07863f640ac1214bfaa243483939e1506d63f51214",
|
||||
"flywheel/replay.mjs": "0670ca0b03701f4afe0b4bca8a3d58d481676b61a94a5b98c6a425aefb1159ab",
|
||||
"flywheel/run.mjs": "6d4f97db16900c45367b6538848cbe1915af999e663720dfc51f2bb1698f1cd0",
|
||||
"package.json": "e508476278467cbe63e836d2b42f24649393fb59ce4fe40925d4418da4e79acc",
|
||||
"package.json": "5d29ef238f310c9ee5c57501ab651acc0f856f831b696ada187de71e4b5935a6",
|
||||
"scripts/sync-skills.mjs": "43715dab61e204dc91bbd61755810e8fdb2f66e2b0c0bd791b4bf48a2e293565",
|
||||
"scripts/update-manifest.mjs": "8f56764b8f70aed55da0c7e2417ae875b0d58d781d839b6db7f115f08af61e6b",
|
||||
"scripts/verify-manifest.mjs": "6491a221762efcfeb3e749ecab243b204f17fd5bc871f3d4025597f31b8f0f10",
|
||||
@@ -43,7 +43,7 @@
|
||||
"skills/verify.md": "2d38d240e9810a7827e2ebd3717dc0f85c646cc92e46c3812fe77c5b9eb40b76",
|
||||
"src/brain.js": "0f16a75aea943acdacc430ff11d5df7ecdec9cca2ab497795ff6f33eaebdfab6",
|
||||
"src/guardrails.js": "aacc8fa6088f7f1ccea3a0b02171a5c516b95d3416ee3ba87add3879a1d6aaad",
|
||||
"src/guidance.js": "beaa130b546fdb495ae8b17e52c46ca8bbe50c218f05705244922d3633125f4e",
|
||||
"src/guidance.js": "583904c854eb17e98cb7d959330989c01990a71cff091515777aba8f345de1bf",
|
||||
"src/hosts/claude-code.js": "2212bc39b49822018800dfe33a471e56bbb4c5233d716bfa7aa4fff77aa23edb",
|
||||
"src/hosts/codex.js": "d41ecd132ce2db7b47aad9cebbc020d70e6810d48c3554858d099ff2e8f6608b",
|
||||
"src/hosts/index.js": "ab276c41ab722bcdf72c2d1649cecbb760ae05c41c1372aae4c2447aa7c11539",
|
||||
@@ -55,7 +55,7 @@
|
||||
"src/spaces.js": "45ef786537cb2a446db5e926e5a1c10b73639d2767dec84611f914f78d4325eb",
|
||||
"src/tools.js": "55960c9a677661763e0317fd54ccc787c2edb39c87371c7fbc40cd55f0761c04"
|
||||
},
|
||||
"filesDigest": "961357bd4458bd6fa759f0a8d5bf042c7b652dcea3700363ae25d03299aa5027",
|
||||
"filesDigest": "28a3bbd9bbcf966df9fae8ec6ea5be3441f6ab535c1636bb1ce33f67b827d423",
|
||||
"brainDigest": "c0fb7b079ded157059b91601361429944697dae3cc42abc00dfe1a680986b0f4",
|
||||
"gateFingerprint": "6e53c784eee38310188948fc75fb49e6b4ebc04e247d01b903fa8c8a92d67bdd",
|
||||
"developmentPins": {
|
||||
|
||||
@@ -1 +1 @@
|
||||
8a4cb8b2a5766455acbf936826d499ec12432e77f9c17f90526ebcdfdc08485d manifest.json
|
||||
478ccaff9aa249bc7ea6e20551ccc9ac88697a3cc91a7b55400337c5e939a19e manifest.json
|
||||
|
||||
4
harness/ruview/package-lock.json
generated
4
harness/ruview/package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@ruvnet/ruview",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@ruvnet/ruview",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.1",
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"ruview": "bin/cli.js"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@ruvnet/ruview",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.1",
|
||||
"description": "RuView WiFi-sensing operator harness — onboard, calibrate, verify, enforce evidence guardrails, and read Cognitum Spaces through explicitly granted OAuth.",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -250,7 +250,7 @@ const CAPABILITIES = Object.freeze([
|
||||
'The projection is read-only and grants no write, pairing, command, policy-approval, or actuator authority.',
|
||||
'MCP requires the credential-use grant; bearer tokens and API keys are never accepted as tool arguments.',
|
||||
'OAuth refresh may rotate the local credential file before a read returns.',
|
||||
'Production evidence covers the legacy flat Spaces read. Versioned collections, spatial memory, and governed actions remain staged until workflow deployment/readback.',
|
||||
'Production evidence covers legacy and versioned HTTPS reads. Spatial memory remains tenant/workspace-local, while governed actions remain separately policy-gated; neither expands OAuth authority.',
|
||||
'Persistent memory is local tenant/workspace state and governed actions expose authorization receipts only; neither expands OAuth authority.',
|
||||
],
|
||||
},
|
||||
|
||||
78
integrations/iphone-lidar/README.md
Normal file
78
integrations/iphone-lidar/README.md
Normal file
@@ -0,0 +1,78 @@
|
||||
# RuView iPhone LiDAR
|
||||
|
||||
This experimental integration provides the native and browser components needed to use a LiDAR-capable iPhone as a RuView geometry sensor. The native source is type-checked against the iOS SDK in CI; physical-device validation is tracked separately below.
|
||||
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
iPhone LiDAR
|
||||
-> ARKit sceneDepth
|
||||
-> depth + confidence + camera intrinsics + device pose
|
||||
-> compact u16 millimeter wire frame
|
||||
-> WebSocket relay
|
||||
-> browser point cloud
|
||||
-> future RuView HAL / fusion ingest
|
||||
```
|
||||
|
||||
The native path is the sensor. The web path is a receiver and visualization surface. Mobile Safari does not expose ARKit scene depth directly to ordinary web pages, so the browser cannot replace the native capture layer on iPhone today.
|
||||
|
||||
## Native iPhone path
|
||||
|
||||
Create an iOS SwiftUI app target in Xcode, deployment target iOS 17 or newer, then add the files under `native/RuViewLiDAR/` to the target.
|
||||
|
||||
Add this Info.plist value:
|
||||
|
||||
```xml
|
||||
<key>NSCameraUsageDescription</key>
|
||||
<string>RuView uses the camera and LiDAR scanner to capture local depth geometry.</string>
|
||||
```
|
||||
|
||||
Run on a physical LiDAR capable iPhone or iPad. The simulator does not provide LiDAR scene depth.
|
||||
|
||||
The app requests `ARWorldTrackingConfiguration` with `.sceneDepth`, checks `supportsFrameSemantics`, extracts `ARDepthData.depthMap` and `confidenceMap`, and never transmits RGB camera frames.
|
||||
|
||||
## Browser path
|
||||
|
||||
```bash
|
||||
cd integrations/iphone-lidar/web
|
||||
npm ci
|
||||
npm test
|
||||
npm start
|
||||
```
|
||||
|
||||
The relay prints a random per-run access token. Open the printed browser URL and set the iPhone endpoint to the printed native URL. They have this form:
|
||||
|
||||
```text
|
||||
http://HOST:8787/?token=TOKEN
|
||||
ws://HOST:8787/ws/lidar?token=TOKEN
|
||||
```
|
||||
|
||||
Set `RUVIEW_LIDAR_TOKEN` to supply the token explicitly. The token only prevents unauthenticated peers from joining the development relay; because `ws://` does not encrypt it, production use requires TLS and `wss://`.
|
||||
|
||||
## Wire format
|
||||
|
||||
Schema: `ruview.lidar.depth.v1`
|
||||
|
||||
Depth is downsampled by 2 in each dimension by default and streamed at a maximum of 15 FPS. Each depth sample is encoded as little endian UInt16 millimeters plus one UInt8 confidence value. `[SYNTHETIC]` Arithmetic sizing reduces the depth payload from roughly 196 KB per 256 x 192 Float32 frame to roughly 37 KB per 128 x 96 frame before base64 and JSON overhead.
|
||||
|
||||
`[SYNTHETIC]` At 15 FPS that is approximately 0.75 MB/s after base64 overhead, versus roughly 8 MB/s for uncompressed Float32 JSON at full resolution. These are sizing estimates, not device or network measurements.
|
||||
|
||||
## Privacy and governance
|
||||
|
||||
The initial implementation labels provenance as `source=live` and `privacyClass=geometry-only`. It sends depth geometry, confidence, camera intrinsics, pose, sequence, and wall clock timestamp. It does not send RGB imagery.
|
||||
|
||||
The development relay requires an ephemeral token and bounds each WebSocket message, but it is not a production trust boundary. Production integration should terminate the WebSocket inside RuView, authenticate the device using the existing sensor identity path, convert each frame into `ruview-hal::Observation`, and attach witness receipts before fusion or persistence.
|
||||
|
||||
## Validation status
|
||||
|
||||
- `[MEASURED]` The committed Node tests cover wire decoding, malformed inputs, relay authentication, static-file restrictions, and live WebSocket forwarding.
|
||||
- `[MEASURED]` GitHub Actions type-checks the native sources with strict concurrency against the iOS 17 SDK.
|
||||
- Physical iPhone capture, end-to-end rendering, confidence-map behavior, and the latency target are not yet measured. A simulator or CI compile does not satisfy the hardware acceptance test.
|
||||
|
||||
## Acceptance test
|
||||
|
||||
1. Run the relay and browser viewer.
|
||||
2. Run the native app on a LiDAR capable iPhone.
|
||||
3. Start LiDAR capture and enable streaming.
|
||||
4. Move the phone through a room.
|
||||
5. Verify the browser shows a changing point cloud, sequence increases monotonically, latency stays below the `[CLAIMED target]` of 150 ms p95 on a local WiFi network, and no RGB payload is present in captured WebSocket frames.
|
||||
121
integrations/iphone-lidar/native/RuViewLiDAR/ContentView.swift
Normal file
121
integrations/iphone-lidar/native/RuViewLiDAR/ContentView.swift
Normal file
@@ -0,0 +1,121 @@
|
||||
import SwiftUI
|
||||
|
||||
struct ContentView: View {
|
||||
@StateObject private var capture = LiDARCaptureManager()
|
||||
@State private var endpoint = "ws://HOST:8787/ws/lidar?token=TOKEN"
|
||||
@State private var streaming = false
|
||||
@State private var status = "Idle"
|
||||
|
||||
private let streamer = WebSocketStreamer()
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
Form {
|
||||
Section("Sensor") {
|
||||
HStack {
|
||||
Text("State")
|
||||
Spacer()
|
||||
Text(stateText)
|
||||
.foregroundStyle(stateColor)
|
||||
}
|
||||
HStack {
|
||||
Text("Capture FPS")
|
||||
Spacer()
|
||||
Text(capture.framesPerSecond.formatted(.number.precision(.fractionLength(1))))
|
||||
}
|
||||
if let frame = capture.lastFrame {
|
||||
HStack {
|
||||
Text("Depth")
|
||||
Spacer()
|
||||
Text("\(frame.depth.width) x \(frame.depth.height)")
|
||||
}
|
||||
HStack {
|
||||
Text("Sequence")
|
||||
Spacer()
|
||||
Text("\(frame.provenance.sequence)")
|
||||
}
|
||||
}
|
||||
|
||||
Button("Start LiDAR") {
|
||||
capture.start(smoothed: false)
|
||||
}
|
||||
.disabled(capture.state == .running)
|
||||
|
||||
Button("Stop") {
|
||||
capture.stop()
|
||||
}
|
||||
.disabled(capture.state != .running)
|
||||
}
|
||||
|
||||
Section("RuView Stream") {
|
||||
TextField("ws://host:port/ws/lidar", text: $endpoint)
|
||||
.textInputAutocapitalization(.never)
|
||||
.autocorrectionDisabled()
|
||||
|
||||
Toggle("Stream geometry", isOn: $streaming)
|
||||
.onChange(of: streaming) { _, enabled in
|
||||
Task {
|
||||
if enabled {
|
||||
do {
|
||||
try await streamer.connect(to: endpoint)
|
||||
status = "Connected"
|
||||
} catch {
|
||||
streaming = false
|
||||
status = error.localizedDescription
|
||||
}
|
||||
} else {
|
||||
await streamer.disconnect()
|
||||
status = "Disconnected"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Text(status)
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
|
||||
Section("Privacy") {
|
||||
Text("This implementation transmits depth geometry, confidence, camera intrinsics, and device pose. RGB camera frames are not transmitted.")
|
||||
.font(.footnote)
|
||||
}
|
||||
}
|
||||
.navigationTitle("RuView LiDAR")
|
||||
.onAppear {
|
||||
capture.onFrame = { frame in
|
||||
guard streaming else { return }
|
||||
Task {
|
||||
do {
|
||||
try await streamer.send(frame, maxFPS: 15, sampleStep: 2)
|
||||
} catch {
|
||||
await MainActor.run {
|
||||
status = error.localizedDescription
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.onDisappear {
|
||||
capture.stop()
|
||||
Task { await streamer.disconnect() }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var stateText: String {
|
||||
switch capture.state {
|
||||
case .idle: return "Idle"
|
||||
case .unsupported: return "No LiDAR"
|
||||
case .running: return "Live"
|
||||
case .failed(let message): return "Error: \(message)"
|
||||
}
|
||||
}
|
||||
|
||||
private var stateColor: Color {
|
||||
switch capture.state {
|
||||
case .running: return .green
|
||||
case .failed, .unsupported: return .red
|
||||
case .idle: return .secondary
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import ARKit
|
||||
import CoreVideo
|
||||
import Foundation
|
||||
|
||||
@MainActor
|
||||
final class LiDARCaptureManager: NSObject, ObservableObject {
|
||||
enum State: Equatable {
|
||||
case idle
|
||||
case unsupported
|
||||
case running
|
||||
case failed(String)
|
||||
}
|
||||
|
||||
@Published private(set) var state: State = .idle
|
||||
@Published private(set) var lastFrame: RuViewLiDARFrame?
|
||||
@Published private(set) var framesPerSecond: Double = 0
|
||||
|
||||
let session = ARSession()
|
||||
var onFrame: (@MainActor @Sendable (RuViewLiDARFrame) -> Void)?
|
||||
|
||||
private var sequence: UInt64 = 0
|
||||
private var lastTimestamp: TimeInterval?
|
||||
private let processingQueue = DispatchQueue(label: "one.ruv.lidar.capture", qos: .userInitiated)
|
||||
|
||||
override init() {
|
||||
super.init()
|
||||
session.delegate = self
|
||||
session.delegateQueue = processingQueue
|
||||
}
|
||||
|
||||
func start(smoothed: Bool = false) {
|
||||
let configuration = ARWorldTrackingConfiguration()
|
||||
let semantic: ARConfiguration.FrameSemantics = smoothed ? .smoothedSceneDepth : .sceneDepth
|
||||
|
||||
guard ARWorldTrackingConfiguration.supportsFrameSemantics(semantic) else {
|
||||
state = .unsupported
|
||||
return
|
||||
}
|
||||
|
||||
configuration.frameSemantics.insert(semantic)
|
||||
configuration.worldAlignment = .gravity
|
||||
session.run(configuration, options: [.resetTracking, .removeExistingAnchors])
|
||||
state = .running
|
||||
}
|
||||
|
||||
func stop() {
|
||||
session.pause()
|
||||
state = .idle
|
||||
}
|
||||
|
||||
nonisolated private func makeFrame(from frame: ARFrame) -> RuViewLiDARFrame? {
|
||||
guard let sceneDepth = frame.sceneDepth ?? frame.smoothedSceneDepth else { return nil }
|
||||
let depthMap = sceneDepth.depthMap
|
||||
let confidenceMap = sceneDepth.confidenceMap
|
||||
|
||||
guard CVPixelBufferLockBaseAddress(depthMap, .readOnly) == kCVReturnSuccess else {
|
||||
return nil
|
||||
}
|
||||
defer { CVPixelBufferUnlockBaseAddress(depthMap, .readOnly) }
|
||||
|
||||
guard CVPixelBufferGetPixelFormatType(depthMap) == kCVPixelFormatType_DepthFloat32,
|
||||
let depthBase = CVPixelBufferGetBaseAddress(depthMap) else {
|
||||
return nil
|
||||
}
|
||||
|
||||
let width = CVPixelBufferGetWidth(depthMap)
|
||||
let height = CVPixelBufferGetHeight(depthMap)
|
||||
let stride = CVPixelBufferGetBytesPerRow(depthMap) / MemoryLayout<Float>.size
|
||||
let pointer = depthBase.assumingMemoryBound(to: Float.self)
|
||||
|
||||
var meters = [Float]()
|
||||
meters.reserveCapacity(width * height)
|
||||
for y in 0..<height {
|
||||
let row = pointer.advanced(by: y * stride)
|
||||
for x in 0..<width {
|
||||
let value = row[x]
|
||||
meters.append(value.isFinite && value > 0 ? value : 0)
|
||||
}
|
||||
}
|
||||
|
||||
var confidence = [UInt8](repeating: 0, count: width * height)
|
||||
if let confidenceMap,
|
||||
CVPixelBufferGetPixelFormatType(confidenceMap) == kCVPixelFormatType_OneComponent8,
|
||||
CVPixelBufferGetWidth(confidenceMap) == width,
|
||||
CVPixelBufferGetHeight(confidenceMap) == height,
|
||||
CVPixelBufferLockBaseAddress(confidenceMap, .readOnly) == kCVReturnSuccess {
|
||||
defer { CVPixelBufferUnlockBaseAddress(confidenceMap, .readOnly) }
|
||||
if let confidenceBase = CVPixelBufferGetBaseAddress(confidenceMap) {
|
||||
let confidenceStride = CVPixelBufferGetBytesPerRow(confidenceMap)
|
||||
let confidencePointer = confidenceBase.assumingMemoryBound(to: UInt8.self)
|
||||
for y in 0..<height {
|
||||
let row = confidencePointer.advanced(by: y * confidenceStride)
|
||||
for x in 0..<width {
|
||||
confidence[y * width + x] = row[x]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return RuViewLiDARFrame(
|
||||
intrinsics: frame.camera.intrinsics,
|
||||
imageResolution: frame.camera.imageResolution,
|
||||
cameraTransform: frame.camera.transform,
|
||||
depthWidth: width,
|
||||
depthHeight: height,
|
||||
depthMeters: meters,
|
||||
confidence: confidence,
|
||||
sequence: 0,
|
||||
timestamp: Date().timeIntervalSince1970
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
extension LiDARCaptureManager: ARSessionDelegate {
|
||||
nonisolated func session(_ session: ARSession, didUpdate frame: ARFrame) {
|
||||
guard let base = makeFrame(from: frame) else { return }
|
||||
let frameTimestamp = frame.timestamp
|
||||
|
||||
Task { @MainActor in
|
||||
sequence &+= 1
|
||||
let corrected = base.assigningSequence(sequence)
|
||||
|
||||
if let previous = lastTimestamp {
|
||||
let delta = frameTimestamp - previous
|
||||
if delta > 0 { framesPerSecond = 1.0 / delta }
|
||||
}
|
||||
lastTimestamp = frameTimestamp
|
||||
lastFrame = corrected
|
||||
onFrame?(corrected)
|
||||
}
|
||||
}
|
||||
|
||||
nonisolated func session(_ session: ARSession, didFailWithError error: Error) {
|
||||
Task { @MainActor in
|
||||
state = .failed(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
}
|
||||
118
integrations/iphone-lidar/native/RuViewLiDAR/RuViewFrame.swift
Normal file
118
integrations/iphone-lidar/native/RuViewLiDAR/RuViewFrame.swift
Normal file
@@ -0,0 +1,118 @@
|
||||
import Foundation
|
||||
import simd
|
||||
|
||||
struct RuViewLiDARFrame: Codable, Sendable {
|
||||
struct Intrinsics: Codable, Sendable {
|
||||
let fx: Float
|
||||
let fy: Float
|
||||
let cx: Float
|
||||
let cy: Float
|
||||
let imageWidth: Int
|
||||
let imageHeight: Int
|
||||
}
|
||||
|
||||
struct Pose: Codable, Sendable {
|
||||
let matrix: [Float]
|
||||
}
|
||||
|
||||
struct Depth: Codable, Sendable {
|
||||
let width: Int
|
||||
let height: Int
|
||||
let meters: [Float]
|
||||
let confidence: [UInt8]
|
||||
}
|
||||
|
||||
struct Provenance: Codable, Sendable {
|
||||
let sensor: String
|
||||
let source: String
|
||||
let privacyClass: String
|
||||
let sequence: UInt64
|
||||
let timestampNs: UInt64
|
||||
let schema: String
|
||||
}
|
||||
|
||||
let type: String
|
||||
let intrinsics: Intrinsics
|
||||
let pose: Pose
|
||||
let depth: Depth
|
||||
let provenance: Provenance
|
||||
|
||||
init(
|
||||
intrinsics: simd_float3x3,
|
||||
imageResolution: CGSize,
|
||||
cameraTransform: simd_float4x4,
|
||||
depthWidth: Int,
|
||||
depthHeight: Int,
|
||||
depthMeters: [Float],
|
||||
confidence: [UInt8],
|
||||
sequence: UInt64,
|
||||
timestamp: TimeInterval
|
||||
) {
|
||||
self.type = "ruview.lidar.depth.v1"
|
||||
self.intrinsics = Intrinsics(
|
||||
fx: intrinsics.columns.0.x,
|
||||
fy: intrinsics.columns.1.y,
|
||||
cx: intrinsics.columns.2.x,
|
||||
cy: intrinsics.columns.2.y,
|
||||
imageWidth: Int(imageResolution.width),
|
||||
imageHeight: Int(imageResolution.height)
|
||||
)
|
||||
self.pose = Pose(matrix: cameraTransform.columnMajorArray)
|
||||
self.depth = Depth(
|
||||
width: depthWidth,
|
||||
height: depthHeight,
|
||||
meters: depthMeters,
|
||||
confidence: confidence
|
||||
)
|
||||
self.provenance = Provenance(
|
||||
sensor: "apple-arkit-scene-depth",
|
||||
source: "live",
|
||||
privacyClass: "geometry-only",
|
||||
sequence: sequence,
|
||||
timestampNs: UInt64(max(0, timestamp) * 1_000_000_000),
|
||||
schema: "ruview.lidar.depth.v1"
|
||||
)
|
||||
}
|
||||
|
||||
func assigningSequence(_ sequence: UInt64) -> RuViewLiDARFrame {
|
||||
RuViewLiDARFrame(
|
||||
type: type,
|
||||
intrinsics: intrinsics,
|
||||
pose: pose,
|
||||
depth: depth,
|
||||
provenance: Provenance(
|
||||
sensor: provenance.sensor,
|
||||
source: provenance.source,
|
||||
privacyClass: provenance.privacyClass,
|
||||
sequence: sequence,
|
||||
timestampNs: provenance.timestampNs,
|
||||
schema: provenance.schema
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
private init(
|
||||
type: String,
|
||||
intrinsics: Intrinsics,
|
||||
pose: Pose,
|
||||
depth: Depth,
|
||||
provenance: Provenance
|
||||
) {
|
||||
self.type = type
|
||||
self.intrinsics = intrinsics
|
||||
self.pose = pose
|
||||
self.depth = depth
|
||||
self.provenance = provenance
|
||||
}
|
||||
}
|
||||
|
||||
private extension simd_float4x4 {
|
||||
var columnMajorArray: [Float] {
|
||||
[
|
||||
columns.0.x, columns.0.y, columns.0.z, columns.0.w,
|
||||
columns.1.x, columns.1.y, columns.1.z, columns.1.w,
|
||||
columns.2.x, columns.2.y, columns.2.z, columns.2.w,
|
||||
columns.3.x, columns.3.y, columns.3.z, columns.3.w
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import SwiftUI
|
||||
|
||||
@main
|
||||
struct RuViewLiDARApp: App {
|
||||
var body: some Scene {
|
||||
WindowGroup {
|
||||
ContentView()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import Foundation
|
||||
|
||||
actor WebSocketStreamer {
|
||||
enum StreamError: Error {
|
||||
case invalidURL
|
||||
}
|
||||
|
||||
struct WirePacket: Codable {
|
||||
struct Depth: Codable {
|
||||
let width: Int
|
||||
let height: Int
|
||||
let encoding: String
|
||||
let millimetersBase64: String
|
||||
let confidenceBase64: String
|
||||
}
|
||||
|
||||
let type: String
|
||||
let intrinsics: RuViewLiDARFrame.Intrinsics
|
||||
let pose: RuViewLiDARFrame.Pose
|
||||
let depth: Depth
|
||||
let provenance: RuViewLiDARFrame.Provenance
|
||||
}
|
||||
|
||||
private var task: URLSessionWebSocketTask?
|
||||
private let encoder = JSONEncoder()
|
||||
private var lastSentNs: UInt64 = 0
|
||||
|
||||
func connect(to endpoint: String) throws {
|
||||
guard let url = URL(string: endpoint),
|
||||
url.scheme == "ws" || url.scheme == "wss" else {
|
||||
throw StreamError.invalidURL
|
||||
}
|
||||
task?.cancel(with: .goingAway, reason: nil)
|
||||
let socket = URLSession.shared.webSocketTask(with: url)
|
||||
socket.resume()
|
||||
task = socket
|
||||
}
|
||||
|
||||
func disconnect() {
|
||||
task?.cancel(with: .goingAway, reason: nil)
|
||||
task = nil
|
||||
}
|
||||
|
||||
func send(_ frame: RuViewLiDARFrame, maxFPS: UInt64 = 15, sampleStep: Int = 2) async throws {
|
||||
guard let task else { return }
|
||||
|
||||
let timestamp = frame.provenance.timestampNs
|
||||
let minDelta = 1_000_000_000 / max(1, maxFPS)
|
||||
guard timestamp >= lastSentNs + minDelta else { return }
|
||||
lastSentNs = timestamp
|
||||
|
||||
let packet = Self.makeWirePacket(frame, sampleStep: max(1, sampleStep))
|
||||
let data = try encoder.encode(packet)
|
||||
guard let string = String(data: data, encoding: .utf8) else { return }
|
||||
try await task.send(.string(string))
|
||||
}
|
||||
|
||||
static func makeWirePacket(_ frame: RuViewLiDARFrame, sampleStep: Int) -> WirePacket {
|
||||
let step = max(1, sampleStep)
|
||||
let sourceWidth = frame.depth.width
|
||||
let sourceHeight = frame.depth.height
|
||||
let width = (sourceWidth + step - 1) / step
|
||||
let height = (sourceHeight + step - 1) / step
|
||||
|
||||
var millimeters = Data(capacity: width * height * 2)
|
||||
var confidence = Data(capacity: width * height)
|
||||
|
||||
for y in stride(from: 0, to: sourceHeight, by: step) {
|
||||
for x in stride(from: 0, to: sourceWidth, by: step) {
|
||||
let index = y * sourceWidth + x
|
||||
let meters = frame.depth.meters[index]
|
||||
let mm = UInt16(clamping: Int((meters * 1000).rounded()))
|
||||
var littleEndian = mm.littleEndian
|
||||
withUnsafeBytes(of: &littleEndian) { millimeters.append(contentsOf: $0) }
|
||||
confidence.append(frame.depth.confidence[index])
|
||||
}
|
||||
}
|
||||
|
||||
return WirePacket(
|
||||
type: frame.type,
|
||||
intrinsics: frame.intrinsics,
|
||||
pose: frame.pose,
|
||||
depth: WirePacket.Depth(
|
||||
width: width,
|
||||
height: height,
|
||||
encoding: "u16le-mm+u8-confidence",
|
||||
millimetersBase64: millimeters.base64EncodedString(),
|
||||
confidenceBase64: confidence.base64EncodedString()
|
||||
),
|
||||
provenance: frame.provenance
|
||||
)
|
||||
}
|
||||
}
|
||||
108
integrations/iphone-lidar/web/app.mjs
Normal file
108
integrations/iphone-lidar/web/app.mjs
Normal file
@@ -0,0 +1,108 @@
|
||||
import { decodeLiDARPacket, depthToPointCloud } from './codec.mjs';
|
||||
|
||||
const canvas = document.querySelector('#view');
|
||||
const ctx = canvas.getContext('2d');
|
||||
const status = document.querySelector('#status');
|
||||
const fpsEl = document.querySelector('#fps');
|
||||
const pointsEl = document.querySelector('#points');
|
||||
const seqEl = document.querySelector('#seq');
|
||||
const latencyEl = document.querySelector('#latency');
|
||||
const sensorEl = document.querySelector('#sensor');
|
||||
|
||||
let lastFrameAt = performance.now();
|
||||
let yaw = 0.3;
|
||||
let pitch = -0.15;
|
||||
let scale = 120;
|
||||
|
||||
function connect() {
|
||||
const token = new URLSearchParams(location.search).get('token');
|
||||
if (!token) {
|
||||
status.textContent = 'TOKEN REQUIRED';
|
||||
status.dataset.state = 'warn';
|
||||
return;
|
||||
}
|
||||
|
||||
const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||
const socket = new WebSocket(`${protocol}//${location.host}/ws/lidar?token=${encodeURIComponent(token)}`);
|
||||
|
||||
socket.addEventListener('open', () => {
|
||||
status.textContent = 'LIVE';
|
||||
status.dataset.state = 'live';
|
||||
});
|
||||
|
||||
socket.addEventListener('close', () => {
|
||||
status.textContent = 'RECONNECTING';
|
||||
status.dataset.state = 'warn';
|
||||
setTimeout(connect, 1000);
|
||||
});
|
||||
|
||||
socket.addEventListener('message', (event) => {
|
||||
try {
|
||||
const raw = JSON.parse(event.data);
|
||||
const frame = decodeLiDARPacket(raw);
|
||||
const points = depthToPointCloud(frame, 1);
|
||||
render(points);
|
||||
|
||||
const now = performance.now();
|
||||
const delta = now - lastFrameAt;
|
||||
lastFrameAt = now;
|
||||
fpsEl.textContent = delta > 0 ? (1000 / delta).toFixed(1) : '0.0';
|
||||
pointsEl.textContent = points.length.toLocaleString();
|
||||
seqEl.textContent = frame.provenance.sequence;
|
||||
latencyEl.textContent = Math.max(0, Date.now() - Number(frame.provenance.timestampNs / 1_000_000)).toFixed(0);
|
||||
sensorEl.textContent = frame.provenance.sensor;
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
status.textContent = 'FRAME ERROR';
|
||||
status.dataset.state = 'warn';
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function resize() {
|
||||
const dpr = Math.min(devicePixelRatio || 1, 2);
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
canvas.width = Math.max(1, Math.floor(rect.width * dpr));
|
||||
canvas.height = Math.max(1, Math.floor(rect.height * dpr));
|
||||
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
|
||||
}
|
||||
|
||||
function render(points) {
|
||||
resize();
|
||||
const w = canvas.clientWidth;
|
||||
const h = canvas.clientHeight;
|
||||
ctx.clearRect(0, 0, w, h);
|
||||
ctx.fillStyle = '#091018';
|
||||
ctx.fillRect(0, 0, w, h);
|
||||
|
||||
const cy = Math.cos(yaw);
|
||||
const sy = Math.sin(yaw);
|
||||
const cp = Math.cos(pitch);
|
||||
const sp = Math.sin(pitch);
|
||||
|
||||
ctx.fillStyle = '#58e0d2';
|
||||
for (let i = 0; i < points.length; i += 1) {
|
||||
let [x, y, z] = points[i];
|
||||
const rx = x * cy - z * sy;
|
||||
const rz = x * sy + z * cy;
|
||||
const ry = y * cp - rz * sp;
|
||||
const rz2 = y * sp + rz * cp;
|
||||
const perspective = 1 / Math.max(0.35, 1.8 - rz2 * 0.12);
|
||||
const px = w / 2 + rx * scale * perspective;
|
||||
const py = h / 2 + ry * scale * perspective;
|
||||
if (px >= 0 && px < w && py >= 0 && py < h) ctx.fillRect(px, py, 1.4, 1.4);
|
||||
}
|
||||
}
|
||||
|
||||
canvas.addEventListener('pointermove', (event) => {
|
||||
if (!event.buttons) return;
|
||||
yaw += event.movementX * 0.006;
|
||||
pitch += event.movementY * 0.006;
|
||||
});
|
||||
|
||||
canvas.addEventListener('wheel', (event) => {
|
||||
event.preventDefault();
|
||||
scale = Math.max(40, Math.min(400, scale - event.deltaY * 0.2));
|
||||
}, { passive: false });
|
||||
|
||||
connect();
|
||||
121
integrations/iphone-lidar/web/codec.mjs
Normal file
121
integrations/iphone-lidar/web/codec.mjs
Normal file
@@ -0,0 +1,121 @@
|
||||
export function decodeLiDARPacket(packet) {
|
||||
if (!packet || packet.type !== 'ruview.lidar.depth.v1') {
|
||||
throw new Error('Unsupported LiDAR packet type');
|
||||
}
|
||||
|
||||
const { depth } = packet;
|
||||
if (!depth || depth.encoding !== 'u16le-mm+u8-confidence') {
|
||||
throw new Error('Unsupported depth encoding');
|
||||
}
|
||||
|
||||
assertPositiveInteger(depth.width, 'depth.width');
|
||||
assertPositiveInteger(depth.height, 'depth.height');
|
||||
assertIntrinsics(packet.intrinsics);
|
||||
if (!packet.pose || !Array.isArray(packet.pose.matrix) || packet.pose.matrix.length !== 16
|
||||
|| packet.pose.matrix.some((value) => !Number.isFinite(value))) {
|
||||
throw new Error('Invalid camera pose');
|
||||
}
|
||||
|
||||
const mmBytes = base64ToBytes(depth.millimetersBase64, 'millimetersBase64');
|
||||
const confidence = base64ToBytes(depth.confidenceBase64, 'confidenceBase64');
|
||||
const expectedPixels = depth.width * depth.height;
|
||||
|
||||
if (!Number.isSafeInteger(expectedPixels) || expectedPixels > 1_000_000) {
|
||||
throw new Error('Depth dimensions exceed the supported pixel limit');
|
||||
}
|
||||
|
||||
if (mmBytes.byteLength !== expectedPixels * 2) {
|
||||
throw new Error(`Depth payload length mismatch: expected ${expectedPixels * 2}, got ${mmBytes.byteLength}`);
|
||||
}
|
||||
if (confidence.byteLength !== expectedPixels) {
|
||||
throw new Error(`Confidence payload length mismatch: expected ${expectedPixels}, got ${confidence.byteLength}`);
|
||||
}
|
||||
|
||||
const view = new DataView(mmBytes.buffer, mmBytes.byteOffset, mmBytes.byteLength);
|
||||
const meters = new Float32Array(expectedPixels);
|
||||
for (let i = 0; i < expectedPixels; i += 1) {
|
||||
meters[i] = view.getUint16(i * 2, true) / 1000;
|
||||
}
|
||||
|
||||
return {
|
||||
...packet,
|
||||
depth: {
|
||||
width: depth.width,
|
||||
height: depth.height,
|
||||
meters,
|
||||
confidence,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function depthToPointCloud(frame, confidenceThreshold = 1) {
|
||||
const { width, height, meters, confidence } = frame.depth;
|
||||
const { fx, fy, cx, cy, imageWidth, imageHeight } = frame.intrinsics;
|
||||
|
||||
assertPositiveInteger(width, 'depth.width');
|
||||
assertPositiveInteger(height, 'depth.height');
|
||||
assertIntrinsics(frame.intrinsics);
|
||||
if (meters.length !== width * height || confidence.length !== width * height) {
|
||||
throw new Error('Decoded depth array length mismatch');
|
||||
}
|
||||
if (!Number.isFinite(confidenceThreshold) || confidenceThreshold < 0 || confidenceThreshold > 255) {
|
||||
throw new Error('Invalid confidence threshold');
|
||||
}
|
||||
|
||||
const sx = width / imageWidth;
|
||||
const sy = height / imageHeight;
|
||||
const scaledFx = fx * sx;
|
||||
const scaledFy = fy * sy;
|
||||
const scaledCx = cx * sx;
|
||||
const scaledCy = cy * sy;
|
||||
|
||||
const points = [];
|
||||
for (let v = 0; v < height; v += 1) {
|
||||
for (let u = 0; u < width; u += 1) {
|
||||
const index = v * width + u;
|
||||
const z = meters[index];
|
||||
if (!Number.isFinite(z) || z <= 0 || confidence[index] < confidenceThreshold) continue;
|
||||
|
||||
const x = ((u - scaledCx) / scaledFx) * z;
|
||||
const y = ((v - scaledCy) / scaledFy) * z;
|
||||
points.push([x, y === 0 ? 0 : -y, -z]);
|
||||
}
|
||||
}
|
||||
return points;
|
||||
}
|
||||
|
||||
function assertPositiveInteger(value, name) {
|
||||
if (!Number.isSafeInteger(value) || value <= 0) {
|
||||
throw new Error(`${name} must be a positive integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertIntrinsics(intrinsics) {
|
||||
if (!intrinsics
|
||||
|| !Number.isFinite(intrinsics.fx) || intrinsics.fx <= 0
|
||||
|| !Number.isFinite(intrinsics.fy) || intrinsics.fy <= 0
|
||||
|| !Number.isFinite(intrinsics.cx)
|
||||
|| !Number.isFinite(intrinsics.cy)) {
|
||||
throw new Error('Invalid camera intrinsics');
|
||||
}
|
||||
assertPositiveInteger(intrinsics.imageWidth, 'intrinsics.imageWidth');
|
||||
assertPositiveInteger(intrinsics.imageHeight, 'intrinsics.imageHeight');
|
||||
}
|
||||
|
||||
function base64ToBytes(value, name) {
|
||||
if (typeof value !== 'string' || value.length === 0 || value.length % 4 !== 0
|
||||
|| !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) {
|
||||
throw new Error(`${name} must be canonical base64`);
|
||||
}
|
||||
|
||||
if (typeof Buffer !== 'undefined') {
|
||||
return Uint8Array.from(Buffer.from(value, 'base64'));
|
||||
}
|
||||
|
||||
const binary = atob(value);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i += 1) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
69
integrations/iphone-lidar/web/codec.test.mjs
Normal file
69
integrations/iphone-lidar/web/codec.test.mjs
Normal file
@@ -0,0 +1,69 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { decodeLiDARPacket, depthToPointCloud } from './codec.mjs';
|
||||
|
||||
function b64(bytes) {
|
||||
return Buffer.from(bytes).toString('base64');
|
||||
}
|
||||
|
||||
test('decodes u16 millimeter depth and confidence', () => {
|
||||
const packet = {
|
||||
type: 'ruview.lidar.depth.v1',
|
||||
intrinsics: { fx: 100, fy: 100, cx: 1, cy: 1, imageWidth: 2, imageHeight: 2 },
|
||||
pose: { matrix: Array(16).fill(0) },
|
||||
depth: {
|
||||
width: 2,
|
||||
height: 2,
|
||||
encoding: 'u16le-mm+u8-confidence',
|
||||
millimetersBase64: b64([0xe8,0x03,0xd0,0x07,0xb8,0x0b,0xa0,0x0f]),
|
||||
confidenceBase64: b64([2,2,1,0]),
|
||||
},
|
||||
provenance: { sensor: 'test', source: 'live', privacyClass: 'geometry-only', sequence: 1, timestampNs: 1, schema: 'ruview.lidar.depth.v1' },
|
||||
};
|
||||
|
||||
const frame = decodeLiDARPacket(packet);
|
||||
assert.deepEqual(Array.from(frame.depth.meters), [1,2,3,4]);
|
||||
assert.deepEqual(Array.from(frame.depth.confidence), [2,2,1,0]);
|
||||
});
|
||||
|
||||
test('rejects malformed payload length', () => {
|
||||
assert.throws(() => decodeLiDARPacket({
|
||||
type: 'ruview.lidar.depth.v1',
|
||||
intrinsics: { fx: 100, fy: 100, cx: 1, cy: 1, imageWidth: 2, imageHeight: 2 },
|
||||
pose: { matrix: Array(16).fill(0) },
|
||||
depth: { width: 2, height: 2, encoding: 'u16le-mm+u8-confidence', millimetersBase64: b64([1,2]), confidenceBase64: b64([1,1,1,1]) },
|
||||
}), /length mismatch/);
|
||||
});
|
||||
|
||||
test('rejects invalid dimensions, intrinsics, pose, and base64', () => {
|
||||
const valid = {
|
||||
type: 'ruview.lidar.depth.v1',
|
||||
intrinsics: { fx: 100, fy: 100, cx: 0, cy: 0, imageWidth: 1, imageHeight: 1 },
|
||||
pose: { matrix: Array(16).fill(0) },
|
||||
depth: {
|
||||
width: 1,
|
||||
height: 1,
|
||||
encoding: 'u16le-mm+u8-confidence',
|
||||
millimetersBase64: b64([0xe8, 0x03]),
|
||||
confidenceBase64: b64([2]),
|
||||
},
|
||||
};
|
||||
|
||||
assert.throws(() => decodeLiDARPacket({ ...valid, depth: { ...valid.depth, width: 0 } }), /positive integer/);
|
||||
assert.throws(() => decodeLiDARPacket({ ...valid, intrinsics: { ...valid.intrinsics, fx: 0 } }), /intrinsics/);
|
||||
assert.throws(() => decodeLiDARPacket({ ...valid, pose: { matrix: [1] } }), /pose/);
|
||||
assert.throws(() => decodeLiDARPacket({
|
||||
...valid,
|
||||
depth: { ...valid.depth, millimetersBase64: '!!!!' },
|
||||
}), /canonical base64/);
|
||||
});
|
||||
|
||||
test('projects depth into a point cloud and honors confidence', () => {
|
||||
const frame = {
|
||||
intrinsics: { fx: 100, fy: 100, cx: 0, cy: 0, imageWidth: 2, imageHeight: 2 },
|
||||
depth: { width: 2, height: 2, meters: Float32Array.from([1,1,1,1]), confidence: Uint8Array.from([2,0,2,0]) },
|
||||
};
|
||||
const points = depthToPointCloud(frame, 1);
|
||||
assert.equal(points.length, 2);
|
||||
assert.deepEqual(points[0], [0, 0, -1]);
|
||||
});
|
||||
36
integrations/iphone-lidar/web/index.html
Normal file
36
integrations/iphone-lidar/web/index.html
Normal file
@@ -0,0 +1,36 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1,viewport-fit=cover" />
|
||||
<title>RuView iPhone LiDAR</title>
|
||||
<link rel="stylesheet" href="./styles.css" />
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<header>
|
||||
<div>
|
||||
<p class="eyebrow">RUVIEW SENSOR BRIDGE</p>
|
||||
<h1>iPhone LiDAR</h1>
|
||||
</div>
|
||||
<span id="status">CONNECTING</span>
|
||||
</header>
|
||||
|
||||
<section class="metrics">
|
||||
<div><strong id="fps">0.0</strong><span>FPS</span></div>
|
||||
<div><strong id="points">0</strong><span>POINTS</span></div>
|
||||
<div><strong id="seq">0</strong><span>SEQ</span></div>
|
||||
<div><strong id="latency">0</strong><span>MS</span></div>
|
||||
</section>
|
||||
|
||||
<canvas id="view"></canvas>
|
||||
|
||||
<footer>
|
||||
<span>Geometry only</span>
|
||||
<span>No RGB upload</span>
|
||||
<span id="sensor">Waiting for sensor</span>
|
||||
</footer>
|
||||
</main>
|
||||
<script type="module" src="./app.mjs"></script>
|
||||
</body>
|
||||
</html>
|
||||
39
integrations/iphone-lidar/web/package-lock.json
generated
Normal file
39
integrations/iphone-lidar/web/package-lock.json
generated
Normal file
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"name": "@ruview/iphone-lidar-web",
|
||||
"version": "0.1.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@ruview/iphone-lidar-web",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"ws": "^8.18.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.21.3",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz",
|
||||
"integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": ">=5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
16
integrations/iphone-lidar/web/package.json
Normal file
16
integrations/iphone-lidar/web/package.json
Normal file
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"name": "@ruview/iphone-lidar-web",
|
||||
"private": true,
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node relay.mjs",
|
||||
"test": "node --test"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"dependencies": {
|
||||
"ws": "^8.18.3"
|
||||
}
|
||||
}
|
||||
166
integrations/iphone-lidar/web/relay.mjs
Normal file
166
integrations/iphone-lidar/web/relay.mjs
Normal file
@@ -0,0 +1,166 @@
|
||||
import { randomBytes, timingSafeEqual } from 'node:crypto';
|
||||
import http from 'node:http';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { extname, join } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { WebSocket, WebSocketServer } from 'ws';
|
||||
|
||||
const root = fileURLToPath(new URL('.', import.meta.url));
|
||||
const staticFiles = new Set(['index.html', 'app.mjs', 'codec.mjs', 'styles.css']);
|
||||
const maxPayloadBytes = 2_000_000;
|
||||
|
||||
function constantTimeEqual(left, right) {
|
||||
const leftBytes = Buffer.from(left, 'utf8');
|
||||
const rightBytes = Buffer.from(right, 'utf8');
|
||||
return leftBytes.length === rightBytes.length && timingSafeEqual(leftBytes, rightBytes);
|
||||
}
|
||||
|
||||
function rejectUpgrade(socket, status, message) {
|
||||
const body = `${message}\n`;
|
||||
socket.end(
|
||||
`HTTP/1.1 ${status}\r\nConnection: close\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: ${Buffer.byteLength(body)}\r\n\r\n${body}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function createLiDARRelay({ token, rootDirectory = root } = {}) {
|
||||
const accessToken = token || randomBytes(24).toString('hex');
|
||||
const clients = new Set();
|
||||
|
||||
const server = http.createServer(async (req, res) => {
|
||||
if (req.method !== 'GET' && req.method !== 'HEAD') {
|
||||
res.writeHead(405, { allow: 'GET, HEAD' }).end();
|
||||
return;
|
||||
}
|
||||
|
||||
let pathname;
|
||||
try {
|
||||
pathname = decodeURIComponent(new URL(req.url || '/', 'http://localhost').pathname);
|
||||
} catch {
|
||||
res.writeHead(400).end('bad path');
|
||||
return;
|
||||
}
|
||||
|
||||
const filename = pathname === '/' ? 'index.html' : pathname.slice(1);
|
||||
if (!staticFiles.has(filename)) {
|
||||
res.writeHead(404).end('not found');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await readFile(join(rootDirectory, filename));
|
||||
const contentType = {
|
||||
'.html': 'text/html; charset=utf-8',
|
||||
'.mjs': 'text/javascript; charset=utf-8',
|
||||
'.css': 'text/css; charset=utf-8',
|
||||
}[extname(filename)] || 'application/octet-stream';
|
||||
res.writeHead(200, {
|
||||
'content-type': contentType,
|
||||
'cache-control': 'no-store',
|
||||
'content-security-policy': "default-src 'self'; connect-src 'self' ws: wss:; img-src 'self'; style-src 'self'; base-uri 'none'; frame-ancestors 'none'",
|
||||
'referrer-policy': 'no-referrer',
|
||||
'x-content-type-options': 'nosniff',
|
||||
});
|
||||
if (req.method === 'HEAD') res.end();
|
||||
else res.end(data);
|
||||
} catch {
|
||||
res.writeHead(404).end('not found');
|
||||
}
|
||||
});
|
||||
|
||||
const wss = new WebSocketServer({ noServer: true, maxPayload: maxPayloadBytes });
|
||||
|
||||
server.on('upgrade', (req, socket, head) => {
|
||||
let url;
|
||||
try {
|
||||
url = new URL(req.url || '/', 'http://localhost');
|
||||
} catch {
|
||||
rejectUpgrade(socket, '400 Bad Request', 'bad websocket URL');
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname !== '/ws/lidar') {
|
||||
rejectUpgrade(socket, '404 Not Found', 'not found');
|
||||
return;
|
||||
}
|
||||
|
||||
const suppliedToken = url.searchParams.get('token') || '';
|
||||
if (!constantTimeEqual(suppliedToken, accessToken)) {
|
||||
rejectUpgrade(socket, '401 Unauthorized', 'valid LiDAR relay token required');
|
||||
return;
|
||||
}
|
||||
|
||||
wss.handleUpgrade(req, socket, head, (websocket) => {
|
||||
wss.emit('connection', websocket, req);
|
||||
});
|
||||
});
|
||||
|
||||
wss.on('connection', (socket) => {
|
||||
clients.add(socket);
|
||||
socket.on('close', () => clients.delete(socket));
|
||||
socket.on('error', () => socket.terminate());
|
||||
socket.on('message', (data, isBinary) => {
|
||||
if (isBinary) return;
|
||||
|
||||
let packet;
|
||||
try {
|
||||
packet = JSON.parse(data.toString());
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
if (packet?.type !== 'ruview.lidar.depth.v1') return;
|
||||
|
||||
for (const peer of clients) {
|
||||
if (peer !== socket && peer.readyState === WebSocket.OPEN) {
|
||||
peer.send(data.toString());
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return {
|
||||
accessToken,
|
||||
server,
|
||||
async listen(port = 0, host = '127.0.0.1') {
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once('error', reject);
|
||||
server.listen(port, host, () => {
|
||||
server.off('error', reject);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
return server.address();
|
||||
},
|
||||
async close() {
|
||||
for (const peer of clients) peer.terminate();
|
||||
await new Promise((resolve) => wss.close(resolve));
|
||||
if (server.listening) {
|
||||
await new Promise((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function configuredPort(value) {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > 65_535) {
|
||||
throw new Error(`PORT must be an integer from 1 to 65535; received ${value}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
const directInvocation = process.argv[1]
|
||||
&& pathToFileURL(process.argv[1]).href === import.meta.url;
|
||||
|
||||
if (directInvocation) {
|
||||
const port = configuredPort(process.env.PORT || '8787');
|
||||
const host = process.env.HOST || '0.0.0.0';
|
||||
const relay = createLiDARRelay({ token: process.env.RUVIEW_LIDAR_TOKEN });
|
||||
await relay.listen(port, host);
|
||||
const encodedToken = encodeURIComponent(relay.accessToken);
|
||||
console.log(`RuView iPhone LiDAR relay listening on ${host}:${port}`);
|
||||
console.log(`Browser: http://<host>:${port}/?token=${encodedToken}`);
|
||||
console.log(`Native endpoint: ws://<host>:${port}/ws/lidar?token=${encodedToken}`);
|
||||
}
|
||||
73
integrations/iphone-lidar/web/relay.test.mjs
Normal file
73
integrations/iphone-lidar/web/relay.test.mjs
Normal file
@@ -0,0 +1,73 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import http from 'node:http';
|
||||
import test from 'node:test';
|
||||
import { WebSocket } from 'ws';
|
||||
import { createLiDARRelay } from './relay.mjs';
|
||||
|
||||
function connect(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = new WebSocket(url);
|
||||
socket.once('open', () => resolve(socket));
|
||||
socket.once('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function request(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
http.get(url, (response) => {
|
||||
response.resume();
|
||||
response.once('end', () => resolve(response));
|
||||
}).once('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
test('relay requires a token, limits static files, and forwards LiDAR frames', async (t) => {
|
||||
const token = 'test-token-for-relay';
|
||||
const relay = createLiDARRelay({ token });
|
||||
const address = await relay.listen(0, '127.0.0.1');
|
||||
const httpBase = `http://127.0.0.1:${address.port}`;
|
||||
const wsBase = `ws://127.0.0.1:${address.port}/ws/lidar`;
|
||||
const sockets = [];
|
||||
|
||||
t.after(async () => {
|
||||
for (const socket of sockets) socket.terminate();
|
||||
await relay.close();
|
||||
});
|
||||
|
||||
const indexResponse = await request(`${httpBase}/?token=${token}`);
|
||||
assert.equal(indexResponse.statusCode, 200);
|
||||
assert.match(indexResponse.headers['content-security-policy'], /frame-ancestors 'none'/);
|
||||
|
||||
const sourceResponse = await request(`${httpBase}/relay.mjs`);
|
||||
assert.equal(sourceResponse.statusCode, 404);
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
const unauthorized = new WebSocket(wsBase);
|
||||
unauthorized.once('unexpected-response', (_request, response) => {
|
||||
assert.equal(response.statusCode, 401);
|
||||
response.resume();
|
||||
resolve();
|
||||
});
|
||||
unauthorized.once('open', () => reject(new Error('unauthorized websocket opened')));
|
||||
unauthorized.once('error', () => {});
|
||||
});
|
||||
|
||||
const sender = await connect(`${wsBase}?token=${encodeURIComponent(token)}`);
|
||||
const receiver = await connect(`${wsBase}?token=${encodeURIComponent(token)}`);
|
||||
sockets.push(sender, receiver);
|
||||
|
||||
const received = new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error('timed out waiting for relayed frame')), 2_000);
|
||||
receiver.once('message', (data, isBinary) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ data, isBinary });
|
||||
});
|
||||
});
|
||||
|
||||
const frame = { type: 'ruview.lidar.depth.v1', provenance: { sequence: 7 } };
|
||||
sender.send(JSON.stringify(frame));
|
||||
const message = await received;
|
||||
|
||||
assert.equal(message.isBinary, false);
|
||||
assert.deepEqual(JSON.parse(message.data.toString()), frame);
|
||||
});
|
||||
1
integrations/iphone-lidar/web/styles.css
Normal file
1
integrations/iphone-lidar/web/styles.css
Normal file
@@ -0,0 +1 @@
|
||||
*{box-sizing:border-box}body{margin:0;background:#05080d;color:#e8f1f5;font-family:ui-monospace,SFMono-Regular,Menlo,monospace}main{min-height:100vh;padding:18px;display:grid;grid-template-rows:auto auto 1fr auto;gap:14px}header{display:flex;align-items:end;justify-content:space-between}h1{margin:0;font-size:clamp(28px,6vw,54px)}.eyebrow{margin:0 0 6px;color:#58e0d2;font-size:11px;letter-spacing:.16em}#status{border:1px solid #2a3946;border-radius:999px;padding:7px 11px;font-size:11px}#status[data-state=live]{color:#58e0d2;border-color:#58e0d2}.metrics{display:grid;grid-template-columns:repeat(4,1fr);gap:8px}.metrics div{background:#0b1219;border:1px solid #17222d;border-radius:10px;padding:10px}.metrics strong{display:block;font-size:18px}.metrics span,footer{font-size:10px;color:#8aa0af}canvas{width:100%;height:100%;min-height:55vh;border-radius:14px;border:1px solid #17222d;background:#091018;touch-action:none}footer{display:flex;gap:16px;flex-wrap:wrap}@media(max-width:640px){.metrics{grid-template-columns:repeat(2,1fr)}canvas{min-height:58vh}}
|
||||
@@ -1,13 +1,9 @@
|
||||
# cargo-audit configuration — v2 workspace
|
||||
# Managed by security audit (fix/security-audit-rustsec-clippy branch).
|
||||
#
|
||||
# This file suppresses advisories in two categories:
|
||||
# A) CVE-bearing advisories in TRANSITIVE deps we cannot upgrade directly
|
||||
# because the parent published crate (ruvector-core 2.2.0) has not yet
|
||||
# published a version with the fix. These are tracked as issues.
|
||||
# B) UNMAINTAINED-only advisories (no CVE) flowing through dependencies
|
||||
# that are purely transitive / build-time and have no user-facing attack
|
||||
# surface in this workspace.
|
||||
# This file suppresses UNMAINTAINED-only advisories (no CVE) flowing through
|
||||
# dependencies that are purely transitive / build-time and have no
|
||||
# user-facing attack surface in this workspace.
|
||||
# Each entry documents the root cause and the mitigation path.
|
||||
|
||||
[advisories]
|
||||
@@ -24,26 +20,6 @@
|
||||
# Mitigation: Accept transitively until Tauri v2 drops GTK3 or a workspace
|
||||
# override path becomes available.
|
||||
ignore = [
|
||||
# -----------------------------------------------------------------------
|
||||
# CATEGORY A — transitive CVEs from ruvector-core 2.2.0 → reqwest 0.11
|
||||
# ruvector-core 2.2.0 (latest on crates.io) depends on reqwest 0.11.27,
|
||||
# which pulls in rustls 0.21 / rustls-webpki 0.101.7. We cannot upgrade
|
||||
# this without a new ruvector-core release. Tracked in issue #812.
|
||||
# The workspace's own TLS stack uses rustls-webpki 0.103.13 (patched);
|
||||
# the vulnerable 0.101.7 instance is not reachable from our TLS code.
|
||||
"RUSTSEC-2026-0098", # rustls-webpki 0.101.7: URI name constraint bypass
|
||||
"RUSTSEC-2026-0099", # rustls-webpki 0.101.7: wildcard name constraint bypass
|
||||
"RUSTSEC-2026-0104", # rustls-webpki 0.101.7: reachable panic in CRL parsing
|
||||
# quinn-proto 0.11.13 is also pulled through midstreamer-quic 0.3 (now
|
||||
# upgraded). The remaining 0.11.13 instance comes from the same
|
||||
# ruvector-core transitive chain. Tracked in issue #812.
|
||||
"RUSTSEC-2026-0037", # quinn-proto 0.11.13: DoS in Quinn endpoints
|
||||
# CRL Distribution Point matching bug — same ruvector-core / reqwest 0.11
|
||||
# transitive chain; rustls-webpki 0.101.7 also affected.
|
||||
"RUSTSEC-2026-0049", # rustls-webpki <0.103.10: CRL authority matching
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# CATEGORY B — unmaintained / no CVE
|
||||
"RUSTSEC-2024-0411", # gdkwayland-sys: unmaintained
|
||||
"RUSTSEC-2024-0412", # gdk: unmaintained
|
||||
"RUSTSEC-2024-0413", # atk: unmaintained
|
||||
|
||||
987
v2/Cargo.lock
generated
987
v2/Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -255,7 +255,7 @@ midstreamer-attractor = "0.2"
|
||||
# ruvector integration (published on crates.io)
|
||||
# Vendored at origin/main (a083bd77f) in vendor/ruvector; using crates.io versions
|
||||
# until published. Bumps per ADR-152 §2.6 (2026-06-10 vendor sync survey).
|
||||
ruvector-core = "2.2.0"
|
||||
ruvector-core = "2.3.0"
|
||||
ruvector-mincut = "2.0.6"
|
||||
ruvector-attn-mincut = "2.0.4"
|
||||
ruvector-temporal-tensor = "2.0.6"
|
||||
|
||||
@@ -31,10 +31,16 @@ homecore = { path = "../homecore", version = "0.1.0-alpha.0" }
|
||||
# Async runtime
|
||||
tokio = { version = "1", features = ["sync", "rt", "rt-multi-thread", "time", "macros"] }
|
||||
|
||||
# SQLite via sqlx — only the lite feature set; no postgres, no tls
|
||||
sqlx = { version = "0.8.1", default-features = false, features = [
|
||||
"runtime-tokio-native-tls",
|
||||
"sqlite",
|
||||
# SQLite-only SQLx crates, pinned in lockstep because their direct APIs are
|
||||
# semver-exempt. Depending on the umbrella `sqlx` package also resolves its
|
||||
# unused MySQL backend (and vulnerable `rsa`) into Cargo.lock.
|
||||
sqlx-core = { version = "=0.8.6", default-features = false, features = [
|
||||
"_rt-tokio",
|
||||
"chrono",
|
||||
"uuid",
|
||||
] }
|
||||
sqlx-sqlite = { version = "=0.8.6", default-features = false, features = [
|
||||
"bundled",
|
||||
"chrono",
|
||||
"uuid",
|
||||
] }
|
||||
|
||||
@@ -26,6 +26,19 @@ use homecore::StateMachine;
|
||||
use crate::dedup::fnv64a_hash;
|
||||
use crate::schema::ALL_DDL;
|
||||
|
||||
// Preserve the narrow `sqlx::*` call surface used in this module while
|
||||
// depending only on SQLx core + SQLite. The umbrella crate resolves unused
|
||||
// database backends into Cargo.lock, including MySQL's vulnerable RSA stack.
|
||||
mod sqlx {
|
||||
pub use sqlx_core::error::Error;
|
||||
pub use sqlx_core::query::query;
|
||||
pub use sqlx_core::query_as::query_as;
|
||||
|
||||
pub mod sqlite {
|
||||
pub use sqlx_sqlite::{SqliteConnectOptions, SqlitePool, SqlitePoolOptions};
|
||||
}
|
||||
}
|
||||
|
||||
type SearchStateRecord = (
|
||||
i64,
|
||||
String,
|
||||
|
||||
@@ -50,7 +50,7 @@ tower-http = { version = "0.6", features = ["fs", "trace", "cors"] }
|
||||
# requires every crate that pulls reqwest to align on rustls-only (tracked in
|
||||
# CHANGELOG / ADR-131 security note).
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde = { version = "1", features = ["derive", "rc"] }
|
||||
serde_yaml = "0.9"
|
||||
# Concurrent fan-out of per-bank RoomState fetches in the gateway (§11 perf).
|
||||
futures = "0.3"
|
||||
|
||||
Submodule v2/crates/ruview-swarm updated: 267aba5be2...5cc4b8625f
@@ -39,7 +39,7 @@ serde = { workspace = true, features = ["derive"], optional = true }
|
||||
serde_json = { workspace = true, optional = true }
|
||||
# MQTT publisher backend (optional). Matches the `rumqttc` choice already in
|
||||
# `wifi-densepose-sensing-server` so both crates share TLS / version posture.
|
||||
rumqttc = { version = "0.24", default-features = false, features = ["use-rustls"], optional = true }
|
||||
rumqttc = { package = "rumqttc-v4-next", version = "0.34", default-features = false, features = ["use-rustls-ring"], optional = true }
|
||||
wifi-veil = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -31,7 +31,7 @@ use crate::PrivacyClass;
|
||||
/// };
|
||||
/// use rumqttc::MqttOptions;
|
||||
///
|
||||
/// let opts = MqttOptions::new("seed-01", "broker.local", 1883);
|
||||
/// let opts = MqttOptions::new("seed-01", ("broker.local", 1883));
|
||||
/// let (retained_pub, _conn) = RumqttPublisher::connect(opts.clone(), 64);
|
||||
/// let mut retained_pub = retained_pub.with_retain(true);
|
||||
/// publish_discovery(&mut retained_pub, "seed-01", PrivacyClass::Anonymous)?;
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
//! use wifi_densepose_bfld::{publish_event, RumqttPublisher};
|
||||
//! use rumqttc::MqttOptions;
|
||||
//!
|
||||
//! let opts = MqttOptions::new("seed-01", "broker.local", 1883);
|
||||
//! let opts = MqttOptions::new("seed-01", ("broker.local", 1883));
|
||||
//! let (mut publisher, mut connection) = RumqttPublisher::connect(opts, 100);
|
||||
//! thread::spawn(move || for _ in connection.iter() { /* drain */ });
|
||||
//! // ... build BfldEvent ...
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
#![cfg(feature = "mqtt")]
|
||||
|
||||
use rumqttc::{Client, Connection, LastWill, MqttOptions, QoS};
|
||||
use rumqttc::{Client, Connection, LastWill, MqttOptions, PublishOptions, QoS};
|
||||
|
||||
use crate::availability::{availability_topic, PAYLOAD_NOT_AVAILABLE};
|
||||
use crate::mqtt_topics::{Publish, TopicMessage};
|
||||
@@ -60,7 +60,7 @@ impl RumqttPublisher {
|
||||
/// shown in the module-level doc example).
|
||||
#[must_use]
|
||||
pub fn connect(opts: MqttOptions, capacity: usize) -> (Self, Connection) {
|
||||
let (client, connection) = Client::new(opts, capacity);
|
||||
let (client, connection) = Client::builder(opts).capacity(capacity).build();
|
||||
(Self::new(client, QoS::AtLeastOnce), connection)
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ impl RumqttPublisher {
|
||||
/// opt in to the LWT without using `connect_with_lwt`.
|
||||
#[must_use]
|
||||
pub fn with_lwt(mut opts: MqttOptions, node_id: &str) -> MqttOptions {
|
||||
// rumqttc 0.24 LastWill::new takes (topic, message, qos, retain).
|
||||
// LastWill::new takes (topic, message, qos, retain).
|
||||
// retain = true so HA sees "offline" on next start even if the session
|
||||
// dropped while HA was down.
|
||||
let will = LastWill::new(
|
||||
@@ -105,6 +105,10 @@ impl Publish for RumqttPublisher {
|
||||
|
||||
fn publish(&mut self, msg: &TopicMessage) -> Result<(), Self::Error> {
|
||||
self.client
|
||||
.publish(&msg.topic, self.qos, self.retain, msg.payload.as_bytes())
|
||||
.publish(
|
||||
&msg.topic,
|
||||
msg.payload.as_bytes(),
|
||||
PublishOptions::new(self.qos).retain(self.retain),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,9 +23,7 @@ use std::thread;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use rumqttc::{Client, Event, Incoming, MqttOptions, Packet, QoS};
|
||||
use wifi_densepose_bfld::{
|
||||
publish_event, BfldEvent, PrivacyClass, RumqttPublisher,
|
||||
};
|
||||
use wifi_densepose_bfld::{publish_event, BfldEvent, PrivacyClass, RumqttPublisher};
|
||||
|
||||
const SUBSCRIBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const RECEIVE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
@@ -69,9 +67,9 @@ fn spawn_subscriber(
|
||||
port: u16,
|
||||
topic_filter: &str,
|
||||
) -> (Receiver<(String, String)>, Receiver<()>) {
|
||||
let mut opts = MqttOptions::new(unique_client_id("bfld-sub"), host, port);
|
||||
opts.set_keep_alive(Duration::from_secs(5));
|
||||
let (client, mut connection) = Client::new(opts, 64);
|
||||
let mut opts = MqttOptions::new(unique_client_id("bfld-sub"), (host, port));
|
||||
opts.set_keep_alive(5);
|
||||
let (client, mut connection) = Client::builder(opts).capacity(64).build();
|
||||
client
|
||||
.subscribe(topic_filter, QoS::AtLeastOnce)
|
||||
.expect("subscribe enqueue");
|
||||
@@ -79,13 +77,18 @@ fn spawn_subscriber(
|
||||
let (incoming_tx, incoming_rx) = channel();
|
||||
let (suback_tx, suback_rx) = channel();
|
||||
thread::spawn(move || {
|
||||
// rumqttc-v4-next stops the connection once every request sender is
|
||||
// dropped. Keep the subscriber client alive for as long as its pump
|
||||
// thread runs; otherwise the broker sees a clean disconnect directly
|
||||
// after SUBACK and no subsequent publications can be delivered.
|
||||
let _client_guard = client;
|
||||
for notification in connection.iter() {
|
||||
match notification {
|
||||
Ok(Event::Incoming(Packet::SubAck(_))) => {
|
||||
let _ = suback_tx.send(());
|
||||
}
|
||||
Ok(Event::Incoming(Incoming::Publish(p))) => {
|
||||
let topic = p.topic.clone();
|
||||
let topic = String::from_utf8_lossy(&p.topic).to_string();
|
||||
let payload = String::from_utf8_lossy(&p.payload).to_string();
|
||||
if incoming_tx.send((topic, payload)).is_err() {
|
||||
break;
|
||||
@@ -141,8 +144,8 @@ fn live_broker_anonymous_event_roundtrips_all_six_topics() {
|
||||
|
||||
// Publisher with its own connection. Spawn a thread iterating the
|
||||
// Connection so publishes actually reach the broker.
|
||||
let mut opts = MqttOptions::new(unique_client_id("bfld-pub"), &host, port);
|
||||
opts.set_keep_alive(Duration::from_secs(5));
|
||||
let mut opts = MqttOptions::new(unique_client_id("bfld-pub"), (host.as_str(), port));
|
||||
opts.set_keep_alive(5);
|
||||
let (mut publisher, mut pub_connection) = RumqttPublisher::connect(opts, 64);
|
||||
thread::spawn(move || {
|
||||
for _ in pub_connection.iter() { /* drain protocol events */ }
|
||||
@@ -197,8 +200,8 @@ fn live_broker_restricted_event_omits_identity_risk() {
|
||||
.recv_timeout(SUBSCRIBE_TIMEOUT)
|
||||
.expect("SubAck within 5s");
|
||||
|
||||
let mut opts = MqttOptions::new(unique_client_id("bfld-pub-r"), &host, port);
|
||||
opts.set_keep_alive(Duration::from_secs(5));
|
||||
let mut opts = MqttOptions::new(unique_client_id("bfld-pub-r"), (host.as_str(), port));
|
||||
opts.set_keep_alive(5);
|
||||
let (mut publisher, mut pub_connection) = RumqttPublisher::connect(opts, 64);
|
||||
thread::spawn(move || for _ in pub_connection.iter() {});
|
||||
thread::sleep(Duration::from_millis(200));
|
||||
|
||||
@@ -9,7 +9,7 @@ use wifi_densepose_bfld::{
|
||||
};
|
||||
|
||||
fn unreachable_opts(client_id: &str) -> MqttOptions {
|
||||
MqttOptions::new(client_id, "127.0.0.1", 1)
|
||||
MqttOptions::new(client_id, ("127.0.0.1", 1))
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -89,7 +89,7 @@ fn caller_built_options_can_opt_in_via_with_lwt_then_pass_to_connect() {
|
||||
// Operators with custom MqttOptions (e.g., TLS, credentials) build their
|
||||
// own opts, then call with_lwt before passing to RumqttPublisher::connect.
|
||||
let mut opts = unreachable_opts("bfld-lwt-6");
|
||||
opts.set_keep_alive(std::time::Duration::from_secs(30));
|
||||
opts.set_keep_alive(30);
|
||||
let opts = with_lwt(opts, "seed-01");
|
||||
let (_publisher, _connection) = RumqttPublisher::connect(opts, 16);
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ use wifi_densepose_bfld::{publish_event, BfldEvent, PrivacyClass, Publish, Rumqt
|
||||
fn unreachable_opts() -> MqttOptions {
|
||||
// Port 1 is reserved (RFC 1700) and the loopback address will refuse
|
||||
// immediately — perfect for a construction smoke test that must not block.
|
||||
MqttOptions::new("bfld-smoke-iter23", "127.0.0.1", 1)
|
||||
MqttOptions::new("bfld-smoke-iter23", ("127.0.0.1", 1))
|
||||
}
|
||||
|
||||
fn sample_event() -> BfldEvent {
|
||||
|
||||
@@ -110,7 +110,7 @@ rand = "0.8"
|
||||
# client (ADR-115 §10 references). `rustls` is preferred over openssl on
|
||||
# Windows to keep parity with the rest of the workspace (`ureq` above also
|
||||
# uses rustls).
|
||||
rumqttc = { version = "0.24", default-features = false, features = ["use-rustls"], optional = true }
|
||||
rumqttc = { package = "rumqttc-v4-next", version = "0.34", default-features = false, features = ["use-rustls-ring"], optional = true }
|
||||
|
||||
# `otel` feature — OTLP log export (`telemetry` module). Same gating
|
||||
# principle as `mqtt`: the heavy exporter stack (opentelemetry SDK +
|
||||
|
||||
@@ -286,6 +286,9 @@ struct Esp32Frame {
|
||||
/// ADR-110 byte 18: PPDU type the CSI was sampled from. Pre-ADR-110
|
||||
/// firmware sends 0 ⇒ `PpduType::HtLegacy`.
|
||||
ppdu_type: wifi_densepose_hardware::PpduType,
|
||||
/// ADR-110 byte 19 metadata, including whether this frame was captured
|
||||
/// while the node had a valid IEEE 802.15.4 mesh-time solution.
|
||||
adr018_flags: wifi_densepose_hardware::Adr018Flags,
|
||||
amplitudes: Vec<f64>,
|
||||
phases: Vec<f64>,
|
||||
}
|
||||
@@ -675,6 +678,12 @@ struct NodeState {
|
||||
latest_sync: Option<wifi_densepose_hardware::SyncPacket>,
|
||||
/// Last time a sync packet from this node was received (for staleness).
|
||||
latest_sync_at: Option<std::time::Instant>,
|
||||
/// Sequence number of the newest CSI frame admitted to `frame_history`.
|
||||
/// Kept alongside the history so multistatic fusion can timestamp the
|
||||
/// exact sample it consumes, rather than the host's UDP arrival time.
|
||||
latest_csi_sequence: Option<u32>,
|
||||
/// Whether byte 19 bit 4 marked that newest admitted CSI frame as synced.
|
||||
latest_csi_sync_valid: bool,
|
||||
/// ADR-110 iter 18: EMA-tracked CSI frame rate for this node.
|
||||
/// Replaces the hardcoded 20 Hz fallback in
|
||||
/// `mesh_aligned_us_for_csi_frame` once `csi_fps_samples ≥ 5`.
|
||||
@@ -832,6 +841,9 @@ impl NodeState {
|
||||
/// staleness gate).
|
||||
pub(crate) fn mesh_aligned_us(&self, local_at_frame_us: u64) -> Option<u64> {
|
||||
let sync = self.latest_sync.as_ref()?;
|
||||
if !sync.flags.is_valid {
|
||||
return None;
|
||||
}
|
||||
let seen_at = self.latest_sync_at?;
|
||||
// Drop stale syncs — firmware emits at ~0.5 Hz default, anything
|
||||
// older than 9 s likely means the mesh transport dropped.
|
||||
@@ -850,10 +862,20 @@ impl NodeState {
|
||||
/// no fresh sync has been observed for this node.
|
||||
pub(crate) fn mesh_aligned_us_for_csi_frame(&self, frame_sequence: u32) -> Option<u64> {
|
||||
let sync = self.latest_sync.as_ref()?;
|
||||
if !sync.flags.is_valid {
|
||||
return None;
|
||||
}
|
||||
let seen_at = self.latest_sync_at?;
|
||||
if seen_at.elapsed() > std::time::Duration::from_secs(9) {
|
||||
return None;
|
||||
}
|
||||
// A recently-received sync datagram can overtake an older CSI
|
||||
// datagram in UDP delivery order. Only extrapolate forward (including
|
||||
// a genuine u32 wrap); otherwise fall back to host arrival time.
|
||||
let delta_frames = frame_sequence.wrapping_sub(sync.sequence);
|
||||
if delta_frames > i32::MAX as u32 {
|
||||
return None;
|
||||
}
|
||||
// Iter 18: use the measured per-node fps once we have ≥5 inter-frame
|
||||
// samples; until then fall back to the 20 Hz firmware ceiling. The
|
||||
// §A0.12 capture showed real bench fps ≈ 10, so the measured value
|
||||
@@ -862,6 +884,16 @@ impl NodeState {
|
||||
Some(sync.mesh_aligned_us_for_sequence(frame_sequence, fps))
|
||||
}
|
||||
|
||||
/// Mesh timestamp for the newest CSI frame admitted to `frame_history`.
|
||||
/// Both the frame-level sync-valid bit and a fresh, valid sync packet are
|
||||
/// required; callers retain their existing host-arrival fallback.
|
||||
pub(crate) fn mesh_aligned_us_for_latest_csi_frame(&self) -> Option<u64> {
|
||||
if !self.latest_csi_sync_valid {
|
||||
return None;
|
||||
}
|
||||
self.mesh_aligned_us_for_csi_frame(self.latest_csi_sequence?)
|
||||
}
|
||||
|
||||
/// ADR-110 iter 18 — update the per-node observed-fps EMA from a fresh
|
||||
/// CSI frame arrival. Call once per accepted CSI frame from
|
||||
/// `udp_receiver_task`. Uses `last_frame_time` as the previous-frame
|
||||
@@ -927,6 +959,21 @@ impl NodeState {
|
||||
first_sensing_frame
|
||||
}
|
||||
|
||||
/// Record an accepted CSI sample and preserve the wire metadata needed by
|
||||
/// the multistatic bridge to recover capture time. Grid-rejected frames
|
||||
/// intentionally use `observe_csi_frame_arrival` directly because they do
|
||||
/// not replace the sample at the back of `frame_history`.
|
||||
pub(crate) fn observe_accepted_csi_frame(
|
||||
&mut self,
|
||||
sequence: u32,
|
||||
sync_valid: bool,
|
||||
now: std::time::Instant,
|
||||
) -> bool {
|
||||
self.latest_csi_sequence = Some(sequence);
|
||||
self.latest_csi_sync_valid = sync_valid;
|
||||
self.observe_csi_frame_arrival(now)
|
||||
}
|
||||
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
frame_history: VecDeque::new(),
|
||||
@@ -951,6 +998,8 @@ impl NodeState {
|
||||
edge_vitals: None,
|
||||
latest_sync: None,
|
||||
latest_sync_at: None,
|
||||
latest_csi_sequence: None,
|
||||
latest_csi_sync_valid: false,
|
||||
csi_fps_ema: 20.0,
|
||||
csi_fps_samples: 0,
|
||||
latest_features: None,
|
||||
@@ -1945,7 +1994,8 @@ fn parse_esp32_frame(buf: &[u8]) -> Option<Esp32Frame> {
|
||||
// [12..15] sequence (u32 LE)
|
||||
// [16] rssi (i8)
|
||||
// [17] noise_floor (i8)
|
||||
// [18..19] reserved
|
||||
// [18] PPDU type
|
||||
// [19] ADR-018 flags (bit 4 = IEEE 802.15.4 sync valid)
|
||||
// [20..] I/Q data
|
||||
// Issue #1005: until 2026-06 this code read n_subcarriers from byte 6
|
||||
// alone (an ESP32-C6 HE-SU frame's 256 = 0x0100 LE decoded as 0 — the
|
||||
@@ -1966,6 +2016,7 @@ fn parse_esp32_frame(buf: &[u8]) -> Option<Esp32Frame> {
|
||||
};
|
||||
let noise_floor = buf[17] as i8;
|
||||
let ppdu_type = wifi_densepose_hardware::PpduType::from_byte(buf[18]);
|
||||
let adr018_flags = wifi_densepose_hardware::Adr018Flags::from_byte(buf[19]);
|
||||
|
||||
let iq_start = 20;
|
||||
let n_pairs = n_antennas as usize * n_subcarriers as usize;
|
||||
@@ -1995,6 +2046,7 @@ fn parse_esp32_frame(buf: &[u8]) -> Option<Esp32Frame> {
|
||||
rssi,
|
||||
noise_floor,
|
||||
ppdu_type,
|
||||
adr018_flags,
|
||||
amplitudes,
|
||||
phases,
|
||||
})
|
||||
@@ -2024,7 +2076,7 @@ mod issue_1009_n_subcarriers_u16_tests {
|
||||
buf[16] = (-40i8) as u8; // rssi
|
||||
buf[17] = (-90i8) as u8; // noise_floor
|
||||
buf[18] = 0; // ppdu_type
|
||||
buf[19] = 0;
|
||||
buf[19] = 0x10; // ADR-018: IEEE 802.15.4 sync valid
|
||||
for k in 0..n_subcarriers as usize {
|
||||
buf[20 + k * 2] = (5 + (k % 40) as i8) as u8; // i
|
||||
buf[20 + k * 2 + 1] = (k % 30) as u8; // q
|
||||
@@ -2047,6 +2099,7 @@ mod issue_1009_n_subcarriers_u16_tests {
|
||||
assert_eq!(frame.node_id, 7);
|
||||
assert_eq!(frame.rssi, -40);
|
||||
assert_eq!(frame.sequence, 42);
|
||||
assert!(frame.adr018_flags.ieee802154_sync_valid);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2941,6 +2994,7 @@ async fn windows_wifi_task(state: SharedState, tick_ms: u64) {
|
||||
rssi: first_rssi.clamp(-128.0, 127.0) as i8,
|
||||
noise_floor: -90,
|
||||
ppdu_type: wifi_densepose_hardware::PpduType::HtLegacy,
|
||||
adr018_flags: wifi_densepose_hardware::Adr018Flags::default(),
|
||||
amplitudes: multi_ap_frame.amplitudes.clone(),
|
||||
phases: multi_ap_frame.phases.clone(),
|
||||
};
|
||||
@@ -3129,6 +3183,7 @@ async fn windows_wifi_fallback_tick(state: &SharedState, seq: u32) {
|
||||
rssi: rssi_dbm as i8,
|
||||
noise_floor: -90,
|
||||
ppdu_type: wifi_densepose_hardware::PpduType::HtLegacy,
|
||||
adr018_flags: wifi_densepose_hardware::Adr018Flags::default(),
|
||||
amplitudes: vec![signal_pct],
|
||||
phases: vec![0.0],
|
||||
};
|
||||
@@ -3504,6 +3559,7 @@ fn generate_simulated_frame(tick: u64) -> Esp32Frame {
|
||||
rssi: (-40.0 + 5.0 * (t * 0.2).sin()) as i8,
|
||||
noise_floor: -90,
|
||||
ppdu_type: wifi_densepose_hardware::PpduType::HtLegacy,
|
||||
adr018_flags: wifi_densepose_hardware::Adr018Flags::default(),
|
||||
amplitudes,
|
||||
phases,
|
||||
}
|
||||
@@ -6702,8 +6758,11 @@ async fn udp_receiver_task(
|
||||
// ADR-110 iter 19 — feed the per-node fps EMA from real
|
||||
// CSI arrivals. The helper sets `last_frame_time` as a
|
||||
// side effect, so the previous bare assignment is gone.
|
||||
let first_sensing_frame =
|
||||
ns.observe_csi_frame_arrival(std::time::Instant::now());
|
||||
let first_sensing_frame = ns.observe_accepted_csi_frame(
|
||||
frame.sequence,
|
||||
frame.adr018_flags.ieee802154_sync_valid,
|
||||
std::time::Instant::now(),
|
||||
);
|
||||
if first_sensing_frame && telemetry::curated_events_enabled() {
|
||||
info!(name: semconv::EVENT_RUVIEW_NODE_ONLINE, { "ruview.node.id" = node_id }, "node {node_id} online (CSI)");
|
||||
}
|
||||
@@ -9327,6 +9386,31 @@ mod sync_snapshot_helper_tests {
|
||||
"10 s old sync must trigger the 9 s staleness gate");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn latest_csi_mesh_time_requires_both_validity_signals() {
|
||||
let now = std::time::Instant::now();
|
||||
let mut ns = NodeState::new();
|
||||
ns.apply_sync_packet(populated_sync(9), now);
|
||||
|
||||
ns.observe_accepted_csi_frame(21, false, now);
|
||||
assert!(
|
||||
ns.mesh_aligned_us_for_latest_csi_frame().is_none(),
|
||||
"an unsynchronized CSI capture must use the host-time fallback"
|
||||
);
|
||||
|
||||
ns.observe_accepted_csi_frame(21, true, now + std::time::Duration::from_millis(50));
|
||||
assert_eq!(
|
||||
ns.mesh_aligned_us_for_latest_csi_frame(),
|
||||
Some(27_684_885)
|
||||
);
|
||||
|
||||
ns.latest_sync.as_mut().unwrap().flags.is_valid = false;
|
||||
assert!(
|
||||
ns.mesh_aligned_us_for_latest_csi_frame().is_none(),
|
||||
"an invalid sync packet must not timestamp even a flagged CSI frame"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_reflects_leader_state() {
|
||||
// Same data shape that /api/v1/mesh emits for a leader node.
|
||||
|
||||
@@ -30,7 +30,10 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use rumqttc::{AsyncClient, ClientError, EventLoop, MqttOptions, QoS, Transport, TlsConfiguration};
|
||||
use rumqttc::{
|
||||
AsyncClient, ClientError, EventLoop, MqttOptions, PublishOptions, QoS, Transport,
|
||||
TlsConfiguration,
|
||||
};
|
||||
use tokio::sync::broadcast;
|
||||
use tokio::task::JoinHandle;
|
||||
use tracing::{error, info, warn};
|
||||
@@ -70,14 +73,14 @@ const NODE_SNAPSHOT_STALE_AFTER: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Build a `rumqttc::MqttOptions` from validated [`MqttConfig`].
|
||||
fn build_mqtt_options(cfg: &MqttConfig) -> MqttOptions {
|
||||
let mut opts = MqttOptions::new(&cfg.client_id, &cfg.host, cfg.port);
|
||||
opts.set_keep_alive(Duration::from_secs(30));
|
||||
let mut opts = MqttOptions::new(&cfg.client_id, (cfg.host.as_str(), cfg.port));
|
||||
opts.set_keep_alive(30);
|
||||
opts.set_clean_session(true);
|
||||
|
||||
if let (Some(u), Some(p)) = (cfg.username.as_deref(), cfg.password.as_deref()) {
|
||||
opts.set_credentials(u, p);
|
||||
opts.set_credentials(u.to_owned(), p.as_bytes().to_vec());
|
||||
} else if let Some(u) = cfg.username.as_deref() {
|
||||
opts.set_credentials(u, "");
|
||||
opts.set_credentials(u.to_owned(), Vec::<u8>::new());
|
||||
}
|
||||
|
||||
opts.set_transport(build_transport(&cfg.tls));
|
||||
@@ -223,7 +226,8 @@ async fn run(
|
||||
mut state_rx: broadcast::Receiver<VitalsSnapshot>,
|
||||
) {
|
||||
let opts = build_mqtt_options(&cfg);
|
||||
let (client, mut eventloop): (AsyncClient, EventLoop) = AsyncClient::new(opts, 256);
|
||||
let (client, mut eventloop): (AsyncClient, EventLoop) =
|
||||
AsyncClient::builder(opts).capacity(256).build();
|
||||
|
||||
let entities = DiscoveryBuilder::enabled_entities(
|
||||
cfg.privacy_mode,
|
||||
@@ -369,7 +373,13 @@ async fn publish_all_discovery(
|
||||
let cfg = b.build(e);
|
||||
let topic = b.config_topic(e);
|
||||
let payload = serde_json::to_string(&cfg).expect("discovery payload always serialises");
|
||||
client.publish(&topic, QoS::AtLeastOnce, true, payload).await?;
|
||||
client
|
||||
.publish(
|
||||
&topic,
|
||||
payload,
|
||||
PublishOptions::new(QoS::AtLeastOnce).retained(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -380,7 +390,13 @@ async fn publish_availability(
|
||||
state: &str,
|
||||
) -> Result<(), ClientError> {
|
||||
for topic in &avail.online_topics {
|
||||
client.publish(topic, QoS::AtLeastOnce, true, state).await?;
|
||||
client
|
||||
.publish(
|
||||
topic,
|
||||
state,
|
||||
PublishOptions::new(QoS::AtLeastOnce).retained(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -441,7 +457,13 @@ async fn publish_state(client: &AsyncClient, m: &StateMessage) -> Result<(), Cli
|
||||
1 => QoS::AtLeastOnce,
|
||||
_ => QoS::ExactlyOnce,
|
||||
};
|
||||
client.publish(&m.topic, qos, m.retain, m.payload.clone()).await
|
||||
client
|
||||
.publish(
|
||||
&m.topic,
|
||||
m.payload.clone(),
|
||||
PublishOptions::new(qos).retain(m.retain),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -24,7 +24,13 @@ const DEFAULT_FREQ_MHZ: u32 = 2437; // Channel 6
|
||||
|
||||
/// Monotonic reference point for timestamp generation. All node timestamps
|
||||
/// are relative to this instant, avoiding wall-clock/monotonic mixing issues.
|
||||
static EPOCH: LazyLock<Instant> = LazyLock::new(Instant::now);
|
||||
/// Backdate the lazy initialization beyond the active-node window so frames
|
||||
/// recorded just before the first bridge call retain their arrival-time skew.
|
||||
static EPOCH: LazyLock<Instant> = LazyLock::new(|| {
|
||||
Instant::now()
|
||||
.checked_sub(STALE_THRESHOLD + STALE_THRESHOLD)
|
||||
.unwrap_or_else(Instant::now)
|
||||
});
|
||||
|
||||
/// Shared length-only canonicalizer (issue #1170). The default 56-tone grid
|
||||
/// matches what `MultistaticFuser` (ADR-154) expects. Stateless and immutable,
|
||||
@@ -54,10 +60,18 @@ pub fn node_frame_from_state(node_id: u8, ns: &NodeState) -> Option<MultiBandCsi
|
||||
let n_sub = amplitude.len();
|
||||
let phase = vec![0.0_f32; n_sub];
|
||||
|
||||
// Monotonic timestamp: microseconds since a shared process-local epoch.
|
||||
// All nodes use the same reference so the fuser's guard_interval_us check
|
||||
// compares apples to apples. No wall-clock mixing (immune to NTP jumps).
|
||||
let timestamp_us = last_time.duration_since(*EPOCH).as_micros() as u64;
|
||||
// Prefer the capture timestamp recovered from the node's mesh sync. This
|
||||
// keeps UDP scheduling jitter out of the fuser's cross-node guard. Older
|
||||
// firmware, stale sync state, and frames without the sync-valid bit retain
|
||||
// the process-local host-arrival fallback.
|
||||
let timestamp_us = ns
|
||||
.mesh_aligned_us_for_latest_csi_frame()
|
||||
.unwrap_or_else(|| {
|
||||
last_time
|
||||
.checked_duration_since(*EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_micros() as u64
|
||||
});
|
||||
|
||||
let canonical = CanonicalCsiFrame {
|
||||
amplitude,
|
||||
@@ -173,6 +187,7 @@ pub fn compute_person_score_from_amplitudes(amplitudes: &[f32]) -> f64 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::VecDeque;
|
||||
use wifi_densepose_hardware::{SyncPacket, SyncPacketFlags};
|
||||
|
||||
/// Helper: build a minimal NodeState for testing. Uses `NodeState::new()`
|
||||
/// then mutates the `pub(crate)` fields the bridge needs.
|
||||
@@ -225,6 +240,99 @@ mod tests {
|
||||
assert_eq!(ch.hardware_type, HardwareType::Esp32S3);
|
||||
}
|
||||
|
||||
fn mark_mesh_timed_frame(
|
||||
ns: &mut NodeState,
|
||||
node_id: u8,
|
||||
sync_sequence: u32,
|
||||
frame_sequence: u32,
|
||||
mesh_epoch_us: u64,
|
||||
host_arrival: Instant,
|
||||
) {
|
||||
ns.apply_sync_packet(
|
||||
SyncPacket {
|
||||
node_id,
|
||||
proto_ver: 1,
|
||||
flags: SyncPacketFlags {
|
||||
is_leader: node_id == 1,
|
||||
is_valid: true,
|
||||
smoothed_used: node_id != 1,
|
||||
},
|
||||
local_us: 10_000_000,
|
||||
epoch_us: mesh_epoch_us,
|
||||
sequence: sync_sequence,
|
||||
},
|
||||
Instant::now(),
|
||||
);
|
||||
ns.observe_accepted_csi_frame(frame_sequence, true, host_arrival);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mesh_timestamp_replaces_skewed_host_arrival_time() {
|
||||
let mut history = VecDeque::new();
|
||||
history.push_back(vec![10.0, 20.0, 30.0]);
|
||||
let host_arrival = Instant::now();
|
||||
let mut ns = make_node_state(history, None, 0);
|
||||
mark_mesh_timed_frame(&mut ns, 1, 100, 101, 1_000_000, host_arrival);
|
||||
|
||||
let frame = node_frame_from_state(1, &ns).expect("mesh-timed frame");
|
||||
assert_eq!(frame.timestamp_us, 1_050_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mesh_time_allows_fusion_despite_udp_arrival_skew() {
|
||||
let base = Instant::now() - Duration::from_millis(500);
|
||||
let mut states = HashMap::new();
|
||||
|
||||
let mut first_history = VecDeque::new();
|
||||
first_history.push_back(vec![1.0; 64]);
|
||||
let mut first = make_node_state(first_history, None, 0);
|
||||
mark_mesh_timed_frame(&mut first, 1, 100, 101, 1_000_000, base);
|
||||
states.insert(1, first);
|
||||
|
||||
let mut second_history = VecDeque::new();
|
||||
second_history.push_back(vec![1.1; 64]);
|
||||
let mut second = make_node_state(second_history, None, 0);
|
||||
mark_mesh_timed_frame(
|
||||
&mut second,
|
||||
2,
|
||||
200,
|
||||
201,
|
||||
1_005_000,
|
||||
base + Duration::from_millis(200),
|
||||
);
|
||||
states.insert(2, second);
|
||||
|
||||
let frames = node_frames_from_states(&states);
|
||||
let spread = frames.iter().map(|f| f.timestamp_us).max().unwrap()
|
||||
- frames.iter().map(|f| f.timestamp_us).min().unwrap();
|
||||
assert_eq!(spread, 5_000, "mesh capture spread, not 200 ms UDP skew");
|
||||
assert!(
|
||||
MultistaticFuser::new().fuse(&frames).is_ok(),
|
||||
"mesh-aligned frames inside the 60 ms guard must fuse"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsynchronized_frames_keep_host_arrival_guard() {
|
||||
let base = Instant::now() - Duration::from_millis(500);
|
||||
let mut states = HashMap::new();
|
||||
|
||||
for (node_id, arrival) in [
|
||||
(1, base),
|
||||
(2, base + Duration::from_millis(200)),
|
||||
] {
|
||||
let mut history = VecDeque::new();
|
||||
history.push_back(vec![1.0; 64]);
|
||||
states.insert(node_id, make_node_state(history, Some(arrival), 0));
|
||||
}
|
||||
|
||||
let frames = node_frames_from_states(&states);
|
||||
assert!(
|
||||
MultistaticFuser::new().fuse(&frames).is_err(),
|
||||
"without valid mesh time, 200 ms arrival skew must still trip the 60 ms guard"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heterogeneous_node_counts_canonicalize_and_fuse() {
|
||||
// Issue #1170 regression: a mixed mesh with HT20 (64-bin) and HT40
|
||||
|
||||
@@ -104,12 +104,11 @@ async fn subscribe_client(port: u16, topics: &[&str]) -> (AsyncClient, EventLoop
|
||||
.unwrap_or(0);
|
||||
let mut opts = MqttOptions::new(
|
||||
format!("ruview-test-sub-{}-{}", std::process::id(), suffix),
|
||||
"127.0.0.1",
|
||||
port,
|
||||
("127.0.0.1", port),
|
||||
);
|
||||
opts.set_keep_alive(Duration::from_secs(10));
|
||||
opts.set_keep_alive(10);
|
||||
opts.set_clean_session(true);
|
||||
let (client, mut eventloop) = AsyncClient::new(opts, 256);
|
||||
let (client, mut eventloop) = AsyncClient::builder(opts).capacity(256).build();
|
||||
for t in topics {
|
||||
client.subscribe(*t, QoS::AtLeastOnce).await.unwrap();
|
||||
}
|
||||
@@ -147,7 +146,11 @@ async fn collect_published(
|
||||
let remain = until - tokio::time::Instant::now();
|
||||
match timeout(remain, eventloop.poll()).await {
|
||||
Ok(Ok(Event::Incoming(Packet::Publish(p)))) => {
|
||||
out.push((p.topic, p.payload.to_vec(), p.retain));
|
||||
out.push((
|
||||
String::from_utf8_lossy(&p.topic).to_string(),
|
||||
p.payload.to_vec(),
|
||||
p.retain,
|
||||
));
|
||||
}
|
||||
Ok(Ok(_)) => {} // ignore other events
|
||||
Ok(Err(e)) => {
|
||||
|
||||
Reference in New Issue
Block a user