smolkik_adm/RuView
1
0
Fork 0
mirror of https://github.com/ruvnet/RuView.git synced 2026-09-01 04:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add native iPhone LiDAR sensor and web viewer (#1684)

Browse Source 
* feat(ios): add RuView LiDAR frame protocol

* feat(ios): capture ARKit scene depth for RuView

* feat(ios): stream compact LiDAR frames over websocket

* feat(ios): add native LiDAR capture UI

* feat(ios): add RuView LiDAR app entrypoint

* feat(web): add LiDAR bridge web package

* feat(web): decode RuView LiDAR wire frames

* feat(web): add local LiDAR websocket relay

* feat(web): add LiDAR browser viewer

* feat(web): render live LiDAR point cloud

* fix(ios): use wall clock time for LiDAR provenance

* feat(web): style LiDAR viewer

* test(web): add LiDAR codec tests

* docs: add iPhone LiDAR integration guide

* docs(adr): define iPhone LiDAR sensor bridge

* fix(ios): harden and validate LiDAR bridge

* fix(ios): qualify LiDAR wire depth type
This commit is contained in:
rUv
2026-08-22 18:06:16 -04:00
committed by GitHub
parent bd110e0eac
commit 0df48df7b2
17 changed files with 1316 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
integrations/iphone-lidar/native/RuViewLiDAR/ContentView.swift Normal file
View File

@@ -0,0 +1,121 @@
import SwiftUI
struct ContentView: View {
@StateObject private var capture = LiDARCaptureManager()
@State private var endpoint = "ws://HOST:8787/ws/lidar?token=TOKEN"
@State private var streaming = false
@State private var status = "Idle"
private let streamer = WebSocketStreamer()
var body: some View {
NavigationStack {
Form {
Section("Sensor") {
HStack {
Text("State")
Spacer()
Text(stateText)
.foregroundStyle(stateColor)
}
HStack {
Text("Capture FPS")
Spacer()
Text(capture.framesPerSecond.formatted(.number.precision(.fractionLength(1))))
}
if let frame = capture.lastFrame {
HStack {
Text("Depth")
Spacer()
Text("\(frame.depth.width) x \(frame.depth.height)")
}
HStack {
Text("Sequence")
Spacer()
Text("\(frame.provenance.sequence)")
}
}
Button("Start LiDAR") {
capture.start(smoothed: false)
}
.disabled(capture.state == .running)
Button("Stop") {
capture.stop()
}
.disabled(capture.state != .running)
}
Section("RuView Stream") {
TextField("ws://host:port/ws/lidar", text: $endpoint)
.textInputAutocapitalization(.never)
.autocorrectionDisabled()
Toggle("Stream geometry", isOn: $streaming)
.onChange(of: streaming) { _, enabled in
Task {
if enabled {
do {
try await streamer.connect(to: endpoint)
status = "Connected"
} catch {
streaming = false
status = error.localizedDescription
}
} else {
await streamer.disconnect()
status = "Disconnected"
}
}
}
Text(status)
.font(.caption)
.foregroundStyle(.secondary)
}
Section("Privacy") {
Text("This implementation transmits depth geometry, confidence, camera intrinsics, and device pose. RGB camera frames are not transmitted.")
.font(.footnote)
}
}
.navigationTitle("RuView LiDAR")
.onAppear {
capture.onFrame = { frame in
guard streaming else { return }
Task {
do {
try await streamer.send(frame, maxFPS: 15, sampleStep: 2)
} catch {
await MainActor.run {
status = error.localizedDescription
}
}
}
}
}
.onDisappear {
capture.stop()
Task { await streamer.disconnect() }
}
}
}
private var stateText: String {
switch capture.state {
case .idle: return "Idle"
case .unsupported: return "No LiDAR"
case .running: return "Live"
case .failed(let message): return "Error: \(message)"
}
}
private var stateColor: Color {
switch capture.state {
case .running: return .green
case .failed, .unsupported: return .red
case .idle: return .secondary
}
}
}

138
integrations/iphone-lidar/native/RuViewLiDAR/LiDARCaptureManager.swift Normal file
View File

@@ -0,0 +1,138 @@
import ARKit
import CoreVideo
import Foundation
@MainActor
final class LiDARCaptureManager: NSObject, ObservableObject {
enum State: Equatable {
case idle
case unsupported
case running
case failed(String)
}
@Published private(set) var state: State = .idle
@Published private(set) var lastFrame: RuViewLiDARFrame?
@Published private(set) var framesPerSecond: Double = 0
let session = ARSession()
var onFrame: (@MainActor @Sendable (RuViewLiDARFrame) -> Void)?
private var sequence: UInt64 = 0
private var lastTimestamp: TimeInterval?
private let processingQueue = DispatchQueue(label: "one.ruv.lidar.capture", qos: .userInitiated)
override init() {
super.init()
session.delegate = self
session.delegateQueue = processingQueue
}
func start(smoothed: Bool = false) {
let configuration = ARWorldTrackingConfiguration()
let semantic: ARConfiguration.FrameSemantics = smoothed ? .smoothedSceneDepth : .sceneDepth
guard ARWorldTrackingConfiguration.supportsFrameSemantics(semantic) else {
state = .unsupported
return
}
configuration.frameSemantics.insert(semantic)
configuration.worldAlignment = .gravity
session.run(configuration, options: [.resetTracking, .removeExistingAnchors])
state = .running
}
func stop() {
session.pause()
state = .idle
}
nonisolated private func makeFrame(from frame: ARFrame) -> RuViewLiDARFrame? {
guard let sceneDepth = frame.sceneDepth ?? frame.smoothedSceneDepth else { return nil }
let depthMap = sceneDepth.depthMap
let confidenceMap = sceneDepth.confidenceMap
guard CVPixelBufferLockBaseAddress(depthMap, .readOnly) == kCVReturnSuccess else {
return nil
}
defer { CVPixelBufferUnlockBaseAddress(depthMap, .readOnly) }
guard CVPixelBufferGetPixelFormatType(depthMap) == kCVPixelFormatType_DepthFloat32,
let depthBase = CVPixelBufferGetBaseAddress(depthMap) else {
return nil
}
let width = CVPixelBufferGetWidth(depthMap)
let height = CVPixelBufferGetHeight(depthMap)
let stride = CVPixelBufferGetBytesPerRow(depthMap) / MemoryLayout<Float>.size
let pointer = depthBase.assumingMemoryBound(to: Float.self)
var meters = [Float]()
meters.reserveCapacity(width * height)
for y in 0..<height {
let row = pointer.advanced(by: y * stride)
for x in 0..<width {
let value = row[x]
meters.append(value.isFinite && value > 0 ? value : 0)
}
}
var confidence = [UInt8](repeating: 0, count: width * height)
if let confidenceMap,
CVPixelBufferGetPixelFormatType(confidenceMap) == kCVPixelFormatType_OneComponent8,
CVPixelBufferGetWidth(confidenceMap) == width,
CVPixelBufferGetHeight(confidenceMap) == height,
CVPixelBufferLockBaseAddress(confidenceMap, .readOnly) == kCVReturnSuccess {
defer { CVPixelBufferUnlockBaseAddress(confidenceMap, .readOnly) }
if let confidenceBase = CVPixelBufferGetBaseAddress(confidenceMap) {
let confidenceStride = CVPixelBufferGetBytesPerRow(confidenceMap)
let confidencePointer = confidenceBase.assumingMemoryBound(to: UInt8.self)
for y in 0..<height {
let row = confidencePointer.advanced(by: y * confidenceStride)
for x in 0..<width {
confidence[y * width + x] = row[x]
}
}
}
}
return RuViewLiDARFrame(
intrinsics: frame.camera.intrinsics,
imageResolution: frame.camera.imageResolution,
cameraTransform: frame.camera.transform,
depthWidth: width,
depthHeight: height,
depthMeters: meters,
confidence: confidence,
sequence: 0,
timestamp: Date().timeIntervalSince1970
)
}
}
extension LiDARCaptureManager: ARSessionDelegate {
nonisolated func session(_ session: ARSession, didUpdate frame: ARFrame) {
guard let base = makeFrame(from: frame) else { return }
let frameTimestamp = frame.timestamp
Task { @MainActor in
sequence &+= 1
let corrected = base.assigningSequence(sequence)
if let previous = lastTimestamp {
let delta = frameTimestamp - previous
if delta > 0 { framesPerSecond = 1.0 / delta }
}
lastTimestamp = frameTimestamp
lastFrame = corrected
onFrame?(corrected)
}
}
nonisolated func session(_ session: ARSession, didFailWithError error: Error) {
Task { @MainActor in
state = .failed(error.localizedDescription)
}
}
}

118
integrations/iphone-lidar/native/RuViewLiDAR/RuViewFrame.swift Normal file
View File

@@ -0,0 +1,118 @@
import Foundation
import simd
struct RuViewLiDARFrame: Codable, Sendable {
struct Intrinsics: Codable, Sendable {
let fx: Float
let fy: Float
let cx: Float
let cy: Float
let imageWidth: Int
let imageHeight: Int
}
struct Pose: Codable, Sendable {
let matrix: [Float]
}
struct Depth: Codable, Sendable {
let width: Int
let height: Int
let meters: [Float]
let confidence: [UInt8]
}
struct Provenance: Codable, Sendable {
let sensor: String
let source: String
let privacyClass: String
let sequence: UInt64
let timestampNs: UInt64
let schema: String
}
let type: String
let intrinsics: Intrinsics
let pose: Pose
let depth: Depth
let provenance: Provenance
init(
intrinsics: simd_float3x3,
imageResolution: CGSize,
cameraTransform: simd_float4x4,
depthWidth: Int,
depthHeight: Int,
depthMeters: [Float],
confidence: [UInt8],
sequence: UInt64,
timestamp: TimeInterval
) {
self.type = "ruview.lidar.depth.v1"
self.intrinsics = Intrinsics(
fx: intrinsics.columns.0.x,
fy: intrinsics.columns.1.y,
cx: intrinsics.columns.2.x,
cy: intrinsics.columns.2.y,
imageWidth: Int(imageResolution.width),
imageHeight: Int(imageResolution.height)
)
self.pose = Pose(matrix: cameraTransform.columnMajorArray)
self.depth = Depth(
width: depthWidth,
height: depthHeight,
meters: depthMeters,
confidence: confidence
)
self.provenance = Provenance(
sensor: "apple-arkit-scene-depth",
source: "live",
privacyClass: "geometry-only",
sequence: sequence,
timestampNs: UInt64(max(0, timestamp) * 1_000_000_000),
schema: "ruview.lidar.depth.v1"
)
}
func assigningSequence(_ sequence: UInt64) -> RuViewLiDARFrame {
RuViewLiDARFrame(
type: type,
intrinsics: intrinsics,
pose: pose,
depth: depth,
provenance: Provenance(
sensor: provenance.sensor,
source: provenance.source,
privacyClass: provenance.privacyClass,
sequence: sequence,
timestampNs: provenance.timestampNs,
schema: provenance.schema
)
)
}
private init(
type: String,
intrinsics: Intrinsics,
pose: Pose,
depth: Depth,
provenance: Provenance
) {
self.type = type
self.intrinsics = intrinsics
self.pose = pose
self.depth = depth
self.provenance = provenance
}
}
private extension simd_float4x4 {
var columnMajorArray: [Float] {
[
columns.0.x, columns.0.y, columns.0.z, columns.0.w,
columns.1.x, columns.1.y, columns.1.z, columns.1.w,
columns.2.x, columns.2.y, columns.2.z, columns.2.w,
columns.3.x, columns.3.y, columns.3.z, columns.3.w
]
}
}

10
integrations/iphone-lidar/native/RuViewLiDAR/RuViewLiDARApp.swift Normal file
View File

@@ -0,0 +1,10 @@
import SwiftUI
@main
struct RuViewLiDARApp: App {
var body: some Scene {
WindowGroup {
ContentView()
}
}
}

93
integrations/iphone-lidar/native/RuViewLiDAR/WebSocketStreamer.swift Normal file
View File

@@ -0,0 +1,93 @@
import Foundation
actor WebSocketStreamer {
enum StreamError: Error {
case invalidURL
}
struct WirePacket: Codable {
struct Depth: Codable {
let width: Int
let height: Int
let encoding: String
let millimetersBase64: String
let confidenceBase64: String
}
let type: String
let intrinsics: RuViewLiDARFrame.Intrinsics
let pose: RuViewLiDARFrame.Pose
let depth: Depth
let provenance: RuViewLiDARFrame.Provenance
}
private var task: URLSessionWebSocketTask?
private let encoder = JSONEncoder()
private var lastSentNs: UInt64 = 0
func connect(to endpoint: String) throws {
guard let url = URL(string: endpoint),
url.scheme == "ws" || url.scheme == "wss" else {
throw StreamError.invalidURL
}
task?.cancel(with: .goingAway, reason: nil)
let socket = URLSession.shared.webSocketTask(with: url)
socket.resume()
task = socket
}
func disconnect() {
task?.cancel(with: .goingAway, reason: nil)
task = nil
}
func send(_ frame: RuViewLiDARFrame, maxFPS: UInt64 = 15, sampleStep: Int = 2) async throws {
guard let task else { return }
let timestamp = frame.provenance.timestampNs
let minDelta = 1_000_000_000 / max(1, maxFPS)
guard timestamp >= lastSentNs + minDelta else { return }
lastSentNs = timestamp
let packet = Self.makeWirePacket(frame, sampleStep: max(1, sampleStep))
let data = try encoder.encode(packet)
guard let string = String(data: data, encoding: .utf8) else { return }
try await task.send(.string(string))
}
static func makeWirePacket(_ frame: RuViewLiDARFrame, sampleStep: Int) -> WirePacket {
let step = max(1, sampleStep)
let sourceWidth = frame.depth.width
let sourceHeight = frame.depth.height
let width = (sourceWidth + step - 1) / step
let height = (sourceHeight + step - 1) / step
var millimeters = Data(capacity: width * height * 2)
var confidence = Data(capacity: width * height)
for y in stride(from: 0, to: sourceHeight, by: step) {
for x in stride(from: 0, to: sourceWidth, by: step) {
let index = y * sourceWidth + x
let meters = frame.depth.meters[index]
let mm = UInt16(clamping: Int((meters * 1000).rounded()))
var littleEndian = mm.littleEndian
withUnsafeBytes(of: &littleEndian) { millimeters.append(contentsOf: $0) }
confidence.append(frame.depth.confidence[index])
}
}
return WirePacket(
type: frame.type,
intrinsics: frame.intrinsics,
pose: frame.pose,
depth: WirePacket.Depth(
width: width,
height: height,
encoding: "u16le-mm+u8-confidence",
millimetersBase64: millimeters.base64EncodedString(),
confidenceBase64: confidence.base64EncodedString()
),
provenance: frame.provenance
)
}
}