diff --git a/miniapps/navigation/miniapp.json b/miniapps/navigation/miniapp.json index 97b2e2feac..c7dd257aa0 100644 --- a/miniapps/navigation/miniapp.json +++ b/miniapps/navigation/miniapp.json @@ -1,7 +1,7 @@ { "$schema": "./node_modules/@mentra/miniapp-cli/schema/miniapp.schema.json", "packageName": "com.mentra.navigation", - "version": "1.1.7", + "version": "1.1.8", "name": "Mentra Maps", "description": "Turn-by-turn walking navigation with compass and Google Maps.", "icon": "icon.png", diff --git a/miniapps/navigation/src/ui/components/Drawer/Drawer.tsx b/miniapps/navigation/src/ui/components/Drawer/Drawer.tsx index e0c66f8c45..46f3f99e8f 100644 --- a/miniapps/navigation/src/ui/components/Drawer/Drawer.tsx +++ b/miniapps/navigation/src/ui/components/Drawer/Drawer.tsx @@ -200,7 +200,14 @@ function DrawerInner({ } return ( -
+ // Lift the whole drawer above the Android 3-button nav bar. The host + // injects the bottom safe-area inset as `--mentra-safe-bottom` (0 on iOS / + // gesture nav). It's applied to the outer wrapper — NOT the measured card — + // so the peek/drag math (sheetHeight, peekOffset, published DrawerOffset) + // is unchanged; the map stays full-bleed behind everything. +
+ // Sit above the Android 3-button nav bar: base 2.5rem (was + // `bottom-10`) + host-injected bottom inset (0 on iOS / gesture nav). + style={{bottom: "calc(2.5rem + var(--mentra-safe-bottom))"}} + className="pointer-events-none fixed inset-x-0 z-[100] flex justify-center px-6">
{message}
diff --git a/miniapps/navigation/src/ui/pages/NavigationPage/components/LocationSearch/LocationSearch.tsx b/miniapps/navigation/src/ui/pages/NavigationPage/components/LocationSearch/LocationSearch.tsx index d55fb4e636..f242dce81d 100644 --- a/miniapps/navigation/src/ui/pages/NavigationPage/components/LocationSearch/LocationSearch.tsx +++ b/miniapps/navigation/src/ui/pages/NavigationPage/components/LocationSearch/LocationSearch.tsx @@ -294,6 +294,10 @@ export function LocationSearch({selected, onSelect, onClear, disabled, devFrozen // is exempt — browsers always allow selection // inside form controls so the user can still edit their // query normally. + // Full-bleed white panel (bg goes under the bars), but pad the + // scroll content so the last result clears the Android 3-button + // nav bar (host-injected inset; 0 on iOS / gesture nav). + style={{paddingBottom: "var(--mentra-safe-bottom)"}} className={`fixed z-40 inset-x-0 bottom-0 top-0 bg-white overflow-auto select-none [-webkit-touch-callout:none] [-webkit-user-select:none] ${safeHeadingSearchResults}`}> {loading ? (
diff --git a/miniapps/navigation/src/ui/pages/NavigationPage/components/NavMap/NavMap.tsx b/miniapps/navigation/src/ui/pages/NavigationPage/components/NavMap/NavMap.tsx index 52f65b5748..944cf0e95b 100644 --- a/miniapps/navigation/src/ui/pages/NavigationPage/components/NavMap/NavMap.tsx +++ b/miniapps/navigation/src/ui/pages/NavigationPage/components/NavMap/NavMap.tsx @@ -106,10 +106,16 @@ export function NavMap({ const devEnabled = isDev || devOverride // Anchor the floating right-rail (zoom / recenter buttons) just above - // whichever drawer is currently mounted. + // whichever drawer is currently mounted, and above the Android 3-button nav + // bar (host-injected `--mentra-safe-bottom`; 0 on iOS / gesture nav). The + // same inset is applied to the drawers' outer wrappers, so this keeps the + // rail the same 12px above whichever drawer is up. const drawerOffset = useDrawerOffset() const fallbackZero = useMotionValue(0) - const railBottom = useTransform(drawerOffset ?? fallbackZero, (h: number) => h + 12) + const railBottom = useTransform( + drawerOffset ?? fallbackZero, + (h: number) => `calc(${h + 12}px + var(--mentra-safe-bottom))`, + ) useEffect(() => { let alive = true diff --git a/miniapps/navigation/src/ui/pages/NavigationPage/components/NavigationRunningDrawer/NavigationRunningDrawer.tsx b/miniapps/navigation/src/ui/pages/NavigationPage/components/NavigationRunningDrawer/NavigationRunningDrawer.tsx index bdd06962cf..a0e476779d 100644 --- a/miniapps/navigation/src/ui/pages/NavigationPage/components/NavigationRunningDrawer/NavigationRunningDrawer.tsx +++ b/miniapps/navigation/src/ui/pages/NavigationPage/components/NavigationRunningDrawer/NavigationRunningDrawer.tsx @@ -88,6 +88,11 @@ export function NavigationRunningDrawer({ animate={{y: 0}} exit={{y: "100%"}} transition={{type: "spring", stiffness: 320, damping: 42}} + // Lift the running bar above the Android 3-button nav bar via the + // host-injected bottom inset (0 on iOS / gesture nav). On the outer + // wrapper so the measured bar height (published as DrawerOffset for + // the right-rail) is unaffected. + style={{paddingBottom: "var(--mentra-safe-bottom)"}} className="fixed left-0 right-0 bottom-0 z-40 pointer-events-none">
" suffix is moved into the tip line next to the distance so the big line stays + * a glanceable maneuver. + */ + private fun parseNavHud(text: String): NavHud { + var eta = "" + val content = mutableListOf() + for (raw in text.split("\n")) { + val line = raw.trim() + if (line.isEmpty()) continue + when { + // Drop the arrow GLYPH line: the app forces it to "↑" until ~10 m before the turn, + // so it reads "straight" for almost the whole leg. We derive the real direction + // from the maneuver instruction instead (see arrowForInstruction). + line == "←" || line == "→" || line == "↑" || line == "↺" || line == "⇆" -> {} + line.startsWith("⊙") -> eta = line.removePrefix("⊙").trim() + else -> content.add(line) + } + } + // The distance-to-turn line (used as the tip) vs the instruction line. + val distRegex = Regex("^(in\\s|now$|arriving)", RegexOption.IGNORE_CASE) + val distLine = content.firstOrNull { distRegex.containsMatchIn(it) } ?: "" + val rawInstruction = content.filter { it != distLine }.joinToString(" ") + val (instruction, road) = shortenInstruction(rawInstruction) + // Arrow direction from the maneuver text, so it points at the turn the whole approach. + val arrow = arrowForInstruction(instruction) + // Welcome/arrival frames (arrow == NONE) carry no live nav data — hide the status bar. + val showStatus = arrow != NavArrow.NONE + // Tip = distance line + road name (e.g. "In 198 m on Octavia St"); welcome gets a prompt. + var tip = listOf(distLine, road).filter { it.isNotEmpty() }.joinToString(" on ") + if (instruction.startsWith("Welcome")) tip = "Choose a location on the map" + // Numeric distance for the status struct: strip the "In " / "Arriving in " prefix. + val distance = + distLine.replace(Regex("^(arriving\\s+in|in)\\s+", RegexOption.IGNORE_CASE), "").trim() + return NavHud(arrow, instruction, tip, distance, eta, showStatus) + } + + /** Turn direction inferred from the maneuver phrase ("Turn left" → LEFT, else STRAIGHT). */ + private fun arrowForInstruction(instruction: String): NavArrow { + val lower = instruction.lowercase() + return when { + // No maneuver to point at — hide the arrow entirely on these frames. + lower.contains("welcome") || lower.contains("arrived") -> NavArrow.NONE + lower.contains("left") -> NavArrow.LEFT + lower.contains("right") -> NavArrow.RIGHT + else -> NavArrow.STRAIGHT + } + } + + /** + * Reduces a raw instruction to a short maneuver phrase (≤3 words) and the road name it + * referenced. "Turn right onto Octavia St" → ("Turn right", "Octavia St"); welcome/arrival + * frames collapse to fixed short labels. + */ + private fun shortenInstruction(raw: String): Pair { + val trimmed = raw.trim() + if (trimmed.isEmpty()) return Pair("", "") + val lower = trimmed.lowercase() + if (lower.contains("welcome")) return Pair("Welcome to Maps", "") + if (lower.contains("arrived")) return Pair("Arrived!", "") + // Peel off the "onto " suffix (or a leading "Onto ") into the road name. + var maneuver = trimmed + var road = "" + val ontoIdx = lower.indexOf(" onto ") + if (ontoIdx >= 0) { + maneuver = trimmed.substring(0, ontoIdx).trim() + road = trimmed.substring(ontoIdx + 6).trim() + } else if (lower.startsWith("onto ")) { + road = trimmed.substring(5).trim() + maneuver = "" + } + val capped = + maneuver.split(Regex("\\s+")).filter { it.isNotEmpty() }.take(3).joinToString(" ") + return Pair(capped, road) + } + + /** Pushes a parsed HUD frame into the nav widgets (arrow icon / instruction / tip / info). */ + private fun pushNavHud(hud: NavHud) { + // naviInstruction (0x02): primary turn text. + enqueueNavWidget( + NimoProtocol.NAV_RES_TURN_TEXT, + NimoProtocol.WIDGET_TEXT_NEW, + hud.instruction.ifEmpty { " " }.toByteArray(Charsets.UTF_8) + ) + // naviInstructionTip (0x04): distance-to-turn line. + enqueueNavWidget( + NimoProtocol.NAV_RES_TIP, + NimoProtocol.WIDGET_TEXT_NEW, + hud.tip.ifEmpty { " " }.toByteArray(Charsets.UTF_8) + ) + // naviInfo (0x03): 65-byte status struct — navType(1) + dist(32) + time(32). Skipped on + // welcome/arrival frames so no (empty) status bar shows there. + if (hud.showStatus) { + val info = ByteArray(NimoProtocol.NAV_INFO_STRUCT_SIZE) + info[0] = NimoProtocol.NAV_TYPE_WALK.toByte() + writeUtf8Padded(info, 1, hud.distance, 32) + writeUtf8Padded(info, 33, hud.eta, 32) + enqueueNavWidget(NimoProtocol.NAV_RES_INFO, NimoProtocol.WIDGET_RAW, info) + } + // naviIcon (0x01): turn-arrow. Only re-send when the DIRECTION changes — re-pushing the + // identical image on every distance tick made the arrow flicker/"move". The bitmap for + // each direction is rendered once and cached. + if (hud.arrow != lastNavArrow) { + lastNavArrow = hud.arrow + enqueueNavWidget( + NimoProtocol.NAV_RES_ARROW, + NimoProtocol.WIDGET_PICTURE, + navArrowContent(hud.arrow) + ) + } + } + + // Arrow widget content for every direction, rendered ONCE at construction (start time) and + // reused — the bitmaps are deterministic, so the send path just selects the pre-built bytes + // instead of re-rendering on each turn. + private val navArrowCache: Map = + NavArrow.values().associateWith { arrow -> + val bmp = buildNavArrowBitmap(arrow) + val gray = + bitmapToGrayscale( + bmp, + NimoProtocol.NAV_ARROW_SIZE, + NimoProtocol.NAV_ARROW_SIZE + ) + bmp.recycle() + buildNavImageContent(gray, NimoProtocol.NAV_ARROW_SIZE, NimoProtocol.NAV_ARROW_SIZE) + } + + private fun navArrowContent(arrow: NavArrow): ByteArray = navArrowCache.getValue(arrow) + + /** A 48x48 white turn-arrow on black, rotated to point left/right/up per [arrow]. */ + private fun buildNavArrowBitmap(arrow: NavArrow): Bitmap { + val size = NimoProtocol.NAV_ARROW_SIZE + val bmp = Bitmap.createBitmap(size, size, Bitmap.Config.ARGB_8888) + val canvas = Canvas(bmp) + canvas.drawColor(Color.BLACK) + // NONE → leave the icon blank (all black = nothing lit) for welcome/arrival frames. + if (arrow == NavArrow.NONE) return bmp + val paint = + Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = Color.WHITE + style = Paint.Style.FILL + } + val s = size.toFloat() + // Rotate the (upward) arrow around the center to face the turn direction. + val rotation = + when (arrow) { + NavArrow.LEFT -> -90f + NavArrow.RIGHT -> 90f + else -> 0f + } + canvas.save() + canvas.rotate(rotation, s / 2f, s / 2f) + val path = + android.graphics.Path().apply { + moveTo(s * 0.5f, s * 0.15f) // tip + lineTo(s * 0.85f, s * 0.55f) // head bottom-right + lineTo(s * 0.62f, s * 0.55f) + lineTo(s * 0.62f, s * 0.85f) // shaft bottom-right + lineTo(s * 0.38f, s * 0.85f) // shaft bottom-left + lineTo(s * 0.38f, s * 0.55f) + lineTo(s * 0.15f, s * 0.55f) // head bottom-left + close() + } + canvas.drawPath(path, paint) + canvas.restore() + return bmp + } + override fun showDashboard() { Bridge.log("NIMO: showDashboard()") textAppEntered = false @@ -1288,22 +1501,15 @@ class Nimo : SGCManager() { // ---------- SGCManager: Device Control ---------- override fun setHeadUpAngle(angle: Int) { - val clamped = angle.coerceIn(0, 90) - Bridge.log("NIMO: setHeadUpAngle($clamped)") - // Enable the head-up display gesture, then set the wake angle. + Bridge.log("NIMO: setHeadUpAngle($angle) — head-up display disabled for now") + // Head-up display is intentionally kept OFF for now (it auto-wakes the dashboard and the + // wake-angle semantics aren't dialed in). Force it disabled regardless of the requested + // angle. TODO: restore the enable + setAngle [optType, deg] path once verified. sendFrame( NimoFrameCodec.encodeFrame( NimoProtocol.CMD_SET_PARAMETER, NimoProtocol.SET_HEADUP_DISPLAY, - byteArrayOf(1) - ) - ) - // setAngle payload is [optType, deg]. TODO: hardware-verify optType semantics. - sendFrame( - NimoFrameCodec.encodeFrame( - NimoProtocol.CMD_SET_PARAMETER, - NimoProtocol.SET_ANGLE, - byteArrayOf(0x01, clamped.toByte()) + byteArrayOf(0) ) ) } @@ -2091,6 +2297,16 @@ class Nimo : SGCManager() { needsAck = false ) ) + // Disable the on-device head-up gesture so raising your head doesn't auto-wake the + // dashboard. Disabled for now; re-enable once the wake-angle behaviour is dialed in. + sendFrame( + NimoFrameCodec.encodeFrame( + NimoProtocol.CMD_SET_PARAMETER, + NimoProtocol.SET_HEADUP_DISPLAY, + byteArrayOf(0), + needsAck = false + ) + ) getBatteryStatus() requestVersionInfo() @@ -2114,6 +2330,8 @@ class Nimo : SGCManager() { textAppEntered = false navAppEntered = false currentGlassesAppId = -1 + lastNavHudKey = "" + lastNavArrow = null pendingText = null audioClient.stop() receiveAssembler.reset() @@ -2295,12 +2513,21 @@ class Nimo : SGCManager() { NimoProtocol.STATE_ENTER -> { currentGlassesAppId = appId if (appId != textAppId) textAppEntered = false - if (appId != NimoProtocol.APP_ID_NAV) navAppEntered = false + if (appId != NimoProtocol.APP_ID_NAV) { + navAppEntered = false + // Force a full HUD repaint next time the nav app reopens. + lastNavHudKey = "" + lastNavArrow = null + } } NimoProtocol.STATE_EXIT -> { if (appId == currentGlassesAppId) currentGlassesAppId = -1 if (appId == textAppId) textAppEntered = false - if (appId == NimoProtocol.APP_ID_NAV) navAppEntered = false + if (appId == NimoProtocol.APP_ID_NAV) { + navAppEntered = false + lastNavHudKey = "" + lastNavArrow = null + } } } } @@ -2356,11 +2583,14 @@ class Nimo : SGCManager() { when (code) { NimoProtocol.INPUT_HEAD_UP -> { DeviceStore.apply("glasses", "headUp", true) - Bridge.sendHeadUp(true) + // Head-up forwarding to the cloud is disabled for now (the wake-angle + // semantics aren't dialed in yet). Keep the local state; don't notify. + // Bridge.sendHeadUp(true) } NimoProtocol.INPUT_HEAD_DOWN -> { DeviceStore.apply("glasses", "headUp", false) - Bridge.sendHeadUp(false) + // Head-up forwarding to the cloud is disabled for now (see INPUT_HEAD_UP). + // Bridge.sendHeadUp(false) } NimoProtocol.INPUT_CLICK_RIGHT, NimoProtocol.INPUT_CLICK_LEFT -> Bridge.sendTouchEvent(DeviceTypes.NIMO, "single_tap", timestamp) @@ -2446,6 +2676,36 @@ class Nimo : SGCManager() { Bridge.sendVersionInfo(mapOf("firmwareVersion" to version)) } + // ---------- Nav Widget Helpers ---------- + + /** Enqueues a single nav-app content widget (chunked, appId 0x01, layout 0). */ + private fun enqueueNavWidget(resId: Int, resType: Int, content: ByteArray) { + enqueueFrames( + NimoFrameCodec.updateContentFrames( + appId = NimoProtocol.APP_ID_NAV, + layoutId = 0, + resId = resId, + resType = resType, + content = content + ) + ) + } + + /** Builds the [imageHeader + payload] blob for a nav image widget from an L8 buffer. */ + private fun buildNavImageContent(grayscale: ByteArray, width: Int, height: Int): ByteArray { + val packed = packL8To2bpp(grayscale) + val (payload, compression) = compressAdaptive(packed) + return NimoFrameCodec.imageHeader( + width = width, + height = height, + formatBpp = NimoProtocol.FORMAT_2BPP, + compression = compression, + originalSize = packed.size, + compressedSize = + if (compression == NimoProtocol.COMPRESSION_NONE) 0 else payload.size + ) + payload + } + // ---------- Bitmap Helpers ---------- /** Aspect-fit the bitmap into [width]x[height] on black, then convert to L8 grayscale. */ diff --git a/mobile/modules/bluetooth-sdk/ios/Source/sgcs/Nimo.swift b/mobile/modules/bluetooth-sdk/ios/Source/sgcs/Nimo.swift index a027e31011..c14c2b6404 100644 --- a/mobile/modules/bluetooth-sdk/ios/Source/sgcs/Nimo.swift +++ b/mobile/modules/bluetooth-sdk/ios/Source/sgcs/Nimo.swift @@ -6,6 +6,7 @@ // separate H-T-L-V Opus microphone channel (char 2025). // +import AudioToolbox import AVFoundation import Compression import CoreBluetooth @@ -113,19 +114,31 @@ enum NimoProtocol { // widget resTypes static let WIDGET_TEXT_NEW = 0x00 + static let WIDGET_TEXT_APPEND = 0x01 + static let WIDGET_RAW = 0x02 static let WIDGET_PICTURE = 0x80 // navigation widget resIds (appId 0x01): mini map 0x00, arrow 0x01, turn text 0x02, // status bar 0x03 (raw), tip 0x04, large map 0x05. static let NAV_RES_MINI_MAP = 0x00 + static let NAV_RES_ARROW = 0x01 static let NAV_RES_TURN_TEXT = 0x02 + static let NAV_RES_INFO = 0x03 + static let NAV_RES_TIP = 0x04 static let NAV_RES_LARGE_MAP = 0x05 // navigation image widget sizes (hard firmware requirements; see IMAGE_PROTOCOL). static let NAV_MINI_MAP_SIZE = 160 + static let NAV_ARROW_SIZE = 48 static let NAV_LARGE_MAP_WIDTH = 452 static let NAV_LARGE_MAP_HEIGHT = 170 + // naviInfo status struct (resId 0x03, raw): navType(1) + dist(32 UTF-8) + time(32 UTF-8). + static let NAV_INFO_STRUCT_SIZE = 65 + static let NAV_TYPE_WALK = 0x00 + static let NAV_TYPE_CYCLE = 0x01 + static let NAV_TYPE_EBIKE = 0x02 + // phone types static let PHONE_TYPE_IOS = 0x01 @@ -457,78 +470,207 @@ enum NimoAudioParser { } } -// MARK: - Opus Decoder (AVAudioConverter) +// MARK: - Opus Decoder (AudioToolbox) + +/// Holds the single Opus packet being fed to the AudioConverter for its C input +/// callback. Uses a manually-allocated buffer + packet-description pointer that +/// stay valid AFTER the callback returns (the converter reads them out of band), +/// which a scoped `withUnsafeBytes` pointer would not. +private final class NimoOpusInputContext { + var buffer: UnsafeMutableRawPointer? + var byteSize: Int = 0 + let packetDescPtr = UnsafeMutablePointer.allocate(capacity: 1) + var consumed = false + + func set(_ data: Data) { + buffer?.deallocate() + byteSize = data.count + let buf = UnsafeMutableRawPointer.allocate(byteCount: max(1, byteSize), alignment: 1) + data.copyBytes(to: buf.assumingMemoryBound(to: UInt8.self), count: byteSize) + buffer = buf + packetDescPtr.pointee = AudioStreamPacketDescription( + mStartOffset: 0, mVariableFramesInPacket: 0, mDataByteSize: UInt32(byteSize) + ) + consumed = false + } + + deinit { + buffer?.deallocate() + packetDescPtr.deallocate() + } +} + +/// Returned by the input proc once the single cached packet is exhausted. Signalling +/// "no more data" with a NON-zero status (instead of 0 packets + noErr) is critical: +/// 0-packets-noErr latches AudioConverter into a permanent end-of-stream state, which +/// makes it decode the first packet and then return 0 frames for every packet after. +private let kNimoOpusNoMoreData: OSStatus = 0x6E6F_4D44 // 'noMD' + +/// AudioConverter input callback: hand over the one cached Opus packet exactly once, +/// then report "no more data" so the converter drains and returns what it decoded. +private func nimoOpusInputProc( + _: AudioConverterRef, + _ ioNumberDataPackets: UnsafeMutablePointer, + _ ioData: UnsafeMutablePointer, + _ outDataPacketDescription: UnsafeMutablePointer?>?, + _ inUserData: UnsafeMutableRawPointer? +) -> OSStatus { + guard let inUserData else { + ioNumberDataPackets.pointee = 0 + return kNimoOpusNoMoreData + } + let ctx = Unmanaged.fromOpaque(inUserData).takeUnretainedValue() + if ctx.consumed || ctx.buffer == nil { + ioNumberDataPackets.pointee = 0 + return kNimoOpusNoMoreData + } + ctx.consumed = true + ioData.pointee.mNumberBuffers = 1 + ioData.pointee.mBuffers.mNumberChannels = 1 + ioData.pointee.mBuffers.mDataByteSize = UInt32(ctx.byteSize) + ioData.pointee.mBuffers.mData = ctx.buffer + ioNumberDataPackets.pointee = 1 + outDataPacketDescription?.pointee = ctx.packetDescPtr + return noErr +} -/// Decodes the glasses' 16 kHz mono Opus frames to 16-bit PCM via the system -/// Opus decoder, so no native Opus library has to be vendored. -private class NimoOpusDecoder { - private let opusFormat: AVAudioFormat - private let pcmFormat: AVAudioFormat - private var converter: AVAudioConverter? +/// Decodes the glasses' mono Opus frames to 16 kHz 16-bit PCM via the system Opus +/// decoder (AudioToolbox), so no native Opus library has to be vendored. +/// +/// Uses the low-level AudioConverter (not AVAudioConverter) specifically so the +/// OpusHead magic cookie can be set — Apple's Opus decoder needs it to initialize, +/// and AVAudioConverter has no way to provide it, so the AVAudioConverter path +/// rejected every packet with an AudioCodec bad-data error. Decodes at Opus' native +/// 48 kHz (the only rate the decoder supports) and downsamples to 16 kHz, mirroring +/// the Android MediaCodec path. +private final class NimoOpusDecoder { + private var converter: AudioConverterRef? + private let input = NimoOpusInputContext() + private var errorLogCount = 0 + private var zeroFrameCount = 0 + private var emittedFirst = false init?() { - var desc = AudioStreamBasicDescription( - mSampleRate: 16000, - mFormatID: kAudioFormatOpus, - mFormatFlags: 0, - mBytesPerPacket: 0, - mFramesPerPacket: 320, // 20 ms at 16 kHz - mBytesPerFrame: 0, - mChannelsPerFrame: 1, - mBitsPerChannel: 0, - mReserved: 0 + var inASBD = AudioStreamBasicDescription( + mSampleRate: 48000, mFormatID: kAudioFormatOpus, mFormatFlags: 0, + mBytesPerPacket: 0, mFramesPerPacket: 960, mBytesPerFrame: 0, + mChannelsPerFrame: 1, mBitsPerChannel: 0, mReserved: 0 + ) + var outASBD = AudioStreamBasicDescription( + mSampleRate: 48000, mFormatID: kAudioFormatLinearPCM, + mFormatFlags: kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked, + mBytesPerPacket: 2, mFramesPerPacket: 1, mBytesPerFrame: 2, + mChannelsPerFrame: 1, mBitsPerChannel: 16, mReserved: 0 ) - guard let inFormat = AVAudioFormat(streamDescription: &desc), - let outFormat = AVAudioFormat( - commonFormat: .pcmFormatInt16, sampleRate: 16000, channels: 1, interleaved: true - ) - else { return nil } - opusFormat = inFormat - pcmFormat = outFormat - converter = AVAudioConverter(from: inFormat, to: outFormat) - if converter == nil { + + var conv: AudioConverterRef? + let status = AudioConverterNew(&inASBD, &outASBD, &conv) + guard status == noErr, let conv else { + Bridge.log("NIMO: AudioConverterNew(Opus) failed: \(status)") return nil } + converter = conv + + // OpusHead identification header (19 bytes) — the same structure the Android + // MediaCodec path passes as csd-0. Set as the decompression magic cookie so + // the Opus decoder can initialize. + var cookie: [UInt8] = Array("OpusHead".utf8) + cookie.append(1) // version + cookie.append(1) // channel count (mono) + cookie.append(contentsOf: [0, 0]) // pre-skip (LE) + let rate: UInt32 = 48000 // input sample rate (LE), matches the ASBD + cookie.append(UInt8(rate & 0xFF)) + cookie.append(UInt8((rate >> 8) & 0xFF)) + cookie.append(UInt8((rate >> 16) & 0xFF)) + cookie.append(UInt8((rate >> 24) & 0xFF)) + cookie.append(contentsOf: [0, 0]) // output gain (LE) + cookie.append(0) // channel mapping family + let cookieStatus = AudioConverterSetProperty( + conv, kAudioConverterDecompressionMagicCookie, UInt32(cookie.count), cookie + ) + if cookieStatus != noErr { + Bridge.log("NIMO: set Opus magic cookie failed: \(cookieStatus)") + } + Bridge.log("NIMO: Opus decoder ready (AudioToolbox @48k, OpusHead cookie)") + } + + deinit { + if let converter { AudioConverterDispose(converter) } } func decode(_ opusFrame: Data) -> Data? { guard let converter, !opusFrame.isEmpty else { return nil } - - let compressed = AVAudioCompressedBuffer( - format: opusFormat, packetCapacity: 1, maximumPacketSize: opusFrame.count - ) - opusFrame.withUnsafeBytes { (raw: UnsafeRawBufferPointer) in - compressed.data.copyMemory(from: raw.baseAddress!, byteCount: opusFrame.count) + input.set(opusFrame) + + let maxFrames: UInt32 = 2880 // up to 60 ms @ 48 kHz + var outSamples = [Int16](repeating: 0, count: Int(maxFrames)) + var producedFrames = maxFrames + var status: OSStatus = noErr + + outSamples.withUnsafeMutableBytes { rawOut in + var bufferList = AudioBufferList( + mNumberBuffers: 1, + mBuffers: AudioBuffer( + mNumberChannels: 1, + mDataByteSize: UInt32(rawOut.count), + mData: rawOut.baseAddress + ) + ) + let ctx = Unmanaged.passUnretained(input).toOpaque() + status = AudioConverterFillComplexBuffer( + converter, nimoOpusInputProc, ctx, &producedFrames, &bufferList, nil + ) } - compressed.byteLength = UInt32(opusFrame.count) - compressed.packetCount = 1 - compressed.packetDescriptions?.pointee = AudioStreamPacketDescription( - mStartOffset: 0, mVariableFramesInPacket: 0, mDataByteSize: UInt32(opusFrame.count) - ) - // Up to 40 ms of output per frame, to be safe. - guard let pcmBuffer = AVAudioPCMBuffer(pcmFormat: pcmFormat, frameCapacity: 640) else { - return nil - } - var fed = false - var error: NSError? - let status = converter.convert(to: pcmBuffer, error: &error) { _, outStatus in - if fed { - outStatus.pointee = .noDataNow - return nil + // Our input proc returns kNimoOpusNoMoreData once the packet is consumed; that + // is the normal terminator, not a failure. Anything else with no output is real. + if producedFrames == 0 { + if status != noErr, status != kNimoOpusNoMoreData { + errorLogCount += 1 + if errorLogCount == 1 || errorLogCount % 100 == 0 { + Bridge.log("NIMO: Opus decode error: \(status) (count=\(errorLogCount))") + } + } else { + zeroFrameCount += 1 + if zeroFrameCount == 1 || zeroFrameCount % 100 == 0 { + Bridge.log("NIMO: Opus decode produced 0 frames (status=\(status), count=\(zeroFrameCount))") + } } - fed = true - outStatus.pointee = .haveData - return compressed - } - if status == .error { - Bridge.log("NIMO: Opus decode error: \(error?.localizedDescription ?? "unknown")") return nil } - guard pcmBuffer.frameLength > 0, let channel = pcmBuffer.int16ChannelData else { - return nil + + let count = Int(producedFrames) + let pcm48 = Array(outSamples.prefix(count)).withUnsafeBufferPointer { Data(buffer: $0) } + // The MentraOS pipeline expects 16 kHz mono → decimate 48 kHz by 3. + let pcm16 = downsample48kTo16k(pcm48) + if !emittedFirst { + emittedFirst = true + Bridge.log( + "NIMO: first Opus PCM decoded (\(count) frames @48k → \(pcm16.count) bytes @16k)" + ) + } + return pcm16 + } + + /// 48 kHz → 16 kHz: average each group of 3 Int16 samples (cheap low-pass + + /// decimate). Matches the Android decoder's downsample48kTo16k. + private func downsample48kTo16k(_ pcm48: Data) -> Data { + let inSamples = pcm48.count / 2 + let outSamples = inSamples / 3 + if outSamples == 0 { return Data() } + var out = Data(count: outSamples * 2) + pcm48.withUnsafeBytes { (src: UnsafeRawBufferPointer) in + let s = src.bindMemory(to: Int16.self) + out.withUnsafeMutableBytes { (dst: UnsafeMutableRawBufferPointer) in + let d = dst.bindMemory(to: Int16.self) + for o in 0 ..< outSamples { + let i = o * 3 + let sum = Int(s[i]) + Int(s[i + 1]) + Int(s[i + 2]) + d[o] = Int16(max(-32768, min(32767, sum / 3))) + } + } } - return Data(bytes: channel[0], count: Int(pcmBuffer.frameLength) * 2) + return out } } @@ -582,6 +724,8 @@ class Nimo: NSObject, SGCManager { static let writeWatchdogSeconds: TimeInterval = 1 // Fall back to the other ear if the preferred one goes quiet for this long. static let micSideFallbackSeconds: TimeInterval = 2 + static let setTimeMaxAttempts = 3 + static let setTimeRetrySeconds: TimeInterval = 0.5 } // The text surface every sendTextWall renders into. The ASR note view (appId 0x04, @@ -627,6 +771,7 @@ class Nimo: NSObject, SGCManager { private var twsConnected = false private var twsTimeoutItem: DispatchWorkItem? private var pairingTimeoutItem: DispatchWorkItem? + private var setTimeAttempts = 0 // Pending acks keyed by (cmd << 8) | key private struct PendingAck { @@ -656,6 +801,10 @@ class Nimo: NSObject, SGCManager { private var textAppEntered = false private var navAppEntered = false private var currentGlassesAppId = -1 + // Last navigation HUD frame pushed, to coalesce the app's ~3s identical re-pushes. + private var lastNavHudKey = "" + // Arrow direction currently shown, so the icon is re-sent only when the turn changes. + private var lastNavArrow: NavArrow? // Battery private var lastBatteryLevel = -1 @@ -808,6 +957,10 @@ class Nimo: NSObject, SGCManager { pendingText = " " } + func sendText(_ text: String) async { + await sendTextWall(text) + } + func sendTextWall(_ text: String) async { // Coalesced: only the most recent pending text survives until the next 100ms drain. pendingText = text @@ -821,26 +974,38 @@ class Nimo: NSObject, SGCManager { _ text: String, x _: Int32, y _: Int32, width _: Int32, height _: Int32, borderWidth _: Int32, borderRadius _: Int32 ) async { - // Navigation pushes turn text via positioned_text. Nimo widgets have fixed geometry, so - // position/border are ignored — funnel the text through the same coalesced path as - // sendTextWall so it renders on the ASR note page. Without this override the base no-op - // silently dropped all navigation text. - // Bridge.log("NIMO: sendPositionedText(text=\(text))") - // pendingText = text + // The navigation app pushes its whole turn-by-turn HUD through positioned_text in one + // combined string (arrow / distance / instruction / ETA). Nimo widgets have fixed + // geometry, so x/y/border are ignored; instead we parse the string and fan the pieces + // out to the dedicated nav widgets (arrow icon, instruction, tip, status struct). + if handshakeState != .ready { return } + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + // Blank pushes (e.g. the app clearing its loading-message region) must not wipe the + // current HUD — ignore them and keep the last frame on screen. + if trimmed.isEmpty { return } + let hud = parseNavHud(trimmed) + // Coalesce: the app re-pushes the same frame every ~3s; only redraw on real changes. + let key = hud.key + if key == lastNavHudKey { return } + lastNavHudKey = key + Bridge.log( + "NIMO: nav HUD → arrow=\(hud.arrow) instr=\"\(hud.instruction)\" tip=\"\(hud.tip)\" eta=\"\(hud.eta)\"" + ) + enterNavAppIfNeeded() + pushNavHud(hud) } func displayBitmap( base64ImageData: String, x _: Int32?, y _: Int32?, width _: Int32?, height _: Int32? ) async -> Bool { // Nimo widgets have fixed geometry; x/y are not honored. Renders into the navigation - // large-map widget (appId 0x01, resId 0x05, 452x170 2bpp) — the full-width nav map, - // which shows far more than the small 160x160 mini-map (resId 0x00). bitmapToGrayscale - // aspect-fits onto black, so a non-matching source aspect letterboxes rather than - // distorts. The nav app must be foregrounded before pushing content (mirrors the text - // path). TODO: hardware-verify the large-map widget renders and the nav app-mode. - let targetWidth = NimoProtocol.NAV_LARGE_MAP_WIDTH - let targetHeight = NimoProtocol.NAV_LARGE_MAP_HEIGHT - Bridge.log("NIMO: displayBitmap → nav large map (navAppEntered=\(navAppEntered))") + // mini-map widget (appId 0x01, resId 0x00, 160x160 2bpp) — the compact "small screen" + // nav map. bitmapToGrayscale aspect-fits onto black, so a non-matching source aspect + // letterboxes rather than distorts. The nav app must be foregrounded before pushing + // content (mirrors the text path). TODO: hardware-verify the mini-map widget renders. + let targetWidth = NimoProtocol.NAV_MINI_MAP_SIZE + let targetHeight = NimoProtocol.NAV_MINI_MAP_SIZE + Bridge.log("NIMO: displayBitmap → nav mini map (navAppEntered=\(navAppEntered))") guard let imageData = Data(base64Encoded: base64ImageData), let image = UIImage(data: imageData) else { @@ -849,37 +1014,266 @@ class Nimo: NSObject, SGCManager { } guard let grayscale = bitmapToGrayscale(image, width: targetWidth, height: targetHeight) else { return false } - let packed = packL8To2bpp(grayscale) - let (payload, compression) = compressAdaptive(packed) - if !navAppEntered { - sendFrame( - NimoFrameCodec.encodeFrame( - cmd: NimoProtocol.CMD_CONTROL_INSTRUCTION, - key: NimoProtocol.CTRL_ENTER_APP, - payload: Data([UInt8(NimoProtocol.APP_ID_NAV), UInt8(NimoProtocol.APP_MODE_STANDALONE)]) - ) + let content = buildNavImageContent(grayscale, width: targetWidth, height: targetHeight) + enterNavAppIfNeeded() + enqueueNavWidget(resId: NimoProtocol.NAV_RES_MINI_MAP, resType: NimoProtocol.WIDGET_PICTURE, content: content) + return true + } + + /// Foregrounds the nav app (appId 0x01) once; corrected by app-state reports. + private func enterNavAppIfNeeded() { + if navAppEntered { return } + sendFrame( + NimoFrameCodec.encodeFrame( + cmd: NimoProtocol.CMD_CONTROL_INSTRUCTION, + key: NimoProtocol.CTRL_ENTER_APP, + payload: Data([UInt8(NimoProtocol.APP_ID_NAV), UInt8(NimoProtocol.APP_MODE_STANDALONE)]) ) - // Optimistic; corrected by app-state reports if the glasses refuse/exit. - navAppEntered = true - } - let content = - NimoFrameCodec.imageHeader( - width: targetWidth, - height: targetHeight, - formatBpp: NimoProtocol.FORMAT_2BPP, - compression: compression, - originalSize: packed.count, - compressedSize: compression == NimoProtocol.COMPRESSION_NONE ? 0 : payload.count - ) + payload - let frames = NimoFrameCodec.updateContentFrames( - appId: NimoProtocol.APP_ID_NAV, - layoutId: 0, - resId: NimoProtocol.NAV_RES_LARGE_MAP, - resType: NimoProtocol.WIDGET_PICTURE, - content: content ) - enqueueFrames(frames) - return true + navAppEntered = true + } + + // MARK: - Navigation HUD Parsing + + /// Turn direction for the arrow icon; none renders a blank icon (no arrow). + private enum NavArrow: String { + case left + case right + case straight + case none + } + + /// The parsed pieces of one navigation HUD frame. + private struct NavHud { + let arrow: NavArrow + let instruction: String + let tip: String + let distance: String + let eta: String + // Whether to show the status struct (dist/ETA bar). Off for welcome/arrival frames. + let showStatus: Bool + + // Coalesce key: identical frames must produce the same string (mirrors Kotlin toString()). + var key: String { + "\(arrow.rawValue)|\(instruction)|\(tip)|\(distance)|\(eta)|\(showStatus)" + } + } + + /// Parses the navigation app's combined HUD string into the pieces the Nimo nav widgets + /// want. The app crams the whole frame into one positioned_text push, e.g.: + /// + /// → ← directional arrow glyph (←/→/↑) + /// In 198 m ← distance-to-turn ("In X" / "Now" / "Arriving in X") + /// Turn right onto Octavia St + /// ← blank line + /// ⊙ 14 min ← trip ETA + /// + /// Non-turn frames (Welcome / Rerouting… / Go back / Starting… / You have arrived …) have no + /// arrow and no ⊙ line; their text falls through to NavHud.instruction. + /// + /// The instruction is kept short (≤3 words — "Turn right", "Welcome to Maps", "Arrived"); any + /// "onto " suffix is moved into the tip line next to the distance so the big line stays + /// a glanceable maneuver. + private func parseNavHud(_ text: String) -> NavHud { + var eta = "" + var content: [String] = [] + for raw in text.split(separator: "\n", omittingEmptySubsequences: false) { + let line = String(raw).trimmingCharacters(in: .whitespaces) + if line.isEmpty { continue } + // Drop the arrow GLYPH line: the app forces it to "↑" until ~10 m before the turn, + // so it reads "straight" for almost the whole leg. We derive the real direction + // from the maneuver instruction instead (see arrowForInstruction). + if line == "←" || line == "→" || line == "↑" || line == "↺" || line == "⇆" { + continue + } + if line.hasPrefix("⊙") { + eta = String(line.dropFirst()).trimmingCharacters(in: .whitespaces) + } else { + content.append(line) + } + } + // The distance-to-turn line (used as the tip) vs the instruction line. + let distLine = content.first { line in + let lower = line.lowercased() + return lower.hasPrefix("in ") || lower == "now" || lower.hasPrefix("arriving") + } ?? "" + let rawInstruction = content.filter { $0 != distLine }.joined(separator: " ") + let (instruction, road) = shortenInstruction(rawInstruction) + // Arrow direction from the maneuver text, so it points at the turn the whole approach. + let arrow = arrowForInstruction(instruction) + // Welcome/arrival frames (arrow == none) carry no live nav data — hide the status bar. + let showStatus = arrow != .none + // Tip = distance line + road name (e.g. "In 198 m on Octavia St"); welcome gets a prompt. + var tip = [distLine, road].filter { !$0.isEmpty }.joined(separator: " on ") + if instruction.hasPrefix("Welcome") { tip = "Choose a location on the map" } + // Numeric distance for the status struct: strip the "In " / "Arriving in " prefix. + var distance = distLine + if let range = distance.range( + of: "^(arriving\\s+in|in)\\s+", options: [.regularExpression, .caseInsensitive] + ) { + distance.removeSubrange(range) + } + distance = distance.trimmingCharacters(in: .whitespaces) + return NavHud( + arrow: arrow, instruction: instruction, tip: tip, distance: distance, eta: eta, + showStatus: showStatus + ) + } + + /// Turn direction inferred from the maneuver phrase ("Turn left" → left, else straight). + private func arrowForInstruction(_ instruction: String) -> NavArrow { + let lower = instruction.lowercased() + if lower.contains("welcome") || lower.contains("arrived") { return .none } + if lower.contains("left") { return .left } + if lower.contains("right") { return .right } + return .straight + } + + /// Reduces a raw instruction to a short maneuver phrase (≤3 words) and the road name it + /// referenced. "Turn right onto Octavia St" → ("Turn right", "Octavia St"); welcome/arrival + /// frames collapse to fixed short labels. + private func shortenInstruction(_ raw: String) -> (String, String) { + let trimmed = raw.trimmingCharacters(in: .whitespaces) + if trimmed.isEmpty { return ("", "") } + let lower = trimmed.lowercased() + if lower.contains("welcome") { return ("Welcome to Maps", "") } + if lower.contains("arrived") { return ("Arrived!", "") } + // Peel off the "onto " suffix (or a leading "Onto ") into the road name. + var maneuver = trimmed + var road = "" + if let ontoRange = trimmed.range(of: " onto ", options: .caseInsensitive) { + maneuver = String(trimmed[trimmed.startIndex ..< ontoRange.lowerBound]) + .trimmingCharacters(in: .whitespaces) + road = String(trimmed[ontoRange.upperBound...]).trimmingCharacters(in: .whitespaces) + } else if lower.hasPrefix("onto ") { + road = String(trimmed.dropFirst(5)).trimmingCharacters(in: .whitespaces) + maneuver = "" + } + let capped = maneuver.split(whereSeparator: { $0 == " " || $0 == "\t" }) + .prefix(3).joined(separator: " ") + return (capped, road) + } + + /// Pushes a parsed HUD frame into the nav widgets (arrow icon / instruction / tip / info). + private func pushNavHud(_ hud: NavHud) { + // naviInstruction (0x02): primary turn text. + enqueueNavWidget( + resId: NimoProtocol.NAV_RES_TURN_TEXT, + resType: NimoProtocol.WIDGET_TEXT_NEW, + content: Data((hud.instruction.isEmpty ? " " : hud.instruction).utf8) + ) + // naviInstructionTip (0x04): distance-to-turn line. + enqueueNavWidget( + resId: NimoProtocol.NAV_RES_TIP, + resType: NimoProtocol.WIDGET_TEXT_NEW, + content: Data((hud.tip.isEmpty ? " " : hud.tip).utf8) + ) + // naviInfo (0x03): 65-byte status struct — navType(1) + dist(32) + time(32). Skipped on + // welcome/arrival frames so no (empty) status bar shows there. + if hud.showStatus { + var info = Data(count: NimoProtocol.NAV_INFO_STRUCT_SIZE) + info[0] = UInt8(NimoProtocol.NAV_TYPE_WALK) + writeUtf8Padded(&info, offset: 1, text: hud.distance, maxLen: 32) + writeUtf8Padded(&info, offset: 33, text: hud.eta, maxLen: 32) + enqueueNavWidget(resId: NimoProtocol.NAV_RES_INFO, resType: NimoProtocol.WIDGET_RAW, content: info) + } + // naviIcon (0x01): turn-arrow. Only re-send when the DIRECTION changes — re-pushing the + // identical image on every distance tick made the arrow flicker/"move". The bitmap for + // each direction is rendered once and cached. + if hud.arrow != lastNavArrow { + lastNavArrow = hud.arrow + enqueueNavWidget( + resId: NimoProtocol.NAV_RES_ARROW, + resType: NimoProtocol.WIDGET_PICTURE, + content: navArrowContent(hud.arrow) + ) + } + } + + // Arrow widget content for every direction, rendered ONCE on first use and reused — the + // bitmaps are deterministic, so the send path just selects the pre-built bytes instead of + // re-rendering on each turn. + private lazy var navArrowCache: [NavArrow: Data] = { + var cache: [NavArrow: Data] = [:] + for arrow in [NavArrow.left, .right, .straight, .none] { + guard let bmp = buildNavArrowBitmap(arrow), + let gray = bitmapToGrayscale( + bmp, width: NimoProtocol.NAV_ARROW_SIZE, height: NimoProtocol.NAV_ARROW_SIZE + ) + else { continue } + cache[arrow] = buildNavImageContent( + gray, width: NimoProtocol.NAV_ARROW_SIZE, height: NimoProtocol.NAV_ARROW_SIZE + ) + } + return cache + }() + + private func navArrowContent(_ arrow: NavArrow) -> Data { + return navArrowCache[arrow] ?? Data() + } + + /// A 48x48 white turn-arrow on black, rotated to point left/right/up per `arrow`. + private func buildNavArrowBitmap(_ arrow: NavArrow) -> UIImage? { + let size = NimoProtocol.NAV_ARROW_SIZE + let s = CGFloat(size) + let renderer = UIGraphicsImageRenderer(size: CGSize(width: s, height: s)) + return renderer.image { ctx in + let cg = ctx.cgContext + cg.setFillColor(UIColor.black.cgColor) + cg.fill(CGRect(x: 0, y: 0, width: s, height: s)) + // none → leave the icon blank (all black = nothing lit) for welcome/arrival frames. + if arrow == .none { return } + // Rotate the (upward) arrow around the center to face the turn direction. + let rotation: CGFloat + switch arrow { + case .left: rotation = -.pi / 2 + case .right: rotation = .pi / 2 + default: rotation = 0 + } + cg.translateBy(x: s / 2, y: s / 2) + cg.rotate(by: rotation) + cg.translateBy(x: -s / 2, y: -s / 2) + let path = UIBezierPath() + path.move(to: CGPoint(x: s * 0.5, y: s * 0.15)) // tip + path.addLine(to: CGPoint(x: s * 0.85, y: s * 0.55)) // head bottom-right + path.addLine(to: CGPoint(x: s * 0.62, y: s * 0.55)) + path.addLine(to: CGPoint(x: s * 0.62, y: s * 0.85)) // shaft bottom-right + path.addLine(to: CGPoint(x: s * 0.38, y: s * 0.85)) // shaft bottom-left + path.addLine(to: CGPoint(x: s * 0.38, y: s * 0.55)) + path.addLine(to: CGPoint(x: s * 0.15, y: s * 0.55)) // head bottom-left + path.close() + UIColor.white.setFill() + path.fill() + } + } + + // MARK: - Nav Widget Helpers + + /// Enqueues a single nav-app content widget (chunked, appId 0x01, layout 0). + private func enqueueNavWidget(resId: Int, resType: Int, content: Data) { + enqueueFrames( + NimoFrameCodec.updateContentFrames( + appId: NimoProtocol.APP_ID_NAV, + layoutId: 0, + resId: resId, + resType: resType, + content: content + ) + ) + } + + /// Builds the [imageHeader + payload] blob for a nav image widget from an L8 buffer. + private func buildNavImageContent(_ grayscale: Data, width: Int, height: Int) -> Data { + let packed = packL8To2bpp(grayscale) + let (payload, compression) = compressAdaptive(packed) + return NimoFrameCodec.imageHeader( + width: width, + height: height, + formatBpp: NimoProtocol.FORMAT_2BPP, + compression: compression, + originalSize: packed.count, + compressedSize: compression == NimoProtocol.COMPRESSION_NONE ? 0 : payload.count + ) + payload } func showDashboard() { @@ -918,22 +1312,15 @@ class Nimo: NSObject, SGCManager { // MARK: - SGCManager: Device Control func setHeadUpAngle(_ angle: Int) { - let clamped = max(0, min(90, angle)) - Bridge.log("NIMO: setHeadUpAngle(\(clamped))") - // Enable the head-up display gesture, then set the wake angle. + Bridge.log("NIMO: setHeadUpAngle(\(angle)) — head-up display disabled for now") + // Head-up display is intentionally kept OFF for now (it auto-wakes the dashboard and the + // wake-angle semantics aren't dialed in). Force it disabled regardless of the requested + // angle. TODO: restore the enable + setAngle [optType, deg] path once verified. sendFrame( NimoFrameCodec.encodeFrame( cmd: NimoProtocol.CMD_SET_PARAMETER, key: NimoProtocol.SET_HEADUP_DISPLAY, - payload: Data([1]) - ) - ) - // setAngle payload is [optType, deg]. TODO: hardware-verify optType semantics. - sendFrame( - NimoFrameCodec.encodeFrame( - cmd: NimoProtocol.CMD_SET_PARAMETER, - key: NimoProtocol.SET_ANGLE, - payload: Data([0x01, UInt8(clamped)]) + payload: Data([0]) ) ) } @@ -1024,11 +1411,7 @@ class Nimo: NSObject, SGCManager { // MARK: - SGCManager: Camera & Media (no camera) - func requestPhoto( - _: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, - compress _: String?, flash _: Bool, save _: Bool, sound _: Bool, exposureTimeNs _: Double?, - iso _: Int? - ) { + func requestPhoto(_: PhotoRequest) { Bridge.log("NIMO: requestPhoto - not supported (no camera)") } @@ -1042,7 +1425,7 @@ class Nimo: NSObject, SGCManager { func sendStreamKeepAlive(_: [String: Any]) {} - func startVideoRecording(requestId _: String, save _: Bool, flash _: Bool, sound _: Bool) { + func startVideoRecording(requestId _: String, save _: Bool, sound _: Bool) { Bridge.log("NIMO: startVideoRecording - not supported") } @@ -1062,7 +1445,7 @@ class Nimo: NSObject, SGCManager { func sendWifiCredentials(_: String, _: String) {} func forgetWifiNetwork(_: String) {} func sendHotspotState(_: Bool) {} - func sendOtaStart() {} + func sendOtaStart(otaVersionUrl _: String?) {} func sendOtaQueryStatus() {} // MARK: - SGCManager: User Context / Gallery / Version @@ -1112,6 +1495,19 @@ class Nimo: NSObject, SGCManager { return true } + // The main Nimo device does NOT advertise its name (or itself) over BLE — + // only the "_ble" ANCS side channel does. On Android the vendor + // discovers it over CLASSIC (BR/EDR) inquiry; iOS can't do classic + // discovery, so instead the glasses are bonded over classic in the iOS + // Settings app, which iOS keeps connected at the system level. + // retrieveConnectedPeripherals surfaces that system-connected device here — + // the primary iOS discovery path (mirrors the vendor's getConnectedDevice(7033)). + checkSystemConnectedDevices() + + // Scan unfiltered and identify the Nimo by name OR advertised service UUID + // (see didDiscover). Filtering the scan by [7033] would silently miss the + // device if it exposes the UART service only in GATT and not in its LE + // advertising packet — so we scan everything and inspect each advert. centralManager!.scanForPeripherals( withServices: nil, options: [CBCentralManagerScanOptionAllowDuplicatesKey: false] @@ -1119,6 +1515,56 @@ class Nimo: NSObject, SGCManager { return true } + /// Surfaces (and auto-connects) Nimo glasses already connected/paired at the + /// iOS system level. The device is bonded over classic BT in Settings and does + /// not re-advertise its name over LE, so a plain scan can't find it — but + /// retrieveConnectedPeripherals(withServices:) returns it with a resolved name. + private func checkSystemConnectedDevices() { + guard let centralManager else { return } + let connected = centralManager.retrieveConnectedPeripherals( + withServices: [NimoBLE.SERVICE_UUID] + ) + Bridge.log( + "NIMO: retrieveConnectedPeripherals([7033]) → \(connected.count) device(s); " + + "target=\(DEVICE_SEARCH_ID)" + ) + for peripheral in connected { + let name = peripheral.name ?? "" + Bridge.log("NIMO: system-connected candidate \(name) (\(peripheral.identifier))") + // System-connected peripherals carry a resolved name; if one is missing + // it can't be name-matched, so connect directly during a targeted connect. + if let realName = peripheral.name { + handleDiscoveredPeripheral(peripheral, name: realName, rssi: 0) + } else if DEVICE_SEARCH_ID != "NOT_SET", self.peripheral == nil { + stopScan() + self.peripheral = peripheral + peripheral.delegate = self + centralManager.connect(peripheral, options: nil) + } + } + } + + /// Shared handling for a candidate Nimo peripheral from either the active scan + /// or the system-connected list: report it to the UI, and auto-connect when it + /// matches the requested device id. Must run on the main queue. + private func handleDiscoveredPeripheral(_ peripheral: CBPeripheral, name: String, rssi: Int) { + guard isNimoMainDevice(name) else { return } + + Bridge.sendDiscoveredDevice(DeviceTypes.NIMO, name, rssi: rssi) + + guard DEVICE_SEARCH_ID != "NOT_SET" else { return } + guard name == DEVICE_SEARCH_ID else { return } + guard self.peripheral == nil else { return } + + Bridge.log("NIMO: Connecting to \(name)") + stopScan() + lastDeviceName = name + lastDeviceUUID = peripheral.identifier.uuidString + self.peripheral = peripheral + peripheral.delegate = self + centralManager?.connect(peripheral, options: nil) + } + private func connectByUUID() -> Bool { guard DEVICE_SEARCH_ID != "NOT_SET", !DEVICE_SEARCH_ID.isEmpty else { return false } guard lastDeviceName == DEVICE_SEARCH_ID, @@ -1278,17 +1724,33 @@ class Nimo: NSObject, SGCManager { cancelTwsTimeout() handshakeState = .awaitingTimeAck Bridge.log("NIMO: TWS OK — sending setTime (awaiting ACK)") + setTimeAttempts = 0 + attemptSetTime() + } + + private func attemptSetTime() { + guard handshakeState == .awaitingTimeAck, peripheral != nil else { return } + setTimeAttempts += 1 sendAwaitingAck( cmd: NimoProtocol.CMD_SET_PARAMETER, key: NimoProtocol.SET_TIME, payload: NimoFrameCodec.encodeDeviceTime() ) { [weak self] ok in guard let self else { return } - if !ok { - Bridge.log("NIMO: setTime ACK failed/timed out — handshake failed") - self.handshakeFailed() - } else { + if ok { self.finishHandshake() + } else if self.setTimeAttempts < Const.setTimeMaxAttempts, + self.handshakeState == .awaitingTimeAck, + self.peripheral != nil { + // The firmware can answer "busy" right after the link comes up — + // give it a moment and retry instead of dropping the connection. + Bridge.log("NIMO: setTime attempt \(self.setTimeAttempts) failed — retrying") + DispatchQueue.main.asyncAfter(deadline: .now() + Const.setTimeRetrySeconds) { + [weak self] in self?.attemptSetTime() + } + } else { + Bridge.log("NIMO: setTime failed after \(self.setTimeAttempts) attempts — handshake failed") + self.handshakeFailed() } } } @@ -1304,6 +1766,16 @@ class Nimo: NSObject, SGCManager { needsAck: false ) ) + // Disable the on-device head-up gesture so raising your head doesn't auto-wake the + // dashboard. Disabled for now; re-enable once the wake-angle behaviour is dialed in. + sendFrame( + NimoFrameCodec.encodeFrame( + cmd: NimoProtocol.CMD_SET_PARAMETER, + key: NimoProtocol.SET_HEADUP_DISPLAY, + payload: Data([0]), + needsAck: false + ) + ) getBatteryStatus() requestVersionInfo() @@ -1329,6 +1801,8 @@ class Nimo: NSObject, SGCManager { textAppEntered = false navAppEntered = false currentGlassesAppId = -1 + lastNavHudKey = "" + lastNavArrow = nil pendingText = nil receiveAssembler.reset() writeQueue.removeAll() @@ -1451,11 +1925,20 @@ class Nimo: NSObject, SGCManager { case NimoProtocol.STATE_ENTER: currentGlassesAppId = appId if appId != textAppId { textAppEntered = false } - if appId != NimoProtocol.APP_ID_NAV { navAppEntered = false } + if appId != NimoProtocol.APP_ID_NAV { + navAppEntered = false + // Force a full HUD repaint next time the nav app reopens. + lastNavHudKey = "" + lastNavArrow = nil + } case NimoProtocol.STATE_EXIT: if appId == currentGlassesAppId { currentGlassesAppId = -1 } if appId == textAppId { textAppEntered = false } - if appId == NimoProtocol.APP_ID_NAV { navAppEntered = false } + if appId == NimoProtocol.APP_ID_NAV { + navAppEntered = false + lastNavHudKey = "" + lastNavArrow = nil + } default: break } @@ -1511,10 +1994,13 @@ class Nimo: NSObject, SGCManager { switch code { case NimoProtocol.INPUT_HEAD_UP: DeviceStore.shared.apply("glasses", "headUp", true) - Bridge.sendHeadUp(true) + // Head-up forwarding to the cloud is disabled for now (the wake-angle + // semantics aren't dialed in yet). Keep the local state; don't notify. + // Bridge.sendHeadUp(true) case NimoProtocol.INPUT_HEAD_DOWN: DeviceStore.shared.apply("glasses", "headUp", false) - Bridge.sendHeadUp(false) + // Head-up forwarding to the cloud is disabled for now (see INPUT_HEAD_UP). + // Bridge.sendHeadUp(false) case NimoProtocol.INPUT_CLICK_RIGHT, NimoProtocol.INPUT_CLICK_LEFT: Bridge.sendTouchEvent( deviceModel: DeviceTypes.NIMO, gestureName: "single_tap", timestamp: timestamp @@ -1765,29 +2251,41 @@ extension Nimo: CBCentralManagerDelegate { advertisementData: [String: Any], rssi: NSNumber ) { - guard - let name = peripheral.name - ?? advertisementData[CBAdvertisementDataLocalNameKey] as? String - else { return } + // The main device advertises service 7033 but no local name; fall back to + // the peripheral's (system-resolved) name, which may still be nil here. + let name = peripheral.name + ?? advertisementData[CBAdvertisementDataLocalNameKey] as? String + let advServices = + advertisementData[CBAdvertisementDataServiceUUIDsKey] as? [CBUUID] ?? [] + let advertisesUart = advServices.contains(NimoBLE.SERVICE_UUID) let rssiValue = rssi.intValue DispatchQueue.main.async { [weak self] in guard let self else { return } - guard self.isNimoMainDevice(name) else { return } - - Bridge.sendDiscoveredDevice(DeviceTypes.NIMO, name, rssi: rssiValue) - - guard self.DEVICE_SEARCH_ID != "NOT_SET" else { return } - guard name == self.DEVICE_SEARCH_ID else { return } - guard self.peripheral == nil else { return } + let nameMatch = name.map(self.isNimoMainDevice) ?? false + // Only log plausible candidates to avoid flooding the log with every + // nearby BLE device. + if nameMatch || advertisesUart { + Bridge.log( + "NIMO: discovered name=\(name ?? "") uartAdv=\(advertisesUart) " + + "services=\(advServices.map { $0.uuidString })" + ) + } - Bridge.log("NIMO: Connecting to \(name)") + if let name, nameMatch { + self.handleDiscoveredPeripheral(peripheral, name: name, rssi: rssiValue) + return + } + // Nameless but advertises the Nimo-exclusive UART service → it IS a Nimo + // main device that just doesn't broadcast a name over LE. During a + // targeted connect, connect to resolve the name (saved in didConnect). + guard advertisesUart, self.DEVICE_SEARCH_ID != "NOT_SET", self.peripheral == nil + else { return } + Bridge.log("NIMO: nameless UART-service match — connecting to resolve name") self.stopScan() - self.lastDeviceName = name - self.lastDeviceUUID = peripheral.identifier.uuidString self.peripheral = peripheral peripheral.delegate = self - central.connect(peripheral, options: nil) + self.centralManager?.connect(peripheral, options: nil) } } @@ -1795,6 +2293,11 @@ extension Nimo: CBCentralManagerDelegate { DispatchQueue.main.async { [weak self] in guard let self else { return } Bridge.log("NIMO: Connected to \(peripheral.name ?? "unknown")") + // Persist the now-resolved name so the next launch can UUID-fast-path, + // and so a connection made via the nameless service match is remembered. + if let name = peripheral.name { + self.lastDeviceName = name + } self.lastDeviceUUID = peripheral.identifier.uuidString peripheral.discoverServices([NimoBLE.SERVICE_UUID]) } diff --git a/mobile/src/app/pairing/btclassic.tsx b/mobile/src/app/pairing/btclassic.tsx index a6f4fb689f..c8a4c5fc45 100644 --- a/mobile/src/app/pairing/btclassic.tsx +++ b/mobile/src/app/pairing/btclassic.tsx @@ -1,10 +1,11 @@ -import {useEffect} from "react" +import {useEffect, useRef} from "react" import {Button, Screen} from "@/components/ignite" import {OnboardingGuide, OnboardingStep} from "@/components/onboarding/OnboardingGuide" import {translate} from "@/i18n" import {focusEffectPreventBack, usePushPrevious} from "@/contexts/NavigationHistoryContext" -import {useGlassesStore} from "@/stores/glasses" -import BluetoothSdk from "@mentra/bluetooth-sdk" +import {selectGlassesConnected, useGlassesStore} from "@/stores/glasses" +import {DeviceTypes} from "@/../../cloud/packages/types/src" +import BluetoothSdk, {type DeviceModel} from "@mentra/bluetooth-sdk" import {SETTINGS, useSetting} from "@/stores/settings" import {SettingsNavigationUtils} from "@/utils/SettingsNavigationUtils" import {useCoreStore} from "@/stores/core" @@ -17,10 +18,21 @@ export default function BtClassicPairingScreen() { const {goBack} = useNavigationStore.getState() const pushPrevious = usePushPrevious() const bluetoothClassicConnected = useGlassesStore((state) => state.bluetoothClassicConnected) + const glassesConnected = useGlassesStore(selectGlassesConnected) const otherBtConnected = useCoreStore((state) => state.otherBtConnected) + const searchResults = useCoreStore((state) => state.searchResults) const [deviceName] = useSetting(SETTINGS.device_name.key) + const [defaultWearable] = useSetting(SETTINGS.default_wearable.key) const {theme} = useAppTheme() + // Nimo has no classic AUDIO profile, so `bluetoothClassicConnected` never flips + // (that flag is set from iOS audio-session detection). Instead we drive the whole + // flow off the real BLE connection: keep re-scanning until the glasses appear as + // system-connected after the user pairs in Settings, then connect and complete. + const isNimo = defaultWearable === DeviceTypes.NIMO + const nimoConnectTriggeredRef = useRef(false) + const nimoAdvancedRef = useRef(false) + focusEffectPreventBack() const handleSuccess = () => { @@ -57,6 +69,46 @@ export default function BtClassicPairingScreen() { } }, [deviceName]) + // Nimo: poll a non-destructive scan (startScan does NOT arm the connect-time + // pairing timeout the way connect does) so the glasses can be detected as + // system-connected as soon as the user finishes pairing in Settings. + useEffect(() => { + if (!isNimo || !defaultWearable) return + if (glassesConnected) return + let cancelled = false + const scanOnce = () => { + BluetoothSdk.startScan(defaultWearable as DeviceModel).catch(() => undefined) + } + scanOnce() + const interval = setInterval(() => { + if (!cancelled) scanOnce() + }, 3000) + return () => { + cancelled = true + clearInterval(interval) + } + }, [isNimo, defaultWearable, glassesConnected]) + + // Nimo: once our glasses show up in the scan results (system-connected after the + // Settings pairing), connect once. They're already present, so this connects fast. + useEffect(() => { + if (!isNimo || nimoConnectTriggeredRef.current) return + if (!searchResults.some((d) => d.name === deviceName)) return + nimoConnectTriggeredRef.current = true + BluetoothSdk.connectDefault().catch((error) => { + console.error("Failed to connect Nimo after Bluetooth Classic pairing:", error) + }) + }, [isNimo, searchResults, deviceName]) + + // Nimo: complete on the real glasses connection (not the audio-only classic flag). + // Hand back to the loading screen underneath, which advances to success. + useEffect(() => { + if (!isNimo || nimoAdvancedRef.current) return + if (!glassesConnected) return + nimoAdvancedRef.current = true + pushPrevious() + }, [isNimo, glassesConnected]) + let steps: OnboardingStep[] = [ { type: "image", diff --git a/mobile/src/app/pairing/scan.tsx b/mobile/src/app/pairing/scan.tsx index 233b695e4a..6497092d94 100644 --- a/mobile/src/app/pairing/scan.tsx +++ b/mobile/src/app/pairing/scan.tsx @@ -98,7 +98,7 @@ export default function SelectGlassesBluetoothScreen() { } const startPairing = async (device: Device) => { - const deviceTypesWithBtClassic = [DeviceTypes.LIVE] + const deviceTypesWithBtClassic = [DeviceTypes.LIVE, DeviceTypes.NIMO] if (Platform.OS === "android" || bluetoothClassicConnected || !deviceTypesWithBtClassic.includes(device.model as DeviceTypes)) { setTimeout(() => { BluetoothSdk.connect(device).catch((error) => { diff --git a/mobile/src/app/pairing/select-glasses-model.tsx b/mobile/src/app/pairing/select-glasses-model.tsx index 782bb24aab..30c5e1cb3e 100644 --- a/mobile/src/app/pairing/select-glasses-model.tsx +++ b/mobile/src/app/pairing/select-glasses-model.tsx @@ -53,7 +53,7 @@ export default function SelectGlassesModelScreen() { } // Glasses models that should only be visible in super mode. - const SUPER_MODE_ONLY_MODELS = new Set([DeviceTypes.NEX, DeviceTypes.NIMO]) + const SUPER_MODE_ONLY_MODELS = new Set([DeviceTypes.NEX]) // Platform-specific glasses options const glassesOptions = diff --git a/mobile/src/services/cloudClient.ts b/mobile/src/services/cloudClient.ts index ec3bcfec06..d5d039354d 100644 --- a/mobile/src/services/cloudClient.ts +++ b/mobile/src/services/cloudClient.ts @@ -32,8 +32,10 @@ type Lc3FrameSizeBytes = 20 | 40 | 60 // Team-friendly defaults for dev builds. Local Cloud V2 is still one tap away // via the METRO_AUTO dev-settings preset; it should not be the invisible // default because it depends on a local stack plus adb reverse/LAN reachability. -const DEFAULT_CORE_URL = "https://core.dev.us-west-2.mentraglass.com" -const DEFAULT_RUNTIME_URL = "https://runtime.dev.us-west-2.mentraglass.com" +// Default to the DEBUG cloud (not dev) so the navigation app lands on debug +// Cloud V2 unless an explicit override/env is set. +const DEFAULT_CORE_URL = "https://core.debug.us-west-2.mentraglass.com" +const DEFAULT_RUNTIME_URL = "https://runtime.debug.us-west-2.mentraglass.com" const CORE_PORT = 3000 const RUNTIME_PORT = 3001