From 280242d0576c14ec887d6c784aba59d7bbe00ac3 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Sun, 18 Oct 2026 13:29:05 +0200 Subject: [PATCH 01/90] Pairs: more than one pair on a phone. Visor remembers the pair it connected to last and keeps each pair's MobileConfig, settings, feature switches, languages, wake word and readings apart; the home swipes between pairs, each shown as Meta pictures it with its name, batteries and Connect, then a page to set up another; a name given on the home reaches the glasses at their next connection (#15) --- .../visor/connection/GlassesService.kt | 12 ++ .../com/macedonga/visor/features/Features.kt | 27 ++- .../com/macedonga/visor/home/HomeFeed.kt | 91 +++++++- .../com/macedonga/visor/home/HomeScreen.kt | 172 +++++++++++---- .../com/macedonga/visor/home/LastSeen.kt | 202 +++++++++++++++--- .../visor/language/LanguageInstaller.kt | 17 +- .../com/macedonga/visor/pairs/PairPrefs.kt | 34 +++ .../kotlin/com/macedonga/visor/pairs/Pairs.kt | 140 ++++++++++++ .../macedonga/visor/settings/GlassesPage.kt | 18 +- .../visor/settings/SettingsScreen.kt | 8 +- .../com/macedonga/visor/settings/WakeWord.kt | 6 +- .../com/macedonga/visor/stage/Frames.kt | 95 +++++++- .../macedonga/visor/stage/GlassesRenderer.kt | 7 +- .../com/macedonga/visor/stage/SharedStage.kt | 99 ++++++++- .../com/macedonga/visor/stage/StageHolder.kt | 28 +++ app/src/main/res/values-it/strings.xml | 4 + app/src/main/res/values/strings.xml | 4 + .../core/connection/GlassesConnection.kt | 34 ++- .../visor/core/identity/IdentityStore.kt | 19 ++ .../core/identity/language/PairPicture.kt | 70 ++++++ .../visor/core/media/GlassesConfig.kt | 34 ++- .../visor/core/settings/GlassesSettings.kt | 21 +- 22 files changed, 1005 insertions(+), 137 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/pairs/PairPrefs.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/pairs/Pairs.kt create mode 100644 core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/PairPicture.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt index 79b08104..cd0f1e09 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt @@ -135,10 +135,22 @@ class GlassesService : Service() { } } else if (intent?.action == ACTION_RECONNECT) { retry() + } else { + targetFrom(intent)?.let(::switchTo) } return START_STICKY } + /** Another pair asked for: the session with this one ends first, as the glasses hold one each. */ + private fun switchTo(target: GlassesTarget) { + if (target.serial == connection.state.value.target?.serial) return + if (live.running.value || media.importing.value) return + scope.launch { + connection.stop() + connection.start(scope, target) + } + } + /** * Asked from the tile: a pair the ladder is waiting to try again is tried now. Never while the * Live or an import has the glasses over Wi-Fi: Bluetooth is paused for them, and a second session ends theirs. diff --git a/app/src/main/kotlin/com/macedonga/visor/features/Features.kt b/app/src/main/kotlin/com/macedonga/visor/features/Features.kt index 05886b50..a2809777 100644 --- a/app/src/main/kotlin/com/macedonga/visor/features/Features.kt +++ b/app/src/main/kotlin/com/macedonga/visor/features/Features.kt @@ -10,6 +10,7 @@ import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.core.settings.GlassesSettings import com.macedonga.visor.core.settings.SettingsState import com.macedonga.visor.core.settings.same +import com.macedonga.visor.pairs.pairPrefs import dagger.hilt.android.qualifiers.ApplicationContext import javax.inject.Inject import javax.inject.Singleton @@ -23,6 +24,7 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch @@ -76,10 +78,12 @@ class Features @Inject constructor( private val settings: GlassesSettings, private val connection: GlassesConnection, private val hearing: Hearing, - @ApplicationContext context: Context, + @ApplicationContext private val context: Context, ) { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - private val prefs = context.getSharedPreferences("features", Context.MODE_PRIVATE) + private var prefsFor = connection.pair.value + @Volatile + private var prefs = pairPrefs(context, "features", prefsFor) // MobileConfig values waiting for the next table, and the features they are for. private val queued = MutableStateFlow>(emptyMap()) @@ -306,6 +310,25 @@ class Features @Inject constructor( .apply() } + // Last, so everything it resets is there: another pair has its own marks, sound and test. + init { + scope.launch { + connection.pair.filterNotNull().distinctUntilChanged().collect { serial -> + if (serial == prefsFor) return@collect + prefsFor = serial + prefs = pairPrefs(context, "features", serial) + flush?.cancel() + synchronized(turning) { turning.clear() } + queued.value = emptyMap() + problem.value = null + refused.value = load(REFUSED) + restart.value = load(RESTART).filter { f -> f.gates.any { it is Gate.Config && !it.live } }.toSet() + _focus.value = FocusStyle.entries.getOrElse(prefs.getInt(FOCUS_STYLE, 0)) { FocusStyle.STANDARD } to prefs.getInt(FOCUS_VOLUME, Hearing.DEFAULT_VOLUME) + _tested.value = prefs.getBoolean(HEARING_TESTED, false) + } + } + } + private companion object { const val GATHER_MS = 1_500L const val RESTART = "restart" diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt index b914cbe1..00cf5632 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt @@ -54,6 +54,7 @@ import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.ImageBitmap import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.graphics.graphicsLayer +import androidx.compose.ui.graphics.lerp import androidx.compose.ui.graphics.vector.ImageVector import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.platform.LocalContext @@ -62,6 +63,7 @@ import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp import com.macedonga.visor.R +import com.macedonga.visor.core.identity.language.GlassesLook import com.macedonga.visor.core.connection.CaseBattery import com.macedonga.visor.core.connection.GlassesState import com.macedonga.visor.core.design.Visor @@ -216,6 +218,79 @@ fun StatusPill( } } +/** + * A pair that is not connected, in one capsule: how it is, or how long ago it said so, its batteries as last read, + * dimmed once old, and the button that connects to it set into the capsule's end. No button while the Live or an + * import has the pair that is connected. + */ +@Composable +fun OtherPairPill(mood: Mood, seen: PairSeen?, peeking: Boolean, onConnect: (() -> Unit)?, modifier: Modifier) { + val colors = Visor.colors + val said = seen?.at?.takeIf { it > 0 } + val fresh = said != null && rememberNow() - said < FRESH_MS + Row( + modifier + .clip(RoundedCornerShape(50)) + .background(colors.surface) + .animateContentSize(spring(dampingRatio = 0.8f, stiffness = 300f)) + .padding(start = 16.dp, end = if (onConnect != null) 4.dp else 16.dp, top = 4.dp, bottom = 4.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + val words = when { + peeking || fresh -> stringResource(moodWords(mood)) + said != null -> stringResource(R.string.pill_seen, shortAgo(said)) + else -> stringResource(R.string.pill_not_connected) + } + Text(words, style = Visor.type.label, color = colors.textSecondary, modifier = Modifier.padding(vertical = 5.dp)) + val glasses = seen?.glasses + if (glasses != null && !peeking) { + Spacer(Modifier.size(12.dp)) + Row(Modifier.graphicsLayer { alpha = if (fresh) 1f else 0.45f }, verticalAlignment = Alignment.CenterVertically) { + Level(VisorIcons.Glasses, glasses, critical = false) + seen.case?.let { + Spacer(Modifier.size(14.dp)) + Level(VisorIcons.Case, it, critical = false) + } + } + } + if (onConnect != null) { + Spacer(Modifier.size(12.dp)) + Box( + Modifier + .clip(RoundedCornerShape(50)) + .background(colors.textPrimary) + .clickable(onClick = onConnect) + .padding(horizontal = 14.dp, vertical = 5.dp), + ) { + Text(stringResource(R.string.glasses_connect), style = Visor.type.label, color = colors.canvas) + } + } + } +} + +/** Which of the pairs the stage shows: one dot each, the shown one drawn out. */ +@Composable +fun PairDots(count: Int, index: Int, modifier: Modifier = Modifier) { + val colors = Visor.colors + Row(modifier, horizontalArrangement = Arrangement.spacedBy(6.dp)) { + repeat(count) { i -> + val on by animateFloatAsState(if (i == index) 1f else 0f, spring(dampingRatio = 0.8f, stiffness = 400f), label = "dot") + Box( + Modifier + .size(width = (5 + 9 * on).dp, height = 5.dp) + .clip(RoundedCornerShape(50)) + .background(lerp(colors.control, colors.textPrimary, on)), + ) + } + } +} + +/** What to call a pair: the name it gave itself, else its frame and colour as Meta has them. */ +@Composable +fun pairName(seen: PairSeen?, look: GlassesLook?): String = + seen?.name ?: look?.styleName?.let { style -> look.colorName?.let { stringResource(R.string.pair_look_name, style, it) } ?: style } + ?: stringResource(R.string.glasses_default_name) + /** * The case's charge as a percentage. The glasses report a level on a scale that is not always 100, and they * know it while the case is away too, so neither a missing percent nor an absent case hides it. @@ -506,20 +581,22 @@ private fun Fact(label: String, value: String, dim: Boolean, modifier: Modifier) } @Composable -fun Places(onMedia: () -> Unit, onLive: () -> Unit, onMetaAi: () -> Unit, onSettings: () -> Unit, modifier: Modifier) { +fun Places(onMedia: () -> Unit, onLive: () -> Unit, onMetaAi: () -> Unit, onSettings: () -> Unit, modifier: Modifier, forThisPair: Boolean = true) { Column(modifier) { Place(VisorIcons.Gallery, stringResource(R.string.home_place_media), stringResource(R.string.home_place_media_sub), onMedia) - Place(VisorIcons.Live, stringResource(R.string.home_place_live), stringResource(R.string.home_place_live_sub), onLive) - Place(VisorIcons.Voice, stringResource(R.string.home_place_meta_ai), stringResource(R.string.home_place_meta_ai_sub), onMetaAi) - Place(VisorIcons.Settings, stringResource(R.string.home_place_settings), stringResource(R.string.home_place_settings_sub), onSettings) + // The Live, Meta AI and settings are the connected pair's: not while the home shows another. + Place(VisorIcons.Live, stringResource(R.string.home_place_live), stringResource(R.string.home_place_live_sub), onLive, forThisPair) + Place(VisorIcons.Voice, stringResource(R.string.home_place_meta_ai), stringResource(R.string.home_place_meta_ai_sub), onMetaAi, forThisPair) + Place(VisorIcons.Settings, stringResource(R.string.home_place_settings), stringResource(R.string.home_place_settings_sub), onSettings, forThisPair) } } @Composable -private fun Place(icon: ImageVector, title: String, detail: String, onClick: () -> Unit) { +private fun Place(icon: ImageVector, title: String, detail: String, onClick: () -> Unit, enabled: Boolean = true) { val colors = Visor.colors + val shown by animateFloatAsState(if (enabled) 1f else 0.35f, tween(220), label = "place") Row( - Modifier.fillMaxWidth().clickable(onClick = onClick).padding(horizontal = 24.dp, vertical = 14.dp), + Modifier.fillMaxWidth().graphicsLayer { alpha = shown }.clickable(enabled = enabled, onClick = onClick).padding(horizontal = 24.dp, vertical = 14.dp), verticalAlignment = Alignment.CenterVertically, ) { Box(Modifier.size(44.dp).clip(RoundedCornerShape(13.dp)).background(colors.raised), contentAlignment = Alignment.Center) { @@ -564,6 +641,8 @@ fun moodWords(mood: Mood): Int = when (mood) { } private val RISE = CubicBezierEasing(0.2f, 0f, 0f, 1f) +// A reading newer than this, from a quick look, counts as now. +private const val FRESH_MS = 5 * 60_000L private const val FIRST_MS = 350f private const val STEP_MS = 80f private const val RISE_MS = 620f diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt index 7c9d4517..45e2795e 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt @@ -28,6 +28,8 @@ import androidx.compose.foundation.gestures.scrollable import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.BoxWithConstraints import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.clickable import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.WindowInsets @@ -43,8 +45,10 @@ import androidx.compose.foundation.layout.wrapContentHeight import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.rememberLazyListState import androidx.compose.foundation.shape.GenericShape +import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.SideEffect import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue @@ -53,11 +57,14 @@ import androidx.compose.runtime.mutableIntStateOf import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.rememberUpdatedState import androidx.compose.runtime.setValue import androidx.compose.runtime.snapshotFlow import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clipToBounds +import androidx.compose.ui.draw.clip +import androidx.compose.ui.res.stringResource import androidx.compose.ui.draw.drawBehind import androidx.compose.ui.geometry.CornerRadius import androidx.compose.ui.geometry.Offset @@ -84,6 +91,9 @@ import com.macedonga.visor.connection.GlassesService import com.macedonga.visor.core.connection.GlassesLink import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.connection.GlassesState +import com.macedonga.visor.pairs.Pairs +import com.macedonga.visor.R +import com.macedonga.visor.settings.TextSheet import com.macedonga.visor.core.design.Visor import com.macedonga.visor.core.identity.IdentityStore import com.macedonga.visor.core.media.LiveStream @@ -101,11 +111,13 @@ import com.macedonga.visor.stage.LightPool import com.macedonga.visor.stage.LocalStage import com.macedonga.visor.stage.Looks import com.macedonga.visor.stage.Mood +import com.macedonga.visor.stage.modelled import com.macedonga.visor.stage.POOL_Y import com.macedonga.visor.stage.SHEET_RADIUS import com.macedonga.visor.stage.STAGE_ASPECT import com.macedonga.visor.stage.StageDebug import com.macedonga.visor.stage.StageLight +import com.macedonga.visor.stage.StagePages import com.macedonga.visor.stage.StageSpot import com.macedonga.visor.stage.closedOf import com.macedonga.visor.stage.liveliness @@ -138,7 +150,8 @@ class HomeViewModel @Inject constructor( private val store: IdentityStore, keeper: LeKeeper, val looks: Looks, - lastSeen: LastSeen, + val lastSeen: LastSeen, + val pairs: Pairs, ) : ViewModel() { /** The LE link is up: the glasses are right here, whatever RFCOMM says. */ val nearby = keeper.connected @@ -146,29 +159,10 @@ class HomeViewModel @Inject constructor( val paired = MutableStateFlow(null) val shots = MutableStateFlow>(emptyList()) val firstName = MutableStateFlow(null) - private val prefs = context.getSharedPreferences("home", Context.MODE_PRIVATE) val lastBattery = lastSeen.battery - val lastFacts = MutableStateFlow(readFacts()) val lastPosture = lastSeen.posture init { - viewModelScope.launch { - connection.state.collect { state -> - val was = lastFacts.value - // A volume the glasses have not reported comes as -1: until they do, the one Android holds for them. - val now = GlassesFacts( - freeMb = state.device?.storage?.freeMb?.takeIf { it >= 0 } ?: was.freeMb, - volume = state.device?.volume?.percent?.toInt()?.takeIf { it in 0..100 } - ?: heldVolume(context, (state.target?.link as? GlassesLink.Bluetooth)?.address) ?: was.volume, - temperatureC = state.device?.battery?.temperatureC?.takeIf { it > -40 && it < 100 } ?: was.temperatureC, - ) - if (now != was) { - lastFacts.value = now - prefs.edit().putLong("free_mb", now.freeMb ?: -1).putInt("volume", now.volume ?: -1) - .putFloat("temperature", now.temperatureC?.toFloat() ?: Float.NaN).apply() - } - } - } // Whatever is saved shows up here as soon as it lands. viewModelScope.launch { media.saved.collect { shots.value = recentShots(context) } } } @@ -177,13 +171,8 @@ class HomeViewModel @Inject constructor( viewModelScope.launch(Dispatchers.IO) { paired.value = runCatching { store.all().isNotEmpty() }.getOrDefault(false) } viewModelScope.launch { shots.value = recentShots(context) } viewModelScope.launch { firstName.value = ownerFirstName(context) } + pairs.refresh() } - - private fun readFacts() = GlassesFacts( - freeMb = prefs.getLong("free_mb", -1).takeIf { it >= 0 }, - volume = prefs.getInt("volume", -1).takeIf { it in 0..100 }, - temperatureC = prefs.getFloat("temperature", Float.NaN).takeIf { !it.isNaN() }?.toDouble(), - ) } /** @@ -241,10 +230,28 @@ fun HomeScreen( val shots by model.shots.collectAsStateWithLifecycle() val lastBattery by model.lastBattery.collectAsStateWithLifecycle() val firstName by model.firstName.collectAsStateWithLifecycle() - val lastFacts by model.lastFacts.collectAsStateWithLifecycle() val lastPosture by model.lastPosture.collectAsStateWithLifecycle() val nearby by model.nearby.collectAsStateWithLifecycle() val paired by model.paired.collectAsStateWithLifecycle() + // Every pair set up here; the stage shows one at a time and a drag goes to the next. + val pairs by model.pairs.all.collectAsStateWithLifecycle() + val seen by model.lastSeen.seen.collectAsStateWithLifecycle() + val looks by model.looks.all.collectAsStateWithLifecycle() + val pictures by model.looks.pictures.collectAsStateWithLifecycle() + val peeking by model.pairs.peeking.collectAsStateWithLifecycle() + val current by model.connection.pair.collectAsStateWithLifecycle() + // The page shown: a pair's serial, null for the pair Visor is for, or ADD_PAGE after the last pair. + var page by remember { mutableStateOf(null) } + val adding = page == ADD_PAGE && pairs.isNotEmpty() + val shownIndex = if (adding) pairs.size else pairs.indexOfFirst { it.serial == (page ?: current) } + val shown = pairs.getOrNull(shownIndex) + // A pair that is not the one Visor is for: it shows as last heard from, and can be connected. + val other = shown != null && current != null && shown.serial != current + // Away from the pair Visor is for, its Live, Meta AI and settings are not this page's. + val elsewhere = other || adding + val shownSeen = shown?.let { seen[it.serial] } + val lastFacts = (if (elsewhere) shownSeen else current?.let(seen::get))?.facts ?: GlassesFacts(null, null, null) + var renaming by remember { mutableStateOf(null) } val forced by StageDebug.mood.collectAsStateWithLifecycle() val variant by StageDebug.variant.collectAsStateWithLifecycle() val haptics = rememberHaptics() @@ -360,7 +367,13 @@ fun HomeScreen( wasLive = live } val mood = forced ?: when { - paired == false -> Mood.ABSENT + paired == false || adding -> Mood.ABSENT + other -> when { + peeking == shown?.serial -> Mood.SEARCHING + shownSeen?.inCase == true -> Mood.IN_CASE + shownSeen?.folded == true -> Mood.ASLEEP + else -> Mood.FAR + } transferring -> Mood.IMPORTING live && state.link != LinkState.READY -> Mood.RECORDING (importing || settling || liveSettling) && state.link != LinkState.READY -> Mood.CONNECTED @@ -378,13 +391,38 @@ fun HomeScreen( LaunchedEffect(stage) { if (firstVisit) stage.light.dark() } StageLight(lookOf(mood, colors)) val look by model.looks.look.collectAsStateWithLifecycle() - stage.pose.variants = variant?.let(::listOf) ?: variantsOf(look) + val shownLook = if (other) shown?.let { looks[it.serial] } else look + stage.pose.variants = variant?.let(::listOf) ?: variantsOf(shownLook) + // The model is a Wayfarer: another frame, or a pair not connected, shows as Meta pictures it, once Meta has. + val currentPicture = current?.takeIf { !modelled(looks[it]) }?.let(pictures::get) + val picture = when { + other -> shown?.let { pictures[it.serial] } + !modelled(shownLook) -> currentPicture + else -> null + } + SideEffect { + stage.picture = picture + stage.vacant = adding + } + // Other screens show the pair Visor is for. + val leavingPicture = rememberUpdatedState(currentPicture) + DisposableEffect(Unit) { + onDispose { + stage.picture = leavingPicture.value + stage.vacant = false + } + } + // A pair shown that is not connected gets a quick look at its batteries. + LaunchedEffect(shown?.serial, other) { + if (other && shown != null) model.pairs.peek(shown.serial) + } + DisposableEffect(Unit) { onDispose { model.pairs.stopPeek() } } // Other screens that show the pair take its mood and hinge from here. SideEffect { stage.mood = mood stage.lastFolded = lastPosture.folded } - Choreography(state, mood, stage.light, stage.pose, haptics::connected, haptics::shutter) + Choreography(if (elsewhere) GlassesState() else state, mood, stage.light, stage.pose, haptics::connected, haptics::shutter) val list = rememberLazyListState() remember { @@ -408,7 +446,7 @@ fun HomeScreen( if (list.firstVisibleItemIndex == 0) list.firstVisibleItemScrollOffset.toFloat() else fullPx } // Folded, the pair sits small in the middle of the strip, clear of the camera cut-out; at launch it fades in. - val spot = StageSpot(closedOf(state, mood, lastPosture.folded), liveliness(mood)) { + val spot = StageSpot(if (elsewhere) closedOf(GlassesState(), mood, shownSeen?.folded ?: true) else closedOf(state, mood, lastPosture.folded), liveliness(mood)) { folding(fullPx, stripPx, fold, HOME_STRIP_SCALE, topPx + (stripPx - topPx) / 2) alpha = entranceAlpha(entrance, STAGE_FADE_FROM_MS, STAGE_FADE_MS) } @@ -469,11 +507,11 @@ fun HomeScreen( waitingCount = fresh.size, leaving = leaving, onLeft = { leaving = false }, ) } - Facts(lastFacts, live = here, lastCapture = lastCapture, modifier = Modifier.padding(top = GAP).appear(entrance, 2)) + Facts(lastFacts, live = here && !elsewhere, lastCapture = lastCapture, modifier = Modifier.padding(top = GAP).appear(entrance, 2)) Spacer(Modifier.height(FOOT)) } } - item(key = "places") { Places(onMedia, onLive, onMetaAi, onSettings, Modifier.appear(entrance, 5)) } + item(key = "places") { Places(onMedia, onLive, onMetaAi, onSettings, Modifier.appear(entrance, 5), forThisPair = !elsewhere) } } @@ -487,18 +525,63 @@ fun HomeScreen( layout(placeable.width, h) { placeable.place(0, 0) } } .scrollable(list, Orientation.Vertical, reverseDirection = true) - .turnsThePair { haptics.tick(0.35f) }, + .turnsThePair( + // The pairs, then the page for adding one. + pages = if (pairs.isNotEmpty() && shownIndex >= 0) StagePages( + can = { d -> shownIndex + d in 0..pairs.size }, + go = { d -> page = pairs.getOrNull(shownIndex + d)?.serial ?: ADD_PAGE }, + ) else null, + ) { haptics.tick(0.35f) }, ) { val fade = Modifier.graphicsLayer { alpha = (1f - (fold * 2.5f).coerceIn(0f, 1f)) * stage.shown(spot) } - Greeting( - firstName, - Modifier.align(Alignment.TopStart).statusBarsPadding().padding(horizontal = 24.dp, vertical = 12.dp).then(fade).appear(entrance, 0), - ) - StatusPill( - state, mood, lastBattery, paired, allowed, transferring, importProgress, waiting = if (importing) 0 else fresh.size, onImport = onImport, - onAllow = { ask.launch(wanted()) }, onSetUp = onSetUp, - modifier = Modifier.align(Alignment.BottomCenter).padding(bottom = 26.dp).then(fade).appear(entrance, 0), - ) + Column(Modifier.align(Alignment.TopStart).statusBarsPadding().padding(horizontal = 24.dp, vertical = 12.dp).then(fade).appear(entrance, 0)) { + Greeting(firstName, Modifier) + // Which pair this is; a tap names it. + if (shown != null) { + Text( + pairName(shownSeen, looks[shown.serial]), + style = Visor.type.label, + color = Visor.colors.textSecondary, + modifier = Modifier + .padding(top = 2.dp) + .clip(RoundedCornerShape(6.dp)) + .clickable { renaming = shown.serial } + .padding(vertical = 4.dp), + ) + } + } + Column( + Modifier.align(Alignment.BottomCenter).padding(bottom = if (pairs.size > 1) 12.dp else 26.dp).then(fade).appear(entrance, 0), + horizontalAlignment = Alignment.CenterHorizontally, + ) { + if (adding) { + Pill(stringResource(R.string.glasses_set_up_new), onSetUp) + } else if (other && shown != null) { + OtherPairPill( + mood, shownSeen, peeking = peeking == shown.serial, + onConnect = if (live || importing) null else ({ + haptics.toggle() + model.pairs.connect(shown.serial) + }), + modifier = Modifier, + ) + } else { + StatusPill( + state, mood, lastBattery, paired, allowed, transferring, importProgress, waiting = if (importing) 0 else fresh.size, onImport = onImport, + onAllow = { ask.launch(wanted()) }, onSetUp = onSetUp, modifier = Modifier, + ) + } + if (pairs.isNotEmpty()) PairDots(pairs.size + 1, shownIndex, Modifier.padding(top = 12.dp)) + } + renaming?.let { serial -> + TextSheet( + stringResource(R.string.glasses_name), seen[serial]?.name.orEmpty(), + onSave = { model.lastSeen.rename(serial, it.trim()) }, + onDismiss = { renaming = null }, + hint = pairName(null, looks[serial]), + valid = { it.isNotBlank() && it.trim().length <= 40 }, + ) + } } } } @@ -539,3 +622,6 @@ private val FOOT = 20.dp private const val SETTLE_MS = 25_000L private const val DONE_HOLD_MS = 2_000L private const val REFRESH_MS = 60_000L + +/** The home's page after the last pair, for setting up another. */ +private const val ADD_PAGE = "+" diff --git a/app/src/main/kotlin/com/macedonga/visor/home/LastSeen.kt b/app/src/main/kotlin/com/macedonga/visor/home/LastSeen.kt index 19f228b4..28798eb0 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/LastSeen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/LastSeen.kt @@ -2,9 +2,12 @@ package com.macedonga.visor.home import android.content.Context import com.macedonga.visor.core.connection.GlassesConnection -import com.macedonga.visor.core.media.LiveStream +import com.macedonga.visor.core.connection.GlassesLink +import com.macedonga.visor.core.connection.GlassesState import com.macedonga.visor.core.media.LivePhase +import com.macedonga.visor.core.media.LiveStream import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.settings.GlassesSettings import dagger.hilt.android.qualifiers.ApplicationContext import javax.inject.Inject import javax.inject.Singleton @@ -12,9 +15,16 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch +import org.json.JSONObject /** The glasses' batteries as last read, for when they are out of reach. */ data class LastBattery(val glasses: Int, val case: Int?, val at: Long) @@ -22,60 +32,182 @@ data class LastBattery(val glasses: Int, val case: Int?, val at: Long) /** How the glasses were the last time they said: folded, docked in the case. Their sleep looks like silence. */ data class LastPosture(val folded: Boolean, val inCase: Boolean) +/** One pair as last heard from: batteries, how it sat, the home's numbers, and the name it gave itself. */ +data class PairSeen( + val glasses: Int? = null, + val case: Int? = null, + val at: Long = 0, + val folded: Boolean = false, + val inCase: Boolean = false, + val freeMb: Long? = null, + val volume: Int? = null, + val temperatureC: Double? = null, + val name: String? = null, + // Named here while away: the glasses take the name when they are next connected. + val nameToSend: Boolean = false, +) { + val battery: LastBattery? get() = glasses?.let { LastBattery(it, case, at) } + val posture: LastPosture get() = LastPosture(folded, inCase) + val facts: GlassesFacts get() = GlassesFacts(freeMb, volume, temperatureC) +} + /** - * How the glasses were when last heard from, kept across launches. One for the process, so the service writes it - * while no screen is open and the home, the widget and the tile all read the same thing. + * How each pair was when last heard from, kept across launches. One for the process, so the service writes it while + * no screen is open and the home, the widget and the tile all read the same thing. [battery] and [posture] are the + * current pair's; the home's other pairs read [seen]. */ @Singleton class LastSeen @Inject constructor( - @ApplicationContext context: Context, - connection: GlassesConnection, + @ApplicationContext private val context: Context, + private val connection: GlassesConnection, + private val settings: GlassesSettings, live: LiveStream, ) { - private val prefs = context.getSharedPreferences("home", Context.MODE_PRIVATE) + private val prefs = context.getSharedPreferences("pairs_seen", Context.MODE_PRIVATE) private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - private val _battery = MutableStateFlow(readBattery()) - private val _posture = MutableStateFlow(LastPosture(prefs.getBoolean("folded", false), prefs.getBoolean("in_case", false))) - val battery: StateFlow = _battery.asStateFlow() - val posture: StateFlow = _posture.asStateFlow() + private val savedAt = java.util.concurrent.ConcurrentHashMap() + private val _seen = MutableStateFlow(prefs.all.mapNotNull { (serial, text) -> (text as? String)?.let(::decode)?.let { serial to it } }.toMap()) + + /** Every pair heard from, by serial. */ + val seen: StateFlow> = _seen.asStateFlow() + + private val current = combine(connection.pair, _seen) { serial, all -> serial?.let(all::get) } + val battery: StateFlow = current.map { it?.battery }.distinctUntilChanged().stateIn(scope, SharingStarted.Eagerly, null) + val posture: StateFlow = + current.map { it?.posture ?: LastPosture(false, false) }.distinctUntilChanged().stateIn(scope, SharingStarted.Eagerly, LastPosture(false, false)) init { + // The pair Visor is for has its readings from the start, the old shared ones included. + scope.launch { connection.pair.filterNotNull().collect { serial -> if (serial !in _seen.value) update(serial) { it } } } + scope.launch { connection.state.collect(::record) } + // The glasses' own name, unless one given here is still on its way to them. scope.launch { - connection.state.collect { state -> - val battery = state.device?.battery ?: return@collect - val case = state.device?.case?.levelPercent() - val was = _battery.value - val now = LastBattery(battery.percent.toInt(), case ?: was?.case, System.currentTimeMillis()) - if (now.glasses != was?.glasses || now.case != was?.case) writeBattery(now) - _battery.value = now - } - } - scope.launch { - connection.state.collect { state -> - val device = state.device ?: return@collect - if (state.link != LinkState.READY) return@collect - // The case's "present" only says the case is known; charging is what says they sit in it. - val was = _posture.value - writePosture(LastPosture(folded = device.hinge?.let { it == "CLOSED" } ?: was.folded, inCase = device.battery?.charging ?: was.inCase)) + settings.state.map { it.pair to (it.readAt != null) to it.stored[NAME_KEY] }.distinctUntilChanged().collect { (read, stored) -> + val (serial, wasRead) = read + if (serial == null || !wasRead) return@collect + val mine = _seen.value[serial]?.takeIf { it.nameToSend }?.name + when { + mine == null -> update(serial) { it.copy(name = stored?.takeIf { n -> n.isNotBlank() && n != LEFTOVER_NAME }) } + stored == mine -> update(serial) { it.copy(nameToSend = false) } + else -> settings.write(NAME_KEY, mine) + } } } // The glasses stream only open and out of the case: a Live that ran says so while Bluetooth is away for it, and // the home it goes back to does not unfold the pair once the link is back. scope.launch { - live.status.collect { status -> if (status.phase == LivePhase.RUNNING) writePosture(LastPosture(folded = false, inCase = false)) } + live.status.collect { status -> + val serial = connection.pair.value + if (status.phase == LivePhase.RUNNING && serial != null) update(serial) { it.copy(folded = false, inCase = false) } + } + } + } + + /** Keep what [state] says about its pair: the connection's own, or a quick look at another pair. */ + fun record(state: GlassesState) { + val target = state.target ?: return + val device = state.device ?: return + update(target.serial) { was -> + val battery = device.battery + var now = was + if (battery != null) now = now.copy(glasses = battery.percent.toInt(), case = device.case?.levelPercent() ?: was.case, at = System.currentTimeMillis()) + // The case's "present" only says the case is known; charging is what says they sit in it. + if (state.link == LinkState.READY) now = now.copy(folded = device.hinge?.let { it == "CLOSED" } ?: was.folded, inCase = battery?.charging ?: was.inCase) + // A volume the glasses have not reported comes as -1: until they do, the one Android holds for them. + now.copy( + freeMb = device.storage?.freeMb?.takeIf { it >= 0 } ?: was.freeMb, + volume = device.volume?.percent?.toInt()?.takeIf { it in 0..100 } + ?: heldVolume(context, (target.link as? GlassesLink.Bluetooth)?.address) ?: was.volume, + temperatureC = battery?.temperatureC?.takeIf { it > -40 && it < 100 } ?: was.temperatureC, + ) } } - private fun writePosture(now: LastPosture) { - if (now == _posture.value) return - _posture.value = now - prefs.edit().putBoolean("folded", now.folded).putBoolean("in_case", now.inCase).apply() + /** Call [serial] [name]: the glasses take it at once when they are here, else when they are next connected. */ + fun rename(serial: String, name: String) { + val here = connection.state.value.let { it.target?.serial == serial && it.link == LinkState.READY } && settings.state.value.pair == serial + val sent = here && settings.write(NAME_KEY, name) + update(serial) { it.copy(name = name, nameToSend = !sent) } + } + + /** Hold [seen] for [serial] in memory only: a debug build's demo pair. */ + @Synchronized + fun keep(serial: String, seen: PairSeen) { + _seen.value = _seen.value + (serial to seen) + } + + /** A pair forgotten: what it last said goes too. */ + @Synchronized + fun forget(serial: String) { + _seen.value = _seen.value - serial + savedAt.remove(serial) + prefs.edit().remove(serial).apply() } - private fun readBattery(): LastBattery? = prefs.getInt("glasses", -1).takeIf { it >= 0 }?.let { - LastBattery(it, prefs.getInt("case", -1).takeIf { c -> c >= 0 }, prefs.getLong("at", 0)) + // The connection, a quick look and the name all write here, from their own threads. + @Synchronized + private fun update(serial: String, change: (PairSeen) -> PairSeen) { + val was = _seen.value[serial] ?: migrated(serial) + val now = change(was) + if (now == _seen.value[serial]) return + _seen.value = _seen.value + (serial to now) + // Every reading moves the time; the file follows only a change, or a minute. + if (now.copy(at = 0) != was.copy(at = 0) || now.at - (savedAt[serial] ?: 0) > 60_000) { + savedAt[serial] = now.at + prefs.edit().putString(serial, encode(now)).apply() + } + } + + /** Before Visor kept pairs apart, the one pair's readings were the home's: the first pair heard from takes them. */ + private fun migrated(serial: String): PairSeen { + val home = context.getSharedPreferences("home", Context.MODE_PRIVATE) + if (!home.contains("glasses") && !home.contains("folded") && !home.contains("free_mb")) return PairSeen() + val old = PairSeen( + glasses = home.getInt("glasses", -1).takeIf { it >= 0 }, + case = home.getInt("case", -1).takeIf { it >= 0 }, + at = home.getLong("at", 0), + folded = home.getBoolean("folded", false), + inCase = home.getBoolean("in_case", false), + freeMb = home.getLong("free_mb", -1).takeIf { it >= 0 }, + volume = home.getInt("volume", -1).takeIf { it in 0..100 }, + temperatureC = home.getFloat("temperature", Float.NaN).takeIf { !it.isNaN() }?.toDouble(), + ) + home.edit().remove("glasses").remove("case").remove("at").remove("folded").remove("in_case") + .remove("free_mb").remove("volume").remove("temperature").apply() + return old } - private fun writeBattery(b: LastBattery) = - prefs.edit().putInt("glasses", b.glasses).putInt("case", b.case ?: -1).putLong("at", b.at).apply() + private fun encode(s: PairSeen): String = JSONObject().apply { + s.glasses?.let { put("glasses", it) } + s.case?.let { put("case", it) } + put("at", s.at) + put("folded", s.folded) + put("in_case", s.inCase) + s.freeMb?.let { put("free_mb", it) } + s.volume?.let { put("volume", it) } + s.temperatureC?.let { put("temperature", it) } + s.name?.let { put("name", it) } + if (s.nameToSend) put("name_to_send", true) + }.toString() + + private fun decode(text: String): PairSeen? = runCatching { + val o = JSONObject(text) + PairSeen( + glasses = o.optInt("glasses", -1).takeIf { it >= 0 }, + case = o.optInt("case", -1).takeIf { it >= 0 }, + at = o.optLong("at", 0), + folded = o.optBoolean("folded"), + inCase = o.optBoolean("in_case"), + freeMb = o.optLong("free_mb", -1).takeIf { it >= 0 }, + volume = o.optInt("volume", -1).takeIf { it in 0..100 }, + temperatureC = o.optDouble("temperature").takeIf { !it.isNaN() }, + name = o.optString("name").takeIf { it.isNotEmpty() }, + nameToSend = o.optBoolean("name_to_send"), + ) + }.getOrNull() + + companion object { + const val NAME_KEY = "device_custom_name" + const val LEFTOVER_NAME = "VisorTest" + } } diff --git a/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt b/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt index 51eb5f3a..e346c393 100644 --- a/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt +++ b/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt @@ -11,6 +11,7 @@ import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.core.settings.GlassesSettings import com.macedonga.visor.log.EventLog import com.macedonga.visor.login.MetaLogin +import com.macedonga.visor.pairs.pairPrefs import dagger.hilt.android.qualifiers.ApplicationContext import java.io.File import java.io.FileOutputStream @@ -28,6 +29,9 @@ import kotlinx.coroutines.ensureActive import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import kotlinx.coroutines.withContext @@ -58,7 +62,9 @@ class LanguageInstaller @Inject constructor( private val log: EventLog, ) { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) - private val prefs = context.getSharedPreferences("languages", Context.MODE_PRIVATE) + // Installed on the glasses: each pair has its own. + @Volatile + private var prefs = pairPrefs(context, "languages", connection.pair.value) private val dir = File(context.filesDir, "packs") private val _work = MutableStateFlow(LanguageWork.Idle) @@ -68,6 +74,15 @@ class LanguageInstaller @Inject constructor( private val _installed = MutableStateFlow(prefs.getStringSet(KEY_INSTALLED, null).orEmpty() + "en_US") val installed: StateFlow> = _installed.asStateFlow() + init { + scope.launch { + connection.pair.filterNotNull().distinctUntilChanged().drop(1).collect { serial -> + prefs = pairPrefs(context, "languages", serial) + _installed.value = prefs.getStringSet(KEY_INSTALLED, null).orEmpty() + "en_US" + } + } + } + /** What Meta has, by pack suffix, and why it could not be asked. */ private val _offers = MutableStateFlow>(emptyMap()) val offers: StateFlow> = _offers.asStateFlow() diff --git a/app/src/main/kotlin/com/macedonga/visor/pairs/PairPrefs.kt b/app/src/main/kotlin/com/macedonga/visor/pairs/PairPrefs.kt new file mode 100644 index 00000000..b9a9a336 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/pairs/PairPrefs.kt @@ -0,0 +1,34 @@ +package com.macedonga.visor.pairs + +import android.content.Context +import android.content.SharedPreferences + +/** + * Visor's own notes about one pair, in a preferences file of its own (`_`). The first pair to ask takes + * what the shared file held from before Visor kept pairs apart; with no pair known yet, the shared file it is. + */ +fun pairPrefs(context: Context, name: String, serial: String?): SharedPreferences { + val shared = context.getSharedPreferences(name, Context.MODE_PRIVATE) + serial ?: return shared + val own = context.getSharedPreferences("${name}_$serial", Context.MODE_PRIVATE) + val old = shared.all + if (old.isNotEmpty()) { + if (own.all.isEmpty()) { + val edit = own.edit() + old.forEach { (key, value) -> + @Suppress("UNCHECKED_CAST") + when (value) { + is Boolean -> edit.putBoolean(key, value) + is Int -> edit.putInt(key, value) + is Long -> edit.putLong(key, value) + is Float -> edit.putFloat(key, value) + is String -> edit.putString(key, value) + is Set<*> -> edit.putStringSet(key, value as Set) + } + } + edit.commit() + } + shared.edit().clear().commit() + } + return own +} diff --git a/app/src/main/kotlin/com/macedonga/visor/pairs/Pairs.kt b/app/src/main/kotlin/com/macedonga/visor/pairs/Pairs.kt new file mode 100644 index 00000000..9362e8b9 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/pairs/Pairs.kt @@ -0,0 +1,140 @@ +package com.macedonga.visor.pairs + +import android.content.Context +import com.macedonga.visor.connection.AndroidTransportFactory +import com.macedonga.visor.connection.GlassesService +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.connection.GlassesLink +import com.macedonga.visor.core.connection.GlassesTarget +import com.macedonga.visor.core.connection.RealSessionHostFactory +import com.macedonga.visor.core.identity.IdentityStore +import com.macedonga.visor.core.media.LiveStream +import com.macedonga.visor.core.media.MediaImport +import com.macedonga.visor.core.transport.LeKeeper +import com.macedonga.visor.home.LastSeen +import com.macedonga.visor.home.PairSeen +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.stage.Looks +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull + +/** A pair set up on this phone. [demo] is one a debug build shows to try the home with: never connected or looked at. */ +data class KnownPair(val serial: String, val address: String, val demo: Boolean = false) + +/** + * The pairs set up on this phone, in the order they were set up. One is connected at a time; the home shows the others + * as they were last heard from, and a pair it shows that is not connected gets a quick look: its own LE link and a + * session that only reads the batteries, the case and the hinge, then lets go. + */ +@Singleton +class Pairs @Inject constructor( + @ApplicationContext private val context: Context, + private val store: IdentityStore, + private val connection: GlassesConnection, + private val live: LiveStream, + private val media: MediaImport, + private val lastSeen: LastSeen, + private val looks: Looks, + private val log: EventLog, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val stored = MutableStateFlow>(emptyList()) + private val demoPair = MutableStateFlow(null) + val all: StateFlow> = combine(stored, demoPair) { pairs, extra -> pairs + listOfNotNull(extra) } + .stateIn(scope, SharingStarted.Eagerly, emptyList()) + + private val _peeking = MutableStateFlow(null) + + /** The pair being looked at right now, if any. */ + val peeking: StateFlow = _peeking.asStateFlow() + + private var peek: Job? = null + private val peekedAt = mutableMapOf() + + fun refresh() { + scope.launch { + val bundles = withContext(Dispatchers.IO) { runCatching { store.all() }.getOrDefault(emptyList()) } + stored.value = bundles.sortedBy { it.createdAt }.map { KnownPair(it.serial, it.glassesAddress) } + bundles.forEach { looks.want(it.serial) } + // Kept in memory and small: the stage needs one for every pair but a Wayfarer, and to tell pairs apart. + looks.wantPictures(bundles.map { it.serial }) + } + } + + /** Connect to [serial] instead of the pair connected now. False while the Live or an import has the glasses. */ + fun connect(serial: String): Boolean { + if (live.running.value || media.importing.value) return false + val pair = all.value.firstOrNull { it.serial == serial }?.takeIf { !it.demo } ?: return false + stopPeek() + GlassesService.start(context, GlassesTarget(GlassesLink.Bluetooth(pair.address), pair.serial)) + return true + } + + /** A quick look at [serial], a pair that is not connected: at most once a minute, never during a Live or an import. */ + fun peek(serial: String) { + if (serial == connection.state.value.target?.serial || serial == connection.pair.value && connection.running.value) return + if (live.running.value || media.importing.value) return + val now = System.currentTimeMillis() + if (now - (peekedAt[serial] ?: 0) < PEEK_EVERY_MS) return + val pair = all.value.firstOrNull { it.serial == serial }?.takeIf { !it.demo } ?: return + peekedAt[serial] = now + peek?.cancel() + peek = scope.launch { + val keeper = LeKeeper(context) + val look = GlassesConnection(store, RealSessionHostFactory(AndroidTransportFactory(context, keeper = keeper)), chooses = false) + _peeking.value = serial + log.add("Pairs: looking at ${pair.serial}") + try { + keeper.hold(pair.address) + // Asleep, they answer on LE only: wait for it a moment before the session picks its link. + withTimeoutOrNull(LE_WAIT_MS) { keeper.connected.first { it } } + look.start(this, GlassesTarget(GlassesLink.Bluetooth(pair.address), pair.serial)) + val seen = withTimeoutOrNull(PEEK_MS) { look.state.first { it.device?.battery != null } } + // The case comes in its own update, when there is one. + if (seen != null) withTimeoutOrNull(CASE_MS) { look.state.first { it.device?.case != null } } + look.state.value.let(lastSeen::record) + log.add(if (seen != null) "Pairs: ${pair.serial} at ${seen.device?.battery?.percent?.toInt()}%" else "Pairs: ${pair.serial} did not answer") + } finally { + // Let go of the socket even when the look was called off: the glasses hold one session. + withContext(NonCancellable) { look.stop() } + keeper.release() + _peeking.value = null + } + } + } + + /** Debug builds: show [pair] beside the real ones, as last heard from with [seen]; null takes it away. */ + fun demo(pair: KnownPair?, seen: PairSeen?) { + demoPair.value?.let { lastSeen.forget(it.serial) } + if (pair != null && seen != null) lastSeen.keep(pair.serial, seen) + demoPair.value = pair?.copy(demo = true) + } + + fun stopPeek() { + peek?.cancel() + peek = null + } + + private companion object { + const val PEEK_EVERY_MS = 60_000L + const val LE_WAIT_MS = 6_000L + const val PEEK_MS = 20_000L + const val CASE_MS = 2_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt index 0ad92028..cb624139 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt @@ -31,6 +31,7 @@ import com.macedonga.visor.OnboardingKey import com.macedonga.visor.R import com.macedonga.visor.connection.GlassesService import com.macedonga.visor.connection.LinkKind +import com.macedonga.visor.home.pairName import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.connection.GlassesLink import com.macedonga.visor.core.connection.GlassesTarget @@ -63,6 +64,7 @@ class GlassesPageModel @Inject constructor( val kind: LinkKind, private val store: IdentityStore, val looks: Looks, + val lastSeen: com.macedonga.visor.home.LastSeen, ) : ViewModel() { private val _glasses = MutableStateFlow>(emptyList()) @@ -83,6 +85,7 @@ class GlassesPageModel @Inject constructor( fun forget(serial: String) { viewModelScope.launch(Dispatchers.IO) { store.remove(serial) + lastSeen.forget(serial) _glasses.value = runCatching { store.all() }.getOrDefault(emptyList()) } } @@ -95,6 +98,8 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( val glasses by model.connection.state.collectAsStateWithLifecycle() val running by model.connection.running.collectAsStateWithLifecycle() val bundles by model.glasses.collectAsStateWithLifecycle() + val seen by model.lastSeen.seen.collectAsStateWithLifecycle() + val looks by model.looks.all.collectAsStateWithLifecycle() val login by model.login.status.collectAsStateWithLifecycle() val le by model.kind.overLe.collectAsStateWithLifecycle() LifecycleResumeEffect(Unit) { @@ -125,17 +130,17 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( item(key = "power") { Row(Modifier.fillMaxWidth().padding(horizontal = SIDE).padding(bottom = 8.dp), horizontalArrangement = Arrangement.spacedBy(36.dp)) { // Away, the last reading the home kept, dimmed. - val last = remember { context.getSharedPreferences("home", android.content.Context.MODE_PRIVATE) } + val last by model.lastSeen.battery.collectAsStateWithLifecycle() val glassesNow = device?.battery?.percent?.toInt() val caseNow = device?.case?.levelPercent() Readout( stringResource(R.string.glasses_battery), - (glassesNow ?: last.getInt("glasses", -1).takeIf { it >= 0 })?.let { "$it%" }, + (glassesNow ?: last?.glasses)?.let { "$it%" }, device?.battery?.charging == true, fresh = glassesNow != null, ) Readout( stringResource(R.string.glasses_case_label), - (caseNow ?: last.getInt("case", -1).takeIf { it >= 0 })?.let { "$it%" }, + (caseNow ?: last?.case)?.let { "$it%" }, device?.case?.charging == true, fresh = caseNow != null, ) } @@ -201,9 +206,12 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( }, *bundles.filter { it.serial != serial }.map { b -> @Composable { - RowFrame(b.serial, onClick = null, sub = b.glassesAddress) { + val other = seen[b.serial] + RowFrame( + pairName(other, looks[b.serial]), onClick = null, + sub = other?.glasses?.let { g -> other.case?.let { c -> stringResource(R.string.glasses_other_batteries, g, c) } ?: "$g%" }, + ) { ActionPill(stringResource(R.string.glasses_connect), { - GlassesService.stop(context) GlassesService.start(context, GlassesTarget(GlassesLink.Bluetooth(b.glassesAddress), b.serial)) }, quiet = true, modifier = Modifier.padding(start = 12.dp)) } diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt index 6923cc54..37af242e 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt @@ -124,7 +124,6 @@ import com.macedonga.visor.gallery.ProgressRing import com.macedonga.visor.gallery.RoundIcon import com.macedonga.visor.gallery.placesAllowed import com.macedonga.visor.gallery.words -import com.macedonga.visor.home.LastBattery import com.macedonga.visor.home.StatusPill import com.macedonga.visor.language.languageOfVoice import com.macedonga.visor.media.spanWords @@ -171,6 +170,7 @@ class SettingsViewModel @Inject constructor( val backups: Backups, val updates: com.macedonga.visor.update.Updates, val features: com.macedonga.visor.features.Features, + val lastSeen: com.macedonga.visor.home.LastSeen, ) : ViewModel() /** @@ -286,11 +286,7 @@ fun SettingsScreen(open: (NavKey) -> Unit, model: SettingsViewModel = hiltViewMo .turnsThePair { host.haptics.tick(0.35f) } .clickable(interactionSource = remember { MutableInteractionSource() }, indication = null) { open(GlassesKey) }, ) { - val remembered = remember { - context.getSharedPreferences("home", android.content.Context.MODE_PRIVATE).let { p -> - p.getInt("glasses", -1).takeIf { it >= 0 }?.let { LastBattery(it, p.getInt("case", -1).takeIf { c -> c >= 0 }, p.getLong("at", 0)) } - } - } + val remembered by model.lastSeen.battery.collectAsStateWithLifecycle() StatusPill( glasses, mood, remembered, paired = glasses.target?.let { true }, allowed = true, importing = false, importProgress = 0f, waiting = 0, onImport = {}, onAllow = {}, onSetUp = { open(OnboardingKey()) }, diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/WakeWord.kt b/app/src/main/kotlin/com/macedonga/visor/settings/WakeWord.kt index acf70298..1afe50a2 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/WakeWord.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/WakeWord.kt @@ -45,11 +45,13 @@ import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.lifecycle.viewModelScope import com.macedonga.visor.R import com.macedonga.visor.core.assistant.Assistant +import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.design.Visor import com.macedonga.visor.core.settings.GlassesSettings import com.macedonga.visor.core.settings.asBool import com.macedonga.visor.features.ChoicePill import com.macedonga.visor.features.StoryWords +import com.macedonga.visor.pairs.pairPrefs import com.macedonga.visor.setup.BigPill import com.macedonga.visor.stage.LightLook import com.macedonga.visor.stage.LocalStage @@ -88,9 +90,11 @@ sealed interface WakeStep { class WakeWordModel @Inject constructor( private val assistant: Assistant, val settings: GlassesSettings, + connection: GlassesConnection, @ApplicationContext context: Context, ) : ViewModel() { - private val prefs = context.getSharedPreferences("wake_word", Context.MODE_PRIVATE) + // Enrolled on the glasses: each pair has its own phrase. + private val prefs = pairPrefs(context, "wake_word", connection.pair.value) private val _enrolled = MutableStateFlow(prefs.getString(ENROLLED, null)) diff --git a/app/src/main/kotlin/com/macedonga/visor/stage/Frames.kt b/app/src/main/kotlin/com/macedonga/visor/stage/Frames.kt index ff14b088..96406734 100644 --- a/app/src/main/kotlin/com/macedonga/visor/stage/Frames.kt +++ b/app/src/main/kotlin/com/macedonga/visor/stage/Frames.kt @@ -1,12 +1,18 @@ package com.macedonga.visor.stage import android.content.Context +import android.graphics.BitmapFactory +import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.graphics.asImageBitmap import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.identity.language.GlassesLook import com.macedonga.visor.core.identity.language.fetchGlassesLook +import com.macedonga.visor.core.identity.language.fetchPairPictures import com.macedonga.visor.log.EventLog import com.macedonga.visor.login.MetaLogin import dagger.hilt.android.qualifiers.ApplicationContext +import java.net.HttpURLConnection +import java.net.URL import java.util.Locale import javax.inject.Inject import javax.inject.Singleton @@ -36,6 +42,9 @@ enum class FrameModel(val asset: String, val styles: Set) { } } +/** Whether Visor has a model of [look]'s frame; an unknown look gets the one there is. */ +fun modelled(look: GlassesLook?): Boolean = look?.style?.lowercase()?.let { style -> FrameModel.entries.any { style in it.styles } } ?: true + /** * The model's material variants to try for [look], best first: one named for the exact colour and lens * (`shiny_transparent_grey_sapphire_transitions`), for a model that has them, then the finish and the lens tint every @@ -91,9 +100,16 @@ class Looks @Inject constructor( val asking: StateFlow = _asking.asStateFlow() val problem: StateFlow = _problem.asStateFlow() + private val _all = MutableStateFlow( + prefs.all.mapNotNull { (key, text) -> if (key.startsWith(LOOK) && text is String) decode(text)?.let { key.removePrefix(LOOK) to it } else null }.toMap(), + ) + + /** Every pair's look known so far, by serial: the home shows each pair as it is. */ + val all: StateFlow> = _all.asStateFlow() + init { scope.launch { - combine(connection.state.map { it.target?.serial ?: it.info?.serial }.distinctUntilChanged(), login.status) { serial, _ -> serial } + combine(connection.pair, connection.state.map { it.info?.serial }.distinctUntilChanged(), login.status) { pair, told, _ -> pair ?: told } .collect { serial -> if (serial != null) ensure(serial, force = false) } } } @@ -104,21 +120,84 @@ class Looks @Inject constructor( scope.launch { ensure(serial, force = true) } } + private val _pictures = MutableStateFlow>(emptyMap()) + + /** Each pair as Meta pictures it, by serial: only in memory, asked for again at each launch. */ + val pictures: StateFlow> = _pictures.asStateFlow() + private val picturing = Mutex() + + /** Ask Meta for the pictures of [serials] it has not given yet. */ + fun wantPictures(serials: List) { + scope.launch { + picturing.withLock { + val missing = serials.filter { it !in _pictures.value } + if (missing.isEmpty() || !login.source.hasCachedLogin) return@withLock + val where = runCatching { fetchPairPictures(login.source.get(), missing) } + .onFailure { log.add("Pictures: ${it.message}") }.getOrNull() ?: return@withLock + for ((serial, picture) in where) { + // The app is dark: the picture made for a dark background. + val bitmap = runCatching { download(picture.dark ?: picture.light) }.onFailure { log.add("Pictures: $serial: ${it.message}") }.getOrNull() + if (bitmap != null) _pictures.value = _pictures.value + (serial to bitmap) + } + } + } + } + + private fun download(url: String): ImageBitmap { + val connection = URL(url).openConnection() as HttpURLConnection + try { + connection.connectTimeout = 20_000 + connection.readTimeout = 20_000 + val bytes = connection.inputStream.use { it.readBytes() } + return checkNotNull(BitmapFactory.decodeByteArray(bytes, 0, bytes.size)) { "not an image" }.asImageBitmap() + } finally { + connection.disconnect() + } + } + + /** Debug builds: a demo pair's look and Meta's picture at [picture], in memory only; a null look takes them away. */ + fun demo(serial: String, look: GlassesLook?, picture: String?) { + if (look == null) { + _all.value = _all.value - serial + _pictures.value = _pictures.value - serial + return + } + _all.value = _all.value + (serial to look) + picture ?: return + scope.launch { + runCatching { download(picture) }.onSuccess { _pictures.value = _pictures.value + (serial to it) } + .onFailure { log.add("Pictures: demo: ${it.message}") } + } + } + + /** Ask for another pair's look, if it is not known yet; the current pair's stays as it is. */ + fun want(serial: String) { + if (serial in _all.value) return + scope.launch { lock.withLock { fetch(serial, force = false) } } + } + private suspend fun ensure(serial: String, force: Boolean) = lock.withLock { this.serial = serial - val kept = prefs.getString(LOOK + serial, null)?.let(::decode) + val kept = _all.value[serial] _look.value = kept if (kept != null) return@withLock + _asking.value = true + val look = fetch(serial, force) + // The pair may have changed while Meta answered. + if (this.serial == serial && look != null) _look.value = look + _asking.value = false + } + + private suspend fun fetch(serial: String, force: Boolean): GlassesLook? { if (!login.source.hasCachedLogin) { _problem.value = LookProblem.LOGGED_OUT - return@withLock + return null } val now = System.currentTimeMillis() - if (!force && now - prefs.getLong(NO_RECORD + serial, 0) < RETRY_MS) return@withLock - _asking.value = true + if (!force && now - prefs.getLong(NO_RECORD + serial, 0) < RETRY_MS) return null // Only language and country: Android adds the regional preferences to the default locale's name. val locale = Locale.getDefault().let { if (it.country.isEmpty()) "en_US" else "${it.language}_${it.country}" } - runCatching { fetchGlassesLook(login.source.get(), serial, locale) } + return runCatching { fetchGlassesLook(login.source.get(), serial, locale) } .onSuccess { look -> if (look == null) { log.add("Look: Meta has no record of $serial") @@ -127,7 +206,7 @@ class Looks @Inject constructor( } else { log.add("Look: ${look.style} ${look.color} ${look.lens}") prefs.edit().putString(LOOK + serial, encode(look)).putString(STYLE, look.style).apply() - _look.value = look + _all.value = _all.value + (serial to look) _problem.value = null } } @@ -135,7 +214,7 @@ class Looks @Inject constructor( log.add("Look: ${it.message}") _problem.value = LookProblem.FAILED } - _asking.value = false + .getOrNull() } private fun encode(l: GlassesLook) = JSONObject() diff --git a/app/src/main/kotlin/com/macedonga/visor/stage/GlassesRenderer.kt b/app/src/main/kotlin/com/macedonga/visor/stage/GlassesRenderer.kt index c0d5fe0b..604efc27 100644 --- a/app/src/main/kotlin/com/macedonga/visor/stage/GlassesRenderer.kt +++ b/app/src/main/kotlin/com/macedonga/visor/stage/GlassesRenderer.kt @@ -49,6 +49,8 @@ class GlassesPose { var tiltPitchDegrees = 0f var shiftX = 0f var shiftY = 0f + /** Sideways out of the light, in stage widths. */ + var slide = 0f /** Its own slow drift, so it never looks frozen. */ var idleYaw = 0f var idlePitch = 0f @@ -251,7 +253,7 @@ class GlassesRenderer(context: Context, private val light: LightFrame, private v } val signature = floatArrayOf( light.level, light.flash, light.core[0], light.core[1], light.core[2], - pose.foldLeft, pose.foldRight, pose.framing, pose.yawDegrees, pose.pitchDegrees, pose.tiltYawDegrees, pose.tiltPitchDegrees, pose.shiftX, pose.shiftY, + pose.foldLeft, pose.foldRight, pose.framing, pose.yawDegrees, pose.pitchDegrees, pose.tiltYawDegrees, pose.tiltPitchDegrees, pose.shiftX, pose.shiftY, pose.slide, pose.idleYaw, pose.idlePitch, pose.idleRoll, pose.idleLift, pose.gestureYaw, pose.led, pose.viewYaw, pose.viewPitch, pose.innerLed, pose.button, if (loaded) 1f else 0f, ) @@ -307,7 +309,8 @@ class GlassesRenderer(context: Context, private val light: LightFrame, private v val m = FloatArray(16) Matrix.setIdentityM(m, 0) // Folded they are flatter, so they settle a little lower into the light. - Matrix.translateM(m, 0, pose.shiftX, pose.shiftY + pose.idleLift - CLOSED_DROP * pose.framing, 0f) + // A stage width is twice the radius the camera fits. + Matrix.translateM(m, 0, pose.shiftX + pose.slide * 2 * FIT_RADIUS.toFloat(), pose.shiftY + pose.idleLift - CLOSED_DROP * pose.framing, 0f) Matrix.rotateM(m, 0, pose.idleRoll, 0f, 0f, 1f) Matrix.rotateM(m, 0, pose.pitchDegrees + pose.tiltPitchDegrees + pose.idlePitch + pose.viewPitch, 1f, 0f, 0f) Matrix.rotateM(m, 0, REST_YAW + pose.yawDegrees + pose.tiltYawDegrees + pose.idleYaw + pose.gestureYaw + pose.viewYaw, 0f, 1f, 0f) diff --git a/app/src/main/kotlin/com/macedonga/visor/stage/SharedStage.kt b/app/src/main/kotlin/com/macedonga/visor/stage/SharedStage.kt index d71965dc..a54480b8 100644 --- a/app/src/main/kotlin/com/macedonga/visor/stage/SharedStage.kt +++ b/app/src/main/kotlin/com/macedonga/visor/stage/SharedStage.kt @@ -6,8 +6,10 @@ import androidx.compose.animation.core.animateFloatAsState import androidx.compose.animation.core.snap import androidx.compose.animation.core.spring import androidx.compose.animation.core.tween +import androidx.compose.foundation.Image import androidx.compose.foundation.gestures.detectHorizontalDragGestures import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.aspectRatio import androidx.compose.foundation.layout.BoxWithConstraints import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth @@ -28,13 +30,17 @@ import androidx.compose.runtime.setValue import androidx.compose.runtime.snapshotFlow import androidx.compose.runtime.withFrameNanos import androidx.compose.ui.Alignment +import androidx.compose.ui.BiasAlignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clipToBounds import androidx.compose.ui.geometry.Offset import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.ImageBitmap import androidx.compose.ui.graphics.TransformOrigin import androidx.compose.ui.graphics.graphicsLayer import androidx.compose.ui.input.pointer.pointerInput +import androidx.compose.ui.input.pointer.util.VelocityTracker +import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.layout.layout import androidx.compose.ui.unit.IntOffset import kotlin.math.abs @@ -125,20 +131,69 @@ fun StageLight(look: LightLook) { } } -/** Dragging sideways turns the pair, and it springs back to rest. */ +/** The pairs a drag on the stage moves between: whether there is one [direction] (1 the next), and going there. */ +class StagePages(val can: (direction: Int) -> Boolean, val go: (direction: Int) -> Unit) + +/** + * Dragging sideways turns the pair, and it springs back to rest. With [pages] it does not turn: it follows the finger + * sideways, and a long drag or a fling takes it out of the light for the next one. + */ @Composable -fun Modifier.turnsThePair(onDetent: () -> Unit): Modifier { +fun Modifier.turnsThePair(pages: StagePages? = null, onDetent: () -> Unit): Modifier { val stage = LocalStage.current val scope = rememberCoroutineScope() val detent = rememberUpdatedState(onDetent) + val paging = rememberUpdatedState(pages) return pointerInput(stage) { var lastDetent = 0 + var travel = 0f + var past = false + val tracker = VelocityTracker() val back = spring(dampingRatio = 0.62f, stiffness = Spring.StiffnessLow) + fun settle() { + scope.launch { stage.yaw.animateTo(0f, back) } + scope.launch { stage.slide.animateTo(0f, back) } + } detectHorizontalDragGestures( - onDragEnd = { scope.launch { stage.yaw.animateTo(0f, back) } }, - onDragCancel = { scope.launch { stage.yaw.animateTo(0f, back) } }, + onDragStart = { + travel = 0f + past = false + tracker.resetTracking() + }, + onDragEnd = { + val pages = paging.value + val fling = tracker.calculateVelocity().x + val direction = when { + pages == null -> 0 + travel < -size.width * PAGE_AT || fling < -PAGE_FLING -> 1 + travel > size.width * PAGE_AT || fling > PAGE_FLING -> -1 + else -> 0 + } + if (direction != 0 && pages!!.can(direction)) { + if (!past) detent.value() + scope.launch { stage.page(direction) { pages.go(direction) } } + } else { + settle() + } + }, + onDragCancel = { settle() }, ) { change, dx -> change.consume() + tracker.addPosition(change.uptimeMillis, change.position) + travel += dx + val pages = paging.value + if (pages != null) { + // Towards a page it follows the finger; at the end of the row it barely moves. A tick says it will go. + val towards = if (travel < 0) 1 else -1 + val can = pages.can(towards) + scope.launch { stage.slide.snapTo(travel / size.width * if (can) FOLLOW else FOLLOW_AT_END) } + val nowPast = can && abs(travel) > size.width * PAGE_AT + if (nowPast != past) { + past = nowPast + if (nowPast) detent.value() + } + return@detectHorizontalDragGestures + } // Free for the first half turn either way, then it resists like a rubber band. val yaw = stage.yaw.value val next = (yaw + dx * DEGREES_PER_PX * (1f - (abs(yaw) / MAX_YAW).coerceIn(0f, 0.9f))).coerceIn(-MAX_YAW, MAX_YAW) @@ -175,6 +230,11 @@ fun SharedStage(stage: StageHolder, shown: Boolean) { snapshotFlow { stage.travelling }.collect { if (it) while (stage.travelling) withFrameNanos(stage::step) } } LaunchedEffect(stage) { snapshotFlow { stage.yaw.value }.collect { stage.pose.yawDegrees = it; stage.renderer.invalidate() } } + LaunchedEffect(stage) { snapshotFlow { stage.slide.value }.collect { stage.pose.slide = it; stage.renderer.invalidate() } } + // At once, not faded: a page changes while the pair is out of sight, and a fade would show both as it slides in. + val picture = stage.picture.takeIf { !stage.vacant } + val pictured = if (picture != null) 1f else 0f + val vacant = if (stage.vacant) 1f else 0f val curtain by animateFloatAsState(if (shown && newest != null) 1f else 0f, if (shown) snap() else tween(CURTAIN_MS), label = "stage") BoxWithConstraints(Modifier.fillMaxSize()) { @@ -208,16 +268,45 @@ fun SharedStage(stage: StageHolder, shown: Boolean) { }, ) { LightPool(stage.light, Offset(0.5f, POOL_Y), Size(0.21f, 0.058f), Size(0.17f, 0.02f), Modifier.fillMaxSize()) - GlassesStage(liveliness = newest?.liveliness ?: 0.25f, active = curtain > 0f, modifier = Modifier.fillMaxSize()) + GlassesStage( + liveliness = newest?.liveliness ?: 0.25f, active = curtain > 0f, + modifier = Modifier.fillMaxSize().graphicsLayer { alpha = 1f - maxOf(pictured, vacant) }, + ) + picture?.let { PairPicture(it, stage, Modifier.fillMaxSize()) } } } } } +/** Meta's picture of the pair, front on where the model would stand, sliding with it. */ +@Composable +private fun PairPicture(picture: ImageBitmap, stage: StageHolder, modifier: Modifier) { + BoxWithConstraints(modifier) { + Image( + picture, null, + Modifier + .align(BiasAlignment(0f, PICTURE_Y)) + .fillMaxWidth(PICTURE_WIDTH) + // Its own shape, so it fills the width rather than drawing at its pixel size. + .aspectRatio(picture.width.toFloat() / picture.height) + .graphicsLayer { translationX = stage.slide.value * size.width / PICTURE_WIDTH }, + contentScale = ContentScale.Fit, + ) + } +} + /** How the pair travels from one screen's place to the next. */ internal val HANDOVER_EASE = CubicBezierEasing(0.2f, 0f, 0f, 1f) internal const val HANDOVER_NS = 640_000_000L private const val CURTAIN_MS = 180 +// As wide as the model looks on the home, and low enough that it ends in the light where the model's temples do. +private const val PICTURE_WIDTH = 0.56f +private const val PICTURE_Y = 0.1f private const val DEGREES_PER_PX = 0.32f private const val MAX_YAW = 160f private const val DETENT_DEGREES = 30f +// A third of the width, or a flick, goes to the next pair. +private const val PAGE_AT = 0.33f +private const val PAGE_FLING = 1_400f +private const val FOLLOW = 0.9f +private const val FOLLOW_AT_END = 0.12f diff --git a/app/src/main/kotlin/com/macedonga/visor/stage/StageHolder.kt b/app/src/main/kotlin/com/macedonga/visor/stage/StageHolder.kt index d3140617..d460fd5c 100644 --- a/app/src/main/kotlin/com/macedonga/visor/stage/StageHolder.kt +++ b/app/src/main/kotlin/com/macedonga/visor/stage/StageHolder.kt @@ -2,14 +2,23 @@ package com.macedonga.visor.stage import android.content.Context import androidx.compose.animation.core.Animatable +import androidx.compose.animation.core.FastOutLinearInEasing +import androidx.compose.animation.core.spring +import androidx.compose.animation.core.tween import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableFloatStateOf import androidx.compose.runtime.mutableStateListOf +import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue import androidx.compose.runtime.staticCompositionLocalOf import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.ImageBitmap import androidx.compose.ui.platform.LocalContext +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch /** * The glasses, their light and their pose for the life of the activity, above navigation: the model loads once and @@ -31,6 +40,23 @@ class StageHolder(private val context: Context, markAtLaunch: Boolean) { var fromLive = false /** Turned by a finger; springs back to 0. */ val yaw = Animatable(0f) + /** Sideways out of the light, in stage widths: a pair leaving for the next one. */ + val slide = Animatable(0f) + /** A pair Visor has no model for, or one not connected, shows as Meta pictures it; null shows the model. */ + var picture by mutableStateOf(null) + /** Nothing under the light: the home's page for adding another pair. */ + var vacant by mutableStateOf(false) + + /** The pair slides out [direction] (1 to the left), [swap] puts the next one up, and it slides in from the far side. */ + suspend fun page(direction: Int, swap: () -> Unit) { + coroutineScope { + launch { yaw.animateTo(0f, tween(PAGE_OUT_MS, easing = FastOutLinearInEasing)) } + slide.animateTo(-direction.toFloat(), tween(PAGE_OUT_MS, easing = FastOutLinearInEasing)) + } + swap() + slide.snapTo(direction.toFloat()) + slide.animateTo(0f, spring(dampingRatio = 0.86f, stiffness = 260f)) + } internal val spots = mutableStateListOf() internal val looks = mutableStateListOf() @@ -120,3 +146,5 @@ fun rememberStageHolder(markAtLaunch: Boolean): StageHolder { DisposableEffect(stage) { onDispose { stage.destroy() } } return stage } + +private const val PAGE_OUT_MS = 200 diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 7127e48c..0edb2fd6 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -61,6 +61,8 @@ %d h fa %d g fa Importo %d%% + Visti %s + Non connessi Importo %d nuovo @@ -411,6 +413,7 @@ Occhiali di %s Ray-Ban Meta Ray-Ban Meta %s + %1$s, %2$s Montatura Colore Lenti @@ -441,6 +444,7 @@ Non collegati Scollega Collega + %1$d%% · custodia %2$d%% Account Meta Accesso a Meta fatto Nessun accesso a Meta diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index c2cb33cf..0ef46253 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -62,6 +62,8 @@ %d h ago %d d ago Importing %d%% + Seen %s + Not connected Importing %d new @@ -412,6 +414,7 @@ %s\'s glasses Ray-Ban Meta Ray-Ban Meta %s + %1$s, %2$s Frame Colour Lenses @@ -442,6 +445,7 @@ Not connected Disconnect Connect + %1$d%% · case %2$d%% Meta account Logged in to Meta Not logged in to Meta diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt index 145bc80f..0e4d0073 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt @@ -34,6 +34,8 @@ class GlassesConnection( private val hosts: SessionHostFactory, // Injectable so a test can keep the store reads on its own virtual clock. private val ioDispatcher: CoroutineDispatcher = Dispatchers.IO, + // False for a quick look at another pair, which leaves the choice of pair as it was. + private val chooses: Boolean = true, ) { private val _state = MutableStateFlow(GlassesState()) @@ -63,6 +65,11 @@ class GlassesConnection( private var job: Job? = null + private val _pair = MutableStateFlow(runCatching { store.currentSerial() }.getOrNull()) + + /** The serial of the pair Visor is for: the one connected, or chosen last. What Visor keeps per pair follows it. */ + val pair: StateFlow = _pair.asStateFlow() + private val _running = MutableStateFlow(false) /** True from [start] until [stop] has let go of the socket. */ @@ -86,25 +93,22 @@ class GlassesConnection( lastOverride = override _running.value = true job = scope.launch { - // Both reads move together, not one withContext each: see the class doc for why - // they leave the caller's dispatcher at all. - val (bundles, bundle) = withContext(ioDispatcher) { - val bundles = store.all() - val bundle = if (override != null) { - bundles.firstOrNull { it.serial == override.serial } - } else { - store.load() - } - bundles to bundle + // The reads and the choice move together, not one withContext each: see the class doc for why + // they leave the caller's dispatcher at all. The pair connected to is the one chosen from now on. + val bundle = withContext(ioDispatcher) { + val bundle = if (override != null) store.all().firstOrNull { it.serial == override.serial } else store.chosen() + if (chooses) (override?.serial ?: bundle?.serial)?.let(store::choose) + bundle } val target = override ?: bundle?.let { GlassesTarget(GlassesLink.Bluetooth(it.glassesAddress), it.serial) } if (target == null) { - _state.value = GlassesState(reason = idleReason(bundles)) + _state.value = GlassesState(reason = "No glasses are set up on this phone yet. Set up glasses to start.") return@launch } + if (chooses) _pair.value = target.serial _state.value = GlassesState(target = target) val opened = hosts.create(target, bundle) host = opened @@ -275,14 +279,6 @@ class GlassesConnection( _state.value = _state.value.copy(diagnosis = checked.diagnosis) } - /** Why there is nothing to connect to, in words a person can act on. */ - private fun idleReason(bundles: List): String = when { - bundles.isEmpty() -> - "No glasses are set up on this phone yet. Set up glasses to start." - else -> - "More than one pair of glasses is set up on this phone. Choose which one to use." - } - companion object { /** 20 seconds, from `__main__.py`'s `_DEFAULT_TIMEOUT_SECONDS`. */ const val CANARY_TIMEOUT_MILLIS: Long = 20_000 diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/IdentityStore.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/IdentityStore.kt index 0aaa0925..a98665f8 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/IdentityStore.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/IdentityStore.kt @@ -21,6 +21,23 @@ class IdentityStore(directory: File, private val sealer: Sealer) { return bundles.firstOrNull { it.serial == serial } } + /** The pair last chosen, else the only one, else the newest set up. */ + fun chosen(): IdentityBundle? { + val bundles = all() + return bundles.firstOrNull { it.serial == currentSerial() } ?: bundles.singleOrNull() ?: bundles.maxByOrNull { it.createdAt } + } + + /** The serial last chosen, kept beside the store and unsealed: it is no secret, and reading it needs no key. */ + fun currentSerial(): String? = currentPath.takeIf { it.isFile }?.readText()?.trim()?.takeIf { it.isNotEmpty() } + + fun choose(serial: String) { + if (serial == currentSerial()) return + directory.mkdirs() + currentPath.writeText(serial) + } + + private val currentPath: File get() = File(directory, CURRENT_FILE) + /** The one owner key this account uses, or null before the first import. */ fun ownerKeyPem(): ByteArray? = all().firstOrNull()?.privateKeyPem @@ -41,6 +58,7 @@ class IdentityStore(directory: File, private val sealer: Sealer) { val bundles = all().filter { it.serial != serial } directory.mkdirs() writeAtomically(sealer.seal(bundlesToJson(bundles).toByteArray(Charsets.UTF_8))) + if (currentSerial() == serial) currentPath.delete() } /** DIVERGENCE: `store.py` calls `Path.write_bytes`, which truncates first. */ @@ -56,6 +74,7 @@ class IdentityStore(directory: File, private val sealer: Sealer) { companion object { const val STORE_FILE = "identities.sealed" + const val CURRENT_FILE = "current" } // DIVERGENCE: `migrate_legacy` does not port. diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/PairPicture.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/PairPicture.kt new file mode 100644 index 00000000..ddeac9f2 --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/PairPicture.kt @@ -0,0 +1,70 @@ +package com.macedonga.visor.core.identity.language + +import com.macedonga.visor.core.identity.HttpPost +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.pyUrlencode +import com.macedonga.visor.core.identity.token.Token +import com.macedonga.visor.core.identity.token.TokenError +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.intOrNull +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.putJsonArray + +/** Where Meta keeps a pair's picture: the frame, colour and lenses as sold, front on, on a clear background. */ +data class PairPicture(val light: String, val dark: String?) + +private const val GRAPH_URL = "https://ar-genai.graph.meta.com/graphql" +private const val ASSETS_BY_SERIAL_DOC = "256686669512680180239290821066" +// Asset category 2 is the device image the Meta app shows for each pair. +private const val DEVICE_IMAGE = 2 + +/** + * WearableDeviceAssetsBySerialNumbers, the Meta app's own query: each pair's picture by serial. The picture for the + * pair's SKU comes first; the one for its device type, a stock frame, only when there is none. + */ +fun fetchPairPictures(token: Token, serials: List, http: HttpPost = UrlConnectionPost): Map { + val body = pyUrlencode( + linkedMapOf( + "fb_api_req_friendly_name" to "WearableDeviceAssetsBySerialNumbers", + "client_doc_id" to ASSETS_BY_SERIAL_DOC, + "variables" to buildJsonObject { putJsonArray("serial_numbers") { serials.forEach { add(JsonPrimitive(it)) } } }.toString(), + "method" to "post", + "pretty" to "false", + "format" to "json", + ), + ) + val reply = try { + http.post( + GRAPH_URL, + mapOf("Authorization" to "OAuth ${token.value}", "Content-Type" to "application/x-www-form-urlencoded"), + body.toByteArray(Charsets.US_ASCII), + ) + } catch (error: Exception) { + throw TokenError("Meta could not be reached: ${error.message}") + } + val json = runCatching { Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject } + .getOrElse { throw TokenError("Meta answered ${reply.status} with no JSON") } + json["errors"]?.let { throw TokenError("Meta refused: ${it.toString().take(200)}") } + val categories = ((json["data"] as? JsonObject)?.get("xar_wearable_device_assets_by_serial_numbers") as? JsonObject)?.get("data") as? JsonArray + ?: return emptyMap() + val images = categories.mapNotNull { it as? JsonObject }.firstOrNull { (it["asset_category"] as? JsonPrimitive)?.intOrNull == DEVICE_IMAGE } + ?: return emptyMap() + return (images["serial_specific"] as? JsonArray).orEmpty().mapNotNull { entry -> + val o = entry as? JsonObject ?: return@mapNotNull null + val serial = o.text("serial_number") ?: return@mapNotNull null + val picture = o.image("sku_specific") ?: o.image("device_specific") ?: return@mapNotNull null + serial to picture + }.toMap() +} + +private fun JsonObject.image(group: String): PairPicture? = + ((this[group] as? JsonObject)?.get("assets") as? JsonArray).orEmpty().mapNotNull { it as? JsonObject } + .firstOrNull { it.text("asset_type") == "IMAGE" && it.text("status") == "ACTIVE" && it.text("asset_manager_url") != null } + ?.let { PairPicture(it.text("asset_manager_url")!!, it.text("asset_manager_url_dark")) } + +private fun JsonObject.text(key: String): String? = (this[key] as? JsonPrimitive)?.contentOrNull?.takeIf { it.isNotBlank() } diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/GlassesConfig.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/GlassesConfig.kt index 52283407..2a4c86ef 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/GlassesConfig.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/GlassesConfig.kt @@ -24,6 +24,8 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex @@ -66,7 +68,12 @@ class GlassesConfig( private val airship: AirshipReceiver, private val media: MediaImport, ) { - private val dir = File(context.filesDir, "mobileconfig").apply { mkdirs() } + private val root = File(context.filesDir, "mobileconfig").apply { mkdirs() } + + // Each pair's own folder: their parameter maps differ with their firmware, and a table built on one pair's values + // must never reach another. + @Volatile + private var dir = File(root, connection.pair.value ?: NO_PAIR) private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private val _state = MutableStateFlow(ConfigState()) val state: StateFlow = _state.asStateFlow() @@ -81,7 +88,29 @@ class GlassesConfig( init { media.bugReportReader = { take(it) } - scope.launch { runCatching { load() } } + scope.launch { + connection.pair.filterNotNull().distinctUntilChanged().collect { serial -> + writes.withLock { runCatching { switchTo(serial) } } + } + } + } + + /** The pair's own values, or none yet; what the last pair had stays in its folder. */ + private fun switchTo(serial: String) { + dir = File(root, serial) + // Before Visor kept pairs apart, the one pair's files sat in the root. + val loose = root.listFiles { f -> f.isFile }.orEmpty() + if (loose.isNotEmpty() && !dir.exists()) { + dir.mkdirs() + loose.forEach { it.renameTo(File(dir, it.name)) } + } + dir.mkdirs() + schemaHash = "" + counts = IntArray(4) + sessionId = "" + boot = null + _state.value = ConfigState() + load() } /** Ask the glasses for a bug report and read their configuration out of it. */ @@ -304,6 +333,7 @@ class GlassesConfig( const val SENT = "sent_at.txt" const val TIMESTAMP = "current_timestamp.txt" const val LAST_STAMP = "last_stamp.txt" + private const val NO_PAIR = "none" const val STAMP = "MMM-dd-yyyy-HH-mm-ss" const val STALE_MARGIN_MS = 15 * 60_000L const val READ_TIMEOUT_MS = 10 * 60_000L diff --git a/core/settings/src/main/kotlin/com/macedonga/visor/core/settings/GlassesSettings.kt b/core/settings/src/main/kotlin/com/macedonga/visor/core/settings/GlassesSettings.kt index 8de23555..edbffb8d 100644 --- a/core/settings/src/main/kotlin/com/macedonga/visor/core/settings/GlassesSettings.kt +++ b/core/settings/src/main/kotlin/com/macedonga/visor/core/settings/GlassesSettings.kt @@ -18,6 +18,7 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asSharedFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch @@ -75,6 +76,8 @@ data class SettingsState( val problem: String = "", // The Meta product id of the glasses last connected (266 for Ray-Ban Meta Gen 2), for what the catalogue offers. val product: Int? = null, + // The serial of the pair these are, so a reader never takes them for another's. + val pair: String? = null, ) /** @@ -90,7 +93,7 @@ class GlassesSettings(private val connection: GlassesConnection, drive: RlDriveR private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - private val _state = MutableStateFlow(SettingsState()) + private val _state = MutableStateFlow(SettingsState(pair = connection.pair.value)) val state: StateFlow = _state.asStateFlow() private val _notes = MutableSharedFlow(extraBufferCapacity = 64) @@ -128,8 +131,22 @@ class GlassesSettings(private val connection: GlassesConnection, drive: RlDriveR init { drive.handler = ::onReply + // Another pair's settings are not these: what was read goes, and the next read is a full one. scope.launch { - connection.state.map { it.info?.build }.distinctUntilChanged().collect { build -> + connection.pair.filterNotNull().distinctUntilChanged().collect { serial -> + reads.withLock { + if (_state.value.pair == serial) return@withLock + versions = emptyMap() + byNamespace = emptyMap() + raw.clear() + retried.clear() + writtenAt.clear() + _state.value = SettingsState(pair = serial) + } + } + } + scope.launch { + connection.state.map { it.target?.serial to it.info?.build }.distinctUntilChanged().collect { (_, build) -> productId(build)?.let { id -> _state.update { it.copy(product = id) } } } } From 93015e6eba4475b35008f37b5bf86a9da8fff36f Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Sun, 18 Oct 2026 13:29:05 +0200 Subject: [PATCH 02/90] Debug API: DEMO_PAIR puts a second pair on the home, a black Skyler never connected or looked at, to try the pair pages with one pair --- .../macedonga/visor/debug/DebugReceiver.kt | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index 28284f43..d542cab0 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -36,7 +36,8 @@ import kotlinx.coroutines.withTimeoutOrNull * recording), LIVE_REPLAY, LIVE_FRAME, BUGREPORT, SETTINGS (--es key), SET (--es key --es value), * READ_SETTINGS, MEDIA, IMPORT (--es handle a,b: by handle prefix), ASK (--es q), ONBOARD (--es page --ef t), * MCP_PROBE / MCP_ADD (--es url --es name --es auth NONE|TOKEN|HEADER --es secret --es header), MCP_STATUS, - * MCP_TOOLS (what the model would get), MCP_CALL (--es tool exposed-name --es args json), MCP_REMOVE (--es id). + * MCP_TOOLS (what the model would get), MCP_CALL (--es tool exposed-name --es args json), MCP_REMOVE (--es id), + * DEMO_PAIR (--ez on false takes it away): a second pair on the home, a black Skyler, never connected or looked at. */ @AndroidEntryPoint class DebugReceiver : BroadcastReceiver() { @@ -53,12 +54,15 @@ class DebugReceiver : BroadcastReceiver() { @Inject lateinit var config: com.macedonga.visor.core.media.GlassesConfig @Inject lateinit var liveSettings: com.macedonga.visor.core.media.LiveSettingsStore @Inject lateinit var mcp: com.macedonga.visor.core.assistant.mcp.McpHub + @Inject lateinit var pairs: com.macedonga.visor.pairs.Pairs + @Inject lateinit var looks: com.macedonga.visor.stage.Looks override fun onReceive(context: Context, intent: Intent) { val action = intent.action?.substringAfterLast('.').orEmpty() Log.i("Visor", "API: $action") when (action) { "STATUS" -> resultData = status() + "DEMO_PAIR" -> resultData = demoPair(intent.getBooleanExtra("on", true)) "CONNECT" -> resultData = runCatching { GlassesService.start(context); "connecting" }.getOrElse { "could not start the service: $it" } "DISCONNECT" -> { GlassesService.stop(context) @@ -351,6 +355,19 @@ class DebugReceiver : BroadcastReceiver() { } } + private fun demoPair(on: Boolean): String { + if (!on) { + pairs.demo(null, null) + looks.demo(DEMO_SERIAL, null, null) + return "demo pair gone" + } + val look = com.macedonga.visor.core.identity.language.GlassesLook("skyler", "Skyler", "shiny_black", "Shiny Black", "clear", "Clear", null) + looks.demo(DEMO_SERIAL, look, DEMO_PICTURE) + val seen = com.macedonga.visor.home.PairSeen(glasses = 64, case = 88, at = System.currentTimeMillis() - 2 * 3_600_000L, folded = true) + pairs.demo(com.macedonga.visor.pairs.KnownPair(DEMO_SERIAL, "00:00:00:00:00:00"), seen) + return "demo pair on: swipe the home's stage" + } + private fun status(): String { val s = connection.state.value val b = s.device?.battery @@ -388,5 +405,9 @@ class DebugReceiver : BroadcastReceiver() { private companion object { val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + const val DEMO_SERIAL = "DEMO00SKYLER00" + // Meta's own picture of a black Skyler, as its asset service hands it out. + const val DEMO_PICTURE = "https://scontent.xx.fbcdn.net/mci_ab/uap/asset_manager/name_revision/?ab_b=e&ab_page=AssetManagerNameAndRevision" + + "&ab_entry=RBM2-Skyler-ShinyBlack-Clear-Concord&set=mwa_device_image_assets&revision=1837680503843762&density=3&mode=dark" } } From a0d5e6dea38ee7df87734f3dcbd7ebd3e817d3f0 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Mon, 19 Oct 2026 21:02:39 +0200 Subject: [PATCH 03/90] Link: a peer that leaves out AirShield's Multikey, as the Neural Band does (parameters 27 to the glasses' 31), gets one HKDF key for AES and HMAC, which ended its link at the first frame with a MAC failure; the peer's EndLinkSetup is taken on a channel of its own, 0x9001 on the band; the LE transport takes a default PSM; debug builds trace small DataX payloads whole --- .../kotlin/com/macedonga/visor/core/protocol/Kdf.kt | 8 +++++++- .../macedonga/visor/core/session/DataXConnection.kt | 4 ++++ .../macedonga/visor/core/session/GlassesSession.kt | 13 ++++++++++--- .../com/macedonga/visor/core/session/SecureLink.kt | 9 +++++++-- .../visor/core/transport/BluetoothLeTransport.kt | 4 +++- 5 files changed, 31 insertions(+), 7 deletions(-) diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Kdf.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Kdf.kt index 7b373e44..80333975 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Kdf.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Kdf.kt @@ -10,6 +10,9 @@ object Kdf { val AIRSHIELD_INFO: ByteArray = "AirShield".toByteArray(Charsets.US_ASCII) val HMAC_DERIVE_LABEL: ByteArray = "hmac_derive".toByteArray(Charsets.US_ASCII) + /** EncryptionParameters bit 4: a separate HMAC key. The Neural Band leaves it out (27, the glasses 31). */ + const val MULTIKEY: Int = 4 + /** Derive ONE direction's key set, in the live-confirmed construction. */ fun deriveDirectionKeys( ecdhSecret: ByteArray, @@ -17,13 +20,16 @@ object Kdf { seed: ByteArray, preambleIv: ByteArray, info: ByteArray = AIRSHIELD_INFO, + // Without Multikey on both ends the AES key is the HMAC key too (CipherBuilder::buildFramingKeys). + multikey: Boolean = true, ): SessionKeys { require(ecdhSecret.size == 32) { "ECDH secret must be 32 bytes, got ${ecdhSecret.size}" } require(preambleIv.size == 16) { "preamble IV must be 16 bytes, got ${preambleIv.size}" } val aesSalt = sha256(nonce, seed) - val hmacSalt = sha256(seed, nonce, HMAC_DERIVE_LABEL) val aesKey = hkdfExpand(hkdfExtract(aesSalt, ecdhSecret), info, 32) + if (!multikey) return SessionKeys(aesKey = aesKey, hmacKey = aesKey, iv = preambleIv) + val hmacSalt = sha256(seed, nonce, HMAC_DERIVE_LABEL) val hmacKey = hkdfExpand(hkdfExtract(hmacSalt, ecdhSecret), info, 32) return SessionKeys(aesKey = aesKey, hmacKey = hmacKey, iv = preambleIv) } diff --git a/core/session/src/main/kotlin/com/macedonga/visor/core/session/DataXConnection.kt b/core/session/src/main/kotlin/com/macedonga/visor/core/session/DataXConnection.kt index da729199..83143a87 100644 --- a/core/session/src/main/kotlin/com/macedonga/visor/core/session/DataXConnection.kt +++ b/core/session/src/main/kotlin/com/macedonga/visor/core/session/DataXConnection.kt @@ -171,6 +171,8 @@ class DataXConnection( append("\n msg ch=0x%04x svc=%d mt=0x%04x len=%d hdr=%d payload=%d".format( m.info.channelId, m.info.serviceId, m.info.messageType, m.info.length, m.info.headerLen, m.payload.size, )) + // Small ones whole: what a new service says is read from here. + if (m.payload.size <= TRACE_PAYLOAD) append(" data=").append(hex(m.payload)) } if (carry.isNotEmpty()) append("\n CARRY %d".format(carry.size)) } @@ -216,6 +218,8 @@ class DataXConnection( @Volatile var trace: ((String) -> Unit)? = null + private const val TRACE_PAYLOAD = 512 + /** The ack channel for a peer-originated channel: clear the secure bit, 0x8000. */ fun mirror(channelId: Int): Int = channelId and 0x7FFF } diff --git a/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt b/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt index 9063fb0b..4fe2eeea 100644 --- a/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt +++ b/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt @@ -204,9 +204,7 @@ class GlassesSession( withLinkDeadline(END_LINK_SETUP_TIMEOUT_MILLIS, "the glasses did not send their EndLinkSetup") { while (true) { val message = mux.receive() - if (message.info.channelId == LINK_SETUP_CHANNEL && - message.info.messageType == LinkSetupMsgType.END_LINK_SETUP.value - ) { + if (isTheirEndLinkSetup(message)) { mux.sendStatus(DataXConnection.mirror(message.info.channelId), SUCCESS) return@withLinkDeadline } @@ -215,6 +213,15 @@ class GlassesSession( } } + // The glasses answer on the link setup channel our RequestEncryption opened; the Neural Band on one of its own, + // 0x9001, that names no service. + private fun isTheirEndLinkSetup(message: DataXMessage): Boolean { + val info = message.info + if (info.messageType != LinkSetupMsgType.END_LINK_SETUP.value || info.flags.hasError) return false + return info.channelId == LINK_SETUP_CHANNEL || + ((info.channelId and 0x8000) != 0 && info.serviceId == 0 && registry.serviceForInbound(info.channelId) == 0) + } + /** Drain what the glasses said during link setup before anyone allocates a local channel. */ private suspend fun settleChannels(mux: DataXConnection) { while (!stopped) { diff --git a/core/session/src/main/kotlin/com/macedonga/visor/core/session/SecureLink.kt b/core/session/src/main/kotlin/com/macedonga/visor/core/session/SecureLink.kt index 1c264483..d89a017f 100644 --- a/core/session/src/main/kotlin/com/macedonga/visor/core/session/SecureLink.kt +++ b/core/session/src/main/kotlin/com/macedonga/visor/core/session/SecureLink.kt @@ -137,10 +137,12 @@ class SecureLink( ecdhSecret = secret, ) + // What both ends support is what the link uses: the band offers 27, no Multikey. + val multikey = (OUR_PARAMETERS and request.supportedParameters and enable.parameters and Kdf.MULTIKEY) != 0 // Keep exactly as measured: TX takes the peer's RequestEncryption challenge as nonce and // our seed. - val rxKeys = Kdf.deriveDirectionKeys(secret, myChallenge, enable.seed, enable.iv) - val txKeys = Kdf.deriveDirectionKeys(secret, request.challenge, mySeed, myIv) + val rxKeys = Kdf.deriveDirectionKeys(secret, myChallenge, enable.seed, enable.iv, multikey = multikey) + val txKeys = Kdf.deriveDirectionKeys(secret, request.challenge, mySeed, myIv, multikey = multikey) rxCrypto = FrameCrypto(rxKeys, initRxSeq = enable.base.toInt()) txCrypto = FrameCrypto(txKeys, initTxSeq = mySeq.toInt()) @@ -224,5 +226,8 @@ class SecureLink( /** How many frames [open] will read while waiting for the peer's two messages. */ const val HANDSHAKE_READS = 8 + + /** The parameters this side's RequestEncryption and EnableEncryption announce. */ + const val OUR_PARAMETERS = 31 } } \ No newline at end of file diff --git a/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt b/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt index 76556fa0..81941f99 100644 --- a/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt +++ b/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt @@ -37,6 +37,8 @@ class BluetoothLeTransport( private val wireLog: WireLog? = null, private val keeper: LeKeeper? = null, private val io: CoroutineDispatcher = Dispatchers.IO, + // The Neural Band's is 129. + private val defaultPsm: Int = META_LE_DEFAULT_PSM, ) : Transport { @Volatile private var socket: BluetoothSocket? = null @@ -51,7 +53,7 @@ class BluetoothLeTransport( } val device = context.getSystemService(BluetoothManager::class.java)?.adapter?.getRemoteDevice(address) ?: throw TransportConnectException(ConnectFailure.BLUETOOTH_OFF, "Bluetooth is off") - val psm = keeper?.psm ?: withTimeoutOrNull(GATT_MS) { readPsm(device) } ?: META_LE_DEFAULT_PSM + val psm = keeper?.psm ?: withTimeoutOrNull(GATT_MS) { readPsm(device) } ?: defaultPsm keeper?.note("opening L2CAP psm $psm, LE link ${if (keeper.connected.value) "up" else "down"}") withContext(io) { val opening = try { From be963fe78f5d635e493ba1abd42d62cbff32337c Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Mon, 19 Oct 2026 21:02:39 +0200 Subject: [PATCH 04/90] Setup: pair_request carries the device's secondary certificate, IdentityResponse field 5, as the Meta app sends it (the Neural Band has one, and Meta's pair answered 500 without it); provisioning hands back the device certificate and the device_ec_public_key of Meta's final receipt, and AirShield key hints can be checked against a key --- .../macedonga/visor/core/identity/KeyHint.kt | 46 +++++++++++++++++++ .../core/identity/provision/Provision.kt | 21 ++++++--- .../identity/provision/ProvisionSession.kt | 15 +++++- 3 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 core/identity/src/main/kotlin/com/macedonga/visor/core/identity/KeyHint.kt diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/KeyHint.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/KeyHint.kt new file mode 100644 index 00000000..1b131035 --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/KeyHint.kt @@ -0,0 +1,46 @@ +package com.macedonga.visor.core.identity + +import java.io.ByteArrayInputStream +import java.security.KeyFactory +import java.security.MessageDigest +import java.security.cert.CertificateFactory +import java.security.interfaces.ECPublicKey +import java.security.spec.X509EncodedKeySpec + +/** The 64 raw bytes, X then Y, of the EC key in a device certificate, as svc36's IdentityResponse carries it. */ +fun devicePublicKey(certDer: ByteArray): ByteArray? = runCatching { + val cert = CertificateFactory.getInstance("X.509").generateCertificate(ByteArrayInputStream(certDer)) + point64(cert.publicKey as ECPublicKey) +}.getOrNull() + +/** A P-256 public key as 64 raw bytes, from raw, 0x04-prefixed or DER (SubjectPublicKeyInfo) form. */ +fun rawPublicKey(encoded: ByteArray): ByteArray? = when { + encoded.size == 64 -> encoded + encoded.size == 65 && encoded[0] == 0x04.toByte() -> encoded.copyOfRange(1, 65) + else -> runCatching { + point64(KeyFactory.getInstance("EC").generatePublic(X509EncodedKeySpec(encoded)) as ECPublicKey) + }.getOrNull() +} + +private fun point64(key: ECPublicKey): ByteArray = fixed32(key.w.affineX.toByteArray()) + fixed32(key.w.affineY.toByteArray()) + +/** + * Whether [hint] names [publicKey]: AirShield's key hint is a 4-byte salt then SHA256(SHA256(key) ‖ salt)[:8] + * (generateKeyHint, research PROTOCOL-REFERENCE). + */ +fun keyHintMatches(hint: ByteArray, publicKey: ByteArray): Boolean { + if (hint.size != 12 || publicKey.size != 64) return false + val salt = hint.copyOfRange(0, 4) + val sha = MessageDigest.getInstance("SHA-256") + val inner = sha.digest(publicKey) + sha.update(inner) + sha.update(salt) + return sha.digest().copyOfRange(0, 8).contentEquals(hint.copyOfRange(4, 12)) +} + +// A BigInteger's bytes carry a sign byte or lose leading zeros; a coordinate is always 32. +private fun fixed32(b: ByteArray): ByteArray = when { + b.size == 32 -> b + b.size > 32 -> b.copyOfRange(b.size - 32, b.size) + else -> ByteArray(32 - b.size) + b +} diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/Provision.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/Provision.kt index 26d00f6a..da2970b4 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/Provision.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/Provision.kt @@ -61,12 +61,12 @@ class Receipt(val signature: ByteArray, val bodyJson: ByteArray) { } } -/** IdentityResponse fields 1 and 2. */ -class DeviceIdentity(val certDer: ByteArray, val serial: String) { +/** IdentityResponse {1: certificate, 2: serial, 3: registration challenge, 4: DRK present, 5: secondary certificate}. */ +class DeviceIdentity(val certDer: ByteArray, val serial: String, val secondaryCert: ByteArray = ByteArray(0)) { companion object { fun decode(data: ByteArray): DeviceIdentity { val fields = Wire.parse(data) - return DeviceIdentity(certDer = fields.bytes(1), serial = String(fields.bytes(2), Charsets.US_ASCII)) + return DeviceIdentity(certDer = fields.bytes(1), serial = String(fields.bytes(2), Charsets.US_ASCII), secondaryCert = fields.bytes(5)) } } } @@ -134,7 +134,7 @@ class PairingBackend( ) } - fun pairRequest(certDer: ByteArray, serial: String, deviceNonce: ByteArray, appPubkey: ByteArray): Receipt { + fun pairRequest(certDer: ByteArray, serial: String, deviceNonce: ByteArray, appPubkey: ByteArray, secondaryCert: ByteArray = ByteArray(0)): Receipt { val fields = common() fields["device_cert"] = Base64.getEncoder().encodeToString(certDer) fields["serial_number"] = serial @@ -142,7 +142,8 @@ class PairingBackend( buildJsonObject { put("device_nonce", Base64.getEncoder().encodeToString(deviceNonce)) put("app_pubkey", Base64.getEncoder().encodeToString(appPubkey)) - put("secondary_cert", "") + // IdentityResponse field 5, as the Meta app sends it; glasses have none. + put("secondary_cert", if (secondaryCert.isEmpty()) "" else Base64.getEncoder().encodeToString(secondaryCert)) }, ) return post("pair_request", fields, "pending_ownership_receipt") @@ -189,6 +190,10 @@ class ProvisioningFlow( private val channel = connection.openChannel(ServiceId.IDENTITY.value) private var first = true + /** The device's certificate from its IdentityResponse, once [run] has it. */ + var deviceCert: ByteArray? = null + private set + private suspend fun send(messageType: IdentityMsgType, payload: ByteArray = ByteArray(0)) { // A peer names a channel's service only on that channel's first message; later ones // carry service_id 0. @@ -224,6 +229,7 @@ class ProvisioningFlow( onStep(ProvisionStep.IDENTITY) send(IdentityMsgType.IDENTITY_REQUEST) val identity = DeviceIdentity.decode(await(IdentityMsgType.IDENTITY_RESPONSE)) + deviceCert = identity.certDer onStep(ProvisionStep.SKIP_CHALLENGE) send(IdentityMsgType.SKIP_CHALLENGE) @@ -231,7 +237,10 @@ class ProvisioningFlow( onStep(ProvisionStep.PAIR_REQUEST) val pending = withContext(Dispatchers.IO) { - backend.pairRequest(certDer = identity.certDer, serial = identity.serial, deviceNonce = nonce, appPubkey = appPubkey) + backend.pairRequest( + certDer = identity.certDer, serial = identity.serial, deviceNonce = nonce, appPubkey = appPubkey, + secondaryCert = identity.secondaryCert, + ) } onStep(ProvisionStep.START_CHANGE_OWNER) diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/ProvisionSession.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/ProvisionSession.kt index 17266091..e4057635 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/ProvisionSession.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/ProvisionSession.kt @@ -12,6 +12,7 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import kotlinx.serialization.json.jsonObject /** Drive one provisioning connection: link up, change owner, hand back the key. */ class ProvisioningResult( @@ -20,6 +21,9 @@ class ProvisioningResult( /** DIVERGENCE: the reference returns the path it wrote. The key itself comes back here. */ val keyPem: ByteArray, val reusedKey: Boolean, + val deviceCert: ByteArray? = null, + /** additional_data.device_ec_public_key in Meta's final receipt, as the Meta app stores it; lease hints name it. */ + val deviceEcPublicKey: ByteArray? = null, ) /** Run change owner over [transport] and return the key it installed. */ @@ -37,6 +41,7 @@ suspend fun provision( onStep(ProvisionStep.CONNECT) transport.connect() + var deviceCert: ByteArray? = null val final = try { coroutineScope { onStep(ProvisionStep.SECURE_LINK) @@ -45,7 +50,8 @@ suspend fun provision( val connection = DataXConnection(link, ChannelRegistry()) val reader = connection.startReader(this) try { - ProvisioningFlow(connection, backend, key.publicKey64, timeoutMillis, onStep).run() + val flow = ProvisioningFlow(connection, backend, key.publicKey64, timeoutMillis, onStep) + flow.run().also { deviceCert = flow.deviceCert } } finally { // Close first: a read blocked on the socket does not answer cancellation, and // the reader would hold this scope open until it did. @@ -63,6 +69,13 @@ suspend fun provision( userId = info.textOf("user_id"), keyPem = key.pem, reusedKey = existingKeyPem != null, + deviceCert = deviceCert, + deviceEcPublicKey = runCatching { + // The Meta app reads it with getString, which takes a string or an object. + val extra = (info["additional_data"] as? kotlinx.serialization.json.JsonObject) + ?: kotlinx.serialization.json.Json.parseToJsonElement(info.textOf("additional_data")).jsonObject + com.macedonga.visor.core.identity.rawPublicKey(java.util.Base64.getDecoder().decode(extra.textOf("device_ec_public_key"))) + }.getOrNull(), ) } From 919777895c25549aac700fbc407bc7f75e84c56c Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Mon, 19 Oct 2026 21:02:40 +0200 Subject: [PATCH 05/90] Status: the Meta Ray-Ban Display sends its device state as HNDeviceState, the common state under field 1, which read as battery 0% and no case; it is unwrapped for the Display, and state that comes before svc100 names the model waits for it --- .../visor/core/connection/GlassesConnection.kt | 18 ++++++++++++++---- .../macedonga/visor/core/connection/Status.kt | 14 +++++++++++--- .../macedonga/visor/core/media/LiveStream.kt | 3 ++- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt index 0e4d0073..ff6b1eea 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt @@ -87,6 +87,9 @@ class GlassesConnection( private val judged = AtomicBoolean(false) private var deadline: Job? = null + // svc28 state that came before svc100 said which model this is. + private val heldStatus = mutableListOf() + /** Choose a target, open a session on it, and keep the state current until [stop]. */ fun start(scope: CoroutineScope, override: GlassesTarget? = null) { lastScope = scope @@ -194,13 +197,13 @@ class GlassesConnection( ) { canaryAnswered = true decodeInto(event) { copy(info = decodePeerDeviceInfo(it.payload)) } + heldStatus.forEach(::applyStatus) + heldStatus.clear() judge(host, bundle) } if (event.serviceId == SVC_STATUS && isDeviceStateType(event.messageType)) { - decodeInto(event) { - val update = decodeDeviceStateUpdate(it.messageType, it.payload) - copy(device = mergeDeviceState(device ?: DeviceState(), update.state)) - } + // How it is framed depends on the model, which svc100 names: what comes first waits for it. + if (_state.value.info == null && heldStatus.size < HELD_STATUS) heldStatus += event else applyStatus(event) } // Airship's file chunks and the live stream's frames are their responders' business, // and thousands of them would push everything else out of this buffer. @@ -213,6 +216,11 @@ class GlassesConnection( } } + private fun applyStatus(event: ServiceData) = decodeInto(event) { + val update = decodeDeviceStateUpdate(it.messageType, it.payload, display = isDisplayBuild(info?.build)) + copy(device = mergeDeviceState(device ?: DeviceState(), update.state)) + } + /** Fold a decoded service message into the state, or report why it would not decode. */ private fun decodeInto(event: ServiceData, apply: GlassesState.(ServiceData) -> GlassesState) { try { @@ -223,6 +231,7 @@ class GlassesConnection( } private fun newAttempt() { + heldStatus.clear() canaryAnswered = false judged.set(false) deadline?.cancel() @@ -285,5 +294,6 @@ class GlassesConnection( private const val SVC_AIRSHIP = 20 private const val SVC_MEDIA_STREAMING_CONTROL = 51350 + private const val HELD_STATUS = 64 } } diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt index 9ad77803..62db46a1 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt @@ -288,9 +288,17 @@ fun decodeDeviceState(payload: ByteArray): DeviceStatus { ) } -/** Decode any svc28 DeviceState message, full or partial, keeping only what it carries. */ -fun decodeDeviceStateUpdate(messageType: Int, payload: ByteArray): DeviceStateUpdate = - DeviceStateUpdate(messageType, DEVICE_STATE_TYPES.getValue(messageType), decodeDeviceStateFull(payload)) +/** + * Decode any svc28 DeviceState message, full or partial, keeping only what it carries. The Display sends it wrapped, + * HNDeviceState {1: stateCommon, 2: deviceLockState}, and Meta's app unwraps it by device type: [display]. + */ +fun decodeDeviceStateUpdate(messageType: Int, payload: ByteArray, display: Boolean = false): DeviceStateUpdate = + DeviceStateUpdate(messageType, DEVICE_STATE_TYPES.getValue(messageType), decodeDeviceStateFull(if (display) Wire.parse(payload).bytes(1) else payload)) + +/** Whether a build fingerprint, "Meta/greatwhite/greatwhite:14/...", is a Meta Ray-Ban Display's. */ +fun isDisplayBuild(build: String?): Boolean = build?.split('/')?.getOrNull(2)?.substringBefore(':')?.lowercase() in DISPLAY_DEVICES + +private val DISPLAY_DEVICES = setOf("greatwhite") private fun text(fields: Wire.Fields, number: Int): String = String(fields.bytes(number), Charsets.UTF_8) diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt index c6411a06..9fb1e4b9 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt @@ -13,6 +13,7 @@ import com.macedonga.visor.core.connection.SessionHostFactory import com.macedonga.visor.core.connection.subscribeToDeviceStatus import com.macedonga.visor.core.connection.DeviceState import com.macedonga.visor.core.connection.decodeDeviceStateUpdate +import com.macedonga.visor.core.connection.isDisplayBuild import com.macedonga.visor.core.connection.isDeviceStateType import com.macedonga.visor.core.connection.mergeDeviceState import com.macedonga.visor.core.identity.IdentityStore @@ -794,7 +795,7 @@ class LiveStream( event.serviceId == 28 && event.messageType == MT_START_APP_RESPONSE -> startAnswer?.complete(Wire.parse(event.payload).varint(1)) event.serviceId == 28 && isDeviceStateType(event.messageType) -> runCatching { - val update = decodeDeviceStateUpdate(event.messageType, event.payload).state + val update = decodeDeviceStateUpdate(event.messageType, event.payload, display = isDisplayBuild(connection.state.value.info?.build)).state _device.update { it?.let { base -> mergeDeviceState(base, update) } ?: update } } // Airship, whose receiver reports for itself; everything else goes in the log. From 752d65a0d6e6ed80a25832f5a21c7d5b1b986b6c Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Mon, 19 Oct 2026 21:02:40 +0200 Subject: [PATCH 06/90] Settings: the Display's rows as the Meta app's registry gives them by device type (resolution, video length, EIS and capture audio shown, aspect ratio and landscape hidden); conversation focus, hearing correction and group conversations are left out on the Display, whose firmware ships none of their code --- .../com/macedonga/visor/features/Catalog.kt | 13 +++++++++++++ .../com/macedonga/visor/features/Features.kt | 3 +++ .../macedonga/visor/features/FeaturesScreen.kt | 4 +++- .../visor/core/settings/settings_ui.json | 18 ++++++++---------- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/features/Catalog.kt b/app/src/main/kotlin/com/macedonga/visor/features/Catalog.kt index a12415a1..af51d6fe 100644 --- a/app/src/main/kotlin/com/macedonga/visor/features/Catalog.kt +++ b/app/src/main/kotlin/com/macedonga/visor/features/Catalog.kt @@ -37,6 +37,10 @@ sealed interface Gate { data class Setting(val key: String, val on: String, val off: String? = null) : Gate } +// Meta Ray-Ban Display, whose image has no hearingservice. +private const val DISPLAY = 259 +private val NO_HEARING_SERVICE = setOf(DISPLAY) + // The hearing service starts focus only above the minimum build MobileConfig gives this model (libhearing's // getDeviceTypeMinSoCVersion); -1 or none, as on glasses Meta never configured, is INVALID_DEVICE. private val FOCUS_FLOORS = @@ -64,6 +68,8 @@ enum class Feature( val topic: Topic, /** Its own page, for a feature with more to set than on and off. */ val page: Boolean = false, + /** Products whose image ships none of its code. */ + val absentOn: Set = emptySet(), ) { // getHasCFAccess reads cf_enabled unless conversation_focus_use_eap_settings is on; without Meta's value it is // false, NO_ACCESS, and the glasses ask for early access. @@ -72,6 +78,7 @@ enum class Feature( listOf(Gate.Setting("hearing_boost_nux_completed", "true"), Gate.Config("hearing_on_sn.cf_enabled", true, live = true)) + FOCUS_FLOORS, Topic.HEARING, page = true, + absentOn = NO_HEARING_SERVICE, ), // setupForHCModes refuses "HC is not supported on this device" (INVALID_DEVICE) while the first is off. The second // is the version pass Meta's server sets beside it ("update your device"). Both apply at once on the glasses. @@ -83,6 +90,7 @@ enum class Feature( ), Topic.HEARING, page = true, + absentOn = NO_HEARING_SERVICE, ), // The group app's MainActivity finishes unless the first two are on; enable_aurachat starts aurachat through // sysprop_map. Without Meta's signed key material the group key is AurachatKeyManager's all-zero DEBUG_KEY. @@ -94,6 +102,8 @@ enum class Feature( Gate.Config("wearable_connectivity_device_features.enable_aurachat", true), ), Topic.HEARING, + // The Display ships neither aurachat nor the group app. + absentOn = setOf(DISPLAY), ), // The glasses' TAR gate (isSgAlarmMCEnabled); Visor's timer service keeps the alarms Meta's app refuses. ALARMS( @@ -122,6 +132,9 @@ enum class Feature( /** The gates for glasses whose build names [device]. */ fun gatesFor(device: String?): List = gates.filter { it !is Gate.Config || it.device == null || it.device == device } + /** Whether [product] has it at all; unknown glasses are given the benefit. */ + fun on(product: Int?): Boolean = product == null || product !in absentOn + companion object { fun byId(id: String): Feature? = entries.firstOrNull { it.id == id } } diff --git a/app/src/main/kotlin/com/macedonga/visor/features/Features.kt b/app/src/main/kotlin/com/macedonga/visor/features/Features.kt index a2809777..5de730bc 100644 --- a/app/src/main/kotlin/com/macedonga/visor/features/Features.kt +++ b/app/src/main/kotlin/com/macedonga/visor/features/Features.kt @@ -65,6 +65,8 @@ data class FeaturesState( /** Turned on with switches the glasses read when they start: waiting for a restart. */ val restart: Set = emptySet(), val problem: String? = null, + /** The Meta product id of the pair, as [Feature.on] reads it. */ + val product: Int? = null, ) /** @@ -122,6 +124,7 @@ class Features @Inject constructor( readAt = c.reportedAt ?: c.readAt, restart = m.restart, problem = m.problem, + product = s.product, ) }.collect { _state.value = it } } diff --git a/app/src/main/kotlin/com/macedonga/visor/features/FeaturesScreen.kt b/app/src/main/kotlin/com/macedonga/visor/features/FeaturesScreen.kt index 8afbe9a9..21639ec1 100644 --- a/app/src/main/kotlin/com/macedonga/visor/features/FeaturesScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/features/FeaturesScreen.kt @@ -84,8 +84,10 @@ fun LazyListScope.featureSection(topic: Topic, open: (NavKey) -> Unit, heading: } @Composable -private fun FeatureSection(features: List, open: (NavKey) -> Unit, heading: Boolean, model: FeaturesModel = hiltViewModel()) { +private fun FeatureSection(all: List, open: (NavKey) -> Unit, heading: Boolean, model: FeaturesModel = hiltViewModel()) { val state by model.features.state.collectAsStateWithLifecycle() + val features = all.filter { it.on(state.product) } + if (features.isEmpty()) return val host = LocalHost.current var details by remember { mutableStateOf(null) } // Meta's switches come in a short report: read once the glasses are there. diff --git a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json index c1c2da01..ae88c9b0 100644 --- a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json +++ b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json @@ -1064,6 +1064,7 @@ "label_it":"Ascolta mentre registri", "note_it":"La riproduzione dell'audio continuerà mentre registri video", "products":[ +259, 262, 263, 264 @@ -4825,7 +4826,8 @@ }, "label_it":"Orientamento dei contenuti multimediali", "not_products":[ -258 +258, +259 ], "note_it":"Foto e video in verticale (3:4) o in orizzontale (4:3)." }, @@ -5835,6 +5837,7 @@ }, "label_it":"Stabilizzazione video", "products":[ +259, 262, 263, 264, @@ -10798,7 +10801,8 @@ "label_it":"Modalità orizzontale", "note_it":"Acquisisci foto e video in modalità orizzontale.", "not_products":[ -258 +258, +259 ] }, { @@ -13328,10 +13332,7 @@ ] }, "label_it":"Risoluzione e frequenza dei fotogrammi", -"note_it":"La modifica della risoluzione o della frequenza dei fotogrammi influisce su autonomia della batteria, spazio, durata di registrazione e orientamento.", -"not_products":[ -259 -] +"note_it":"La modifica della risoluzione o della frequenza dei fotogrammi influisce su autonomia della batteria, spazio, durata di registrazione e orientamento." }, { "key":"sample_app_1_enabled", @@ -14996,10 +14997,7 @@ ] }, "label_it":"Durata del video", -"note_it":"Scegli una durata massima del video. Questi limiti aiutano a ottimizzare la condivisione e lo spazio.", -"not_products":[ -259 -] +"note_it":"Scegli una durata massima del video. Questi limiti aiutano a ottimizzare la condivisione e lo spazio." }, { "key":"video_fps", From 183b3f95cb6304f3371e84d1163d94c919fb30a8 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Mon, 19 Oct 2026 21:02:40 +0200 Subject: [PATCH 07/90] Neural Band: set up from the Display's settings with change owner over its own PSM and a key of its own, then its battery on the home and in Settings and its wrist set from there; Visor answers the Display's lease on the band (svc150) as the Meta app does and steps back from it while the Display holds it; setup no longer offers the band as glasses, and the Display's settings show again (they looked for hypernova in a fingerprint that says greatwhite) --- .../com/macedonga/visor/band/BandLink.kt | 192 ++++++++++++++++++ .../com/macedonga/visor/band/BandSetup.kt | 138 +++++++++++++ .../kotlin/com/macedonga/visor/band/Bands.kt | 78 +++++++ .../com/macedonga/visor/band/DeviceLease.kt | 47 +++++ .../visor/connection/GlassesService.kt | 4 + .../macedonga/visor/di/ConnectionModule.kt | 3 + .../com/macedonga/visor/home/HomeFeed.kt | 8 + .../com/macedonga/visor/home/HomeScreen.kt | 6 + .../com/macedonga/visor/settings/BandPart.kt | 61 ++++++ .../visor/settings/SettingsScreen.kt | 15 +- .../macedonga/visor/settings/TopicScreen.kt | 1 + .../com/macedonga/visor/settings/Topics.kt | 22 +- .../macedonga/visor/setup/NearbyGlasses.kt | 6 +- .../kotlin/com/macedonga/visor/ui/Icons.kt | 3 + app/src/main/res/values-it/strings.xml | 15 ++ app/src/main/res/values/strings.xml | 15 ++ .../com/macedonga/visor/core/protocol/Band.kt | 68 +++++++ .../visor/core/protocol/Constants.kt | 5 +- 18 files changed, 676 insertions(+), 11 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/band/BandSetup.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/band/Bands.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/band/DeviceLease.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt create mode 100644 core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt new file mode 100644 index 00000000..b5e3f9df --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt @@ -0,0 +1,192 @@ +package com.macedonga.visor.band + +import android.content.Context +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.BundleCredentials +import com.macedonga.visor.core.identity.BundleState +import com.macedonga.visor.core.identity.IdentityBundle +import com.macedonga.visor.core.identity.IdentityStore +import com.macedonga.visor.core.identity.IdentityTrustExchange +import com.macedonga.visor.core.identity.nowIso +import com.macedonga.visor.core.protocol.Band +import com.macedonga.visor.core.protocol.ServiceId +import com.macedonga.visor.core.session.GlassesSession +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.LinkStateChanged +import com.macedonga.visor.core.session.ProtocolError +import com.macedonga.visor.core.session.ServiceData +import com.macedonga.visor.core.session.SessionCommand +import com.macedonga.visor.core.transport.BluetoothLeTransport +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import java.util.concurrent.atomic.AtomicInteger +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +/** The band as last heard from. */ +data class BandState( + val serial: String? = null, + val link: LinkState = LinkState.DISCONNECTED, + val battery: Int? = null, + val charging: Boolean? = null, + val leftHanded: Boolean? = null, + val at: Long? = null, + // The Display holds the band; the phone steps back, as Meta's app does. + val leased: Boolean = false, +) + +/** + * The phone's own link to the Neural Band: up while Visor runs for the band's Display, down while the Display holds a + * lease on it (svc150). It reads the battery and the wrist over svc52822 and sets the wrist. + */ +@Singleton +class BandLink @Inject constructor( + @ApplicationContext private val context: Context, + private val connection: GlassesConnection, + private val bands: Bands, + private val store: IdentityStore, + private val log: EventLog, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val gate = Mutex() + private val seq = AtomicInteger(1) + private val leased = MutableStateFlow(false) + private val held = MutableStateFlow(true) + + @Volatile private var session: GlassesSession? = null + private var job: Job? = null + + private val _state = MutableStateFlow(BandState()) + val state: StateFlow = _state.asStateFlow() + + init { + scope.launch { + // A lease lives on the Display's link: it ends with it. + connection.state.collect { if (it.link != LinkState.READY && leased.value) leased.value = false } + } + scope.launch { + combine(connection.running, connection.pair, bands.all, leased, held) { running, pair, _, lease, hold -> + bands.forGlasses(pair)?.takeIf { running && hold && !lease } + }.distinctUntilChanged().collect { band -> gate.withLock { switchTo(band) } } + } + scope.launch { leased.collect { lease -> _state.update { it.copy(leased = lease) } } } + } + + /** The Display's lease on the band, from svc150: a non-relay one sends the phone's link away until it ends. */ + fun leased(on: Boolean) { + leased.value = on + } + + /** Let go of the band for a while, so another session can have it: setting it up again. */ + suspend fun release() { + held.value = false + gate.withLock { switchTo(null) } + } + + fun hold() { + held.value = true + } + + /** Put the band on [left] or the right wrist; read back at once. */ + fun setLeftHanded(left: Boolean): Boolean { + val running = session ?: return false + if (_state.value.link != LinkState.READY) return false + running.submit( + SessionCommand { s -> + val mux = s.connection ?: return@SessionCommand + mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.configRequest(seq.getAndIncrement(), left)) + mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.configRequest(seq.getAndIncrement())) + }, + ) + return true + } + + private suspend fun switchTo(band: BandRecord?) { + if (band != null && _state.value.serial == band.serial && session != null) return + session?.let { + log.add("Band: link down") + it.stop() + } + job?.cancel() + session = null + job = null + if (band == null) { + _state.update { it.copy(link = LinkState.DISCONNECTED) } + return + } + val key = band.key ?: store.ownerKeyPem() ?: return + if (_state.value.serial != band.serial) _state.value = BandState(serial = band.serial, leased = leased.value) + val identity = IdentityBundle(key, band.address, band.serial, nowIso(), "band", BundleState.VERIFIED) + val opened = GlassesSession( + transport = BluetoothLeTransport(context, band.address, defaultPsm = Band.PSM), + trust = IdentityTrustExchange(BundleCredentials(identity)), + ) + session = opened + job = scope.launch { + launch { poll(opened) } + opened.events.collect { event -> + when (event) { + is LinkStateChanged -> { + _state.update { it.copy(link = event.state) } + if (event.state == LinkState.READY) { + log.add("Band: link up") + ask(opened) + } + } + is ServiceData -> if (event.serviceId == ServiceId.WIRELESS_INPUT.value) onBand(event) + is ProtocolError -> log.add("Band: ${event.detail}") + else -> Unit + } + } + } + log.add("Band: connecting to ${band.serial}") + opened.start(scope) + } + + // The band pushes battery changes too; a slow read keeps the reading fresh when it does not. + private suspend fun poll(opened: GlassesSession) { + while (true) { + delay(POLL_MS) + if (_state.value.link == LinkState.READY && session === opened) ask(opened) + } + } + + private fun ask(opened: GlassesSession) = opened.submit( + SessionCommand { s -> + val mux = s.connection ?: return@SessionCommand + mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.batteryRequest(seq.getAndIncrement())) + mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.configRequest(seq.getAndIncrement())) + }, + ) + + private fun onBand(event: ServiceData) { + val battery = runCatching { + when (event.messageType and 0xFF) { + Band.RESPONSE -> Band.response(event.payload).also { r -> + if (r.leftHanded != null) _state.update { it.copy(leftHanded = r.leftHanded) } + }.battery + Band.STREAM_UPDATE -> Band.streamBattery(event.payload) + else -> null + } + }.onFailure { log.add("Band: unreadable message type ${event.messageType}: ${it.message}") }.getOrNull() ?: return + _state.update { it.copy(battery = battery.percent, charging = battery.charging, at = System.currentTimeMillis()) } + } + + private companion object { + const val POLL_MS = 5 * 60_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandSetup.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandSetup.kt new file mode 100644 index 00000000..c0599bc0 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandSetup.kt @@ -0,0 +1,138 @@ +package com.macedonga.visor.band + +import android.annotation.SuppressLint +import android.bluetooth.BluetoothDevice +import android.bluetooth.BluetoothGatt +import android.bluetooth.BluetoothGattCallback +import android.bluetooth.BluetoothManager +import android.bluetooth.BluetoothProfile +import android.content.Context +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.IdentityStore +import com.macedonga.visor.core.identity.provision.WrongOwnerError +import com.macedonga.visor.core.identity.devicePublicKey +import com.macedonga.visor.core.identity.provision.ProvisionStep +import com.macedonga.visor.core.identity.provision.provision +import com.macedonga.visor.core.protocol.Band +import com.macedonga.visor.core.transport.BluetoothLeTransport +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.login.MetaLogin +import com.macedonga.visor.setup.words +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull + +/** Change owner on the Neural Band, as on the glasses: same svc36 steps, same owner key, over its own PSM. */ +@Singleton +class BandSetup @Inject constructor( + @ApplicationContext private val context: Context, + private val connection: GlassesConnection, + private val store: IdentityStore, + private val bands: Bands, + private val link: BandLink, + private val login: MetaLogin, + private val log: EventLog, +) { + sealed interface Status { + data object Idle : Status + data class Running(val step: ProvisionStep?) : Status + data class Done(val serial: String) : Status + data class Failed(val reason: Reason, val detail: String = "") : Status + } + + enum class Reason { NO_GLASSES, NOT_PAIRED, NO_ANSWER, WRONG_OWNER, FAILED } + + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val _status = MutableStateFlow(Status.Idle) + val status: StateFlow = _status.asStateFlow() + + fun start() { + if (_status.value is Status.Running) return + _status.value = Status.Running(null) + scope.launch { + val glasses = connection.pair.value + val owned = withContext(Dispatchers.IO) { store.ownerKeyPem() != null } + if (glasses == null || !owned) return@launch fail(Reason.NO_GLASSES) + link.release() + try { + val candidates = bondedBands() + if (candidates.isEmpty()) return@launch fail(Reason.NOT_PAIRED) + // A band paired more than once leaves a bond per pairing; the one that answers is the band. + val address = candidates.firstOrNull { reachable(it) } ?: return@launch fail(Reason.NO_ANSWER) + log.add("Band setup: changing owner on $address") + val result = provision( + transport = BluetoothLeTransport(context, address, defaultPsm = Band.PSM), + // No existing key: the band gets one of its own, as the Meta app keeps one per device. + source = login.source, + ) { step -> + _status.value = Status.Running(step) + log.add("Band setup: ${step.words()}") + } + // The key Meta's final receipt names is the one the Display's lease hints use; the certificate's is a fallback. + val publicKey = result.deviceEcPublicKey ?: result.deviceCert?.let(::devicePublicKey) + bands.save(BandRecord(result.serial, address, publicKey, glasses, key = result.keyPem)) + val from = if (result.deviceEcPublicKey != null) "receipt" else if (publicKey != null) "certificate" else "none" + log.add("Band setup: owner changed. Serial ${result.serial}, device key from $from") + _status.value = Status.Done(result.serial) + } catch (error: Exception) { + val wrong = error is WrongOwnerError + fail(if (wrong) Reason.WRONG_OWNER else Reason.FAILED, error.message ?: error.javaClass.simpleName) + } finally { + link.hold() + } + } + } + + fun reset() { + if (_status.value !is Status.Running) _status.value = Status.Idle + } + + private fun fail(reason: Reason, detail: String = "") { + log.add("Band setup failed: $reason $detail") + _status.value = Status.Failed(reason, detail) + } + + @SuppressLint("MissingPermission") + private fun bondedBands(): List = runCatching { + context.getSystemService(BluetoothManager::class.java)?.adapter?.bondedDevices.orEmpty() + .filter { it.type != BluetoothDevice.DEVICE_TYPE_CLASSIC && it.name?.startsWith(BAND_NAME) == true } + .map { it.address } + }.getOrDefault(emptyList()) + + /** A GATT connection that comes up within a few seconds: this bond is the band's current one. */ + @SuppressLint("MissingPermission") + private suspend fun reachable(address: String): Boolean { + val device = context.getSystemService(BluetoothManager::class.java)?.adapter?.getRemoteDevice(address) ?: return false + val up = CompletableDeferred() + val gatt = device.connectGatt( + context, false, + object : BluetoothGattCallback() { + override fun onConnectionStateChange(g: BluetoothGatt, status: Int, newState: Int) { + up.complete(newState == BluetoothProfile.STATE_CONNECTED) + } + }, + BluetoothDevice.TRANSPORT_LE, + ) ?: return false + return try { + withTimeoutOrNull(PROBE_MS) { up.await() } == true + } finally { + gatt.disconnect() + gatt.close() + } + } + + private companion object { + const val BAND_NAME = "Meta Band" + const val PROBE_MS = 6_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/band/Bands.kt b/app/src/main/kotlin/com/macedonga/visor/band/Bands.kt new file mode 100644 index 00000000..ff6f1fb9 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/band/Bands.kt @@ -0,0 +1,78 @@ +package com.macedonga.visor.band + +import android.content.Context +import android.util.Base64 +import com.macedonga.visor.core.identity.KeystoreSealer +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.File +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import org.json.JSONArray +import org.json.JSONObject + +/** + * A Neural Band this phone owns: where it is, the key its lease hints name, the Display it goes with, and its own owner + * key, as the Meta app keeps one per device. + */ +data class BandRecord(val serial: String, val address: String, val publicKey: ByteArray?, val glasses: String?, val key: ByteArray? = null) { + override fun equals(other: Any?): Boolean = + other is BandRecord && serial == other.serial && address == other.address && glasses == other.glasses && + publicKey.contentEquals(other.publicKey) && key.contentEquals(other.key) + + override fun hashCode(): Int = serial.hashCode() +} + +/** The bands set up here, kept apart from the glasses' identity store so the home never takes one for a pair. */ +@Singleton +class Bands @Inject constructor(@ApplicationContext context: Context) { + private val file = File(context.filesDir, "bands.json") + private val sealer = KeystoreSealer(alias = "visor.band") + private val _all = MutableStateFlow(read()) + val all: StateFlow> = _all.asStateFlow() + + fun forGlasses(serial: String?): BandRecord? = serial?.let { s -> _all.value.firstOrNull { it.glasses == s } } + + @Synchronized + fun save(band: BandRecord) { + // One band per Display, as the Meta app pairs them. + _all.value = _all.value.filter { it.serial != band.serial && (band.glasses == null || it.glasses != band.glasses) } + band + write() + } + + @Synchronized + fun forget(serial: String) { + _all.value = _all.value.filter { it.serial != serial } + write() + } + + private fun read(): List = runCatching { + val list = JSONArray(file.readText()) + (0 until list.length()).map { i -> + val o = list.getJSONObject(i) + BandRecord( + serial = o.getString("serial"), + address = o.getString("address"), + publicKey = o.optString("public_key").takeIf { it.isNotEmpty() }?.let { Base64.decode(it, Base64.NO_WRAP) }, + glasses = o.optString("glasses").takeIf { it.isNotEmpty() }, + key = o.optString("key").takeIf { it.isNotEmpty() }?.let { sealer.unseal(Base64.decode(it, Base64.NO_WRAP)) }, + ) + } + }.getOrDefault(emptyList()) + + private fun write() { + val list = JSONArray() + for (b in _all.value) { + list.put( + JSONObject().put("serial", b.serial).put("address", b.address).put("glasses", b.glasses ?: "") + .put("public_key", b.publicKey?.let { Base64.encodeToString(it, Base64.NO_WRAP) } ?: "") + .put("key", b.key?.let { Base64.encodeToString(sealer.seal(it), Base64.NO_WRAP) } ?: ""), + ) + } + val tmp = File(file.parentFile, "${file.name}.tmp") + tmp.writeText(list.toString()) + tmp.renameTo(file) + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/band/DeviceLease.kt b/app/src/main/kotlin/com/macedonga/visor/band/DeviceLease.kt new file mode 100644 index 00000000..87f931a8 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/band/DeviceLease.kt @@ -0,0 +1,47 @@ +package com.macedonga.visor.band + +import com.macedonga.visor.core.identity.keyHintMatches +import com.macedonga.visor.core.protocol.NotifyLease +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import javax.inject.Inject +import javax.inject.Singleton + +/** + * svc150, conn_device_lease, as Meta's DeviceLeaseService answers it: the Display names its band by key hint, the phone + * says OK when it owns that band and steps back from it until the channel closes, BadRequest when it does not. + */ +@Singleton +class DeviceLease @Inject constructor( + private val bands: Bands, + private val link: BandLink, + private val log: EventLog, +) : Responder { + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val info = message.info + val reply = DataXConnection.mirror(info.channelId) + if (info.flags.isClosed && message.payload.isEmpty()) { + if (link.state.value.leased) log.add("Band lease: the Display let go") + link.leased(false) + return + } + if (info.messageType != NotifyLease.MESSAGE_TYPE) { + connection.sendStatus(reply, NotifyLease.UNKNOWN_TYPE) + return + } + val lease = runCatching { NotifyLease.decode(message.payload) }.getOrNull() + val band = lease?.let { l -> bands.all.value.firstOrNull { b -> b.publicKey?.let { keyHintMatches(l.keyHint, it) } == true } } + if (lease == null || band == null) { + val hint = lease?.keyHint?.joinToString("") { "%02x".format(it) } + log.add("Band lease: no band here for hint $hint (${bands.all.value.size} known)") + connection.sendStatus(reply, NotifyLease.BAD_REQUEST) + return + } + connection.sendStatus(reply, NotifyLease.OK) + log.add("Band lease: the Display holds ${band.serial} (type ${lease.type})") + if (!lease.relay) link.leased(true) + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt index cd0f1e09..ed6893a9 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt @@ -79,6 +79,10 @@ class GlassesService : Service() { @Inject lateinit var outside: com.macedonga.visor.outside.Outside + // Injected so a Display's band is linked while the service runs. + @Inject + lateinit var band: com.macedonga.visor.band.BandLink + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) private var started = false diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index ce251b27..62ab5707 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -49,6 +49,7 @@ import dagger.Module import dagger.Provides import dagger.hilt.InstallIn import dagger.hilt.android.qualifiers.ApplicationContext +import com.macedonga.visor.core.protocol.ServiceId import dagger.hilt.components.SingletonComponent import java.io.File import javax.inject.Singleton @@ -134,6 +135,7 @@ object ConnectionModule { // A Provider, because these need the connection this factory feeds. hearing: javax.inject.Provider, assistant: javax.inject.Provider, + lease: javax.inject.Provider, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -144,6 +146,7 @@ object ConnectionModule { SVC_GLASSES_PHOTO to photos, SVC_TAR to timers, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, + ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), ) diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt index 00cf5632..5c19932f 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt @@ -138,6 +138,9 @@ fun StatusPill( onAllow: () -> Unit, onSetUp: () -> Unit, modifier: Modifier, + // The Display's Neural Band, when Visor has read it. + band: Int? = null, + bandCharging: Boolean = false, ) { when { !allowed -> return Pill(stringResource(R.string.home_allow_bluetooth), onAllow, modifier) @@ -190,6 +193,11 @@ fun StatusPill( Level(VisorIcons.Case, case, critical = false) if (caseCharging) Icon(VisorIcons.Bolt, null, Modifier.padding(start = 2.dp).size(13.dp), tint = colors.textPrimary) } + if (band != null) { + Spacer(Modifier.size(14.dp)) + Level(VisorIcons.Band, band, critical = band <= 10 && !bandCharging) + if (bandCharging) Icon(VisorIcons.Bolt, null, Modifier.padding(start = 2.dp).size(13.dp), tint = colors.textPrimary) + } } } // Over Wi-Fi Direct the battery is not reported: the last reading stays, and the import takes the place of diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt index 45e2795e..27608986 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt @@ -152,6 +152,8 @@ class HomeViewModel @Inject constructor( val looks: Looks, val lastSeen: LastSeen, val pairs: Pairs, + val band: com.macedonga.visor.band.BandLink, + val bands: com.macedonga.visor.band.Bands, ) : ViewModel() { /** The LE link is up: the glasses are right here, whatever RFCOMM says. */ val nearby = keeper.connected @@ -566,9 +568,13 @@ fun HomeScreen( modifier = Modifier, ) } else { + val band by model.band.state.collectAsStateWithLifecycle() + val bands by model.bands.all.collectAsStateWithLifecycle() + val ours = band.takeIf { b -> b.battery != null && bands.any { it.serial == b.serial && it.glasses == current } } StatusPill( state, mood, lastBattery, paired, allowed, transferring, importProgress, waiting = if (importing) 0 else fresh.size, onImport = onImport, onAllow = { ask.launch(wanted()) }, onSetUp = onSetUp, modifier = Modifier, + band = ours?.battery, bandCharging = ours?.charging == true, ) } if (pairs.isNotEmpty()) PairDots(pairs.size + 1, shownIndex, Modifier.padding(top = 12.dp)) diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt new file mode 100644 index 00000000..3bf703ef --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt @@ -0,0 +1,61 @@ +package com.macedonga.visor.settings + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.res.stringResource +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.macedonga.visor.R +import com.macedonga.visor.band.BandSetup +import com.macedonga.visor.core.session.LinkState + +/** The Neural Band that goes with this Display: set it up, or its battery and the wrist it is on. */ +@Composable +internal fun BandPart(model: SettingsViewModel) { + val pair by model.connection.pair.collectAsStateWithLifecycle() + val all by model.bands.all.collectAsStateWithLifecycle() + val band by model.band.state.collectAsStateWithLifecycle() + val setup by model.bandSetup.status.collectAsStateWithLifecycle() + val running = setup is BandSetup.Status.Running + val problem = (setup as? BandSetup.Status.Failed)?.let { failWords(it) } + @Composable + fun SetUpRow(title: Int) = ValueRow( + stringResource(title), + if (running) stringResource(R.string.band_setting_up) else null, + onClick = if (running) null else ({ model.bandSetup.start() }), + busy = running, + problem = problem, + ) + if (all.none { it.glasses == pair }) { + Rows({ SetUpRow(R.string.band_set_up) }) + return + } + Rows( + { ValueRow(stringResource(R.string.band_battery), band.battery?.let { batteryWords(it, band.charging) }, onClick = null) }, + { + PillsRow( + stringResource(R.string.band_wrist), + listOf(true to stringResource(R.string.band_left), false to stringResource(R.string.band_right)), + band.leftHanded, + onPick = { model.band.setLeftHanded(it) }, + locked = band.link != LinkState.READY, + ) + }, + { SetUpRow(R.string.band_set_up_again) }, + ) +} + +@Composable +internal fun batteryWords(percent: Int, charging: Boolean?): String = + stringResource(if (charging == true) R.string.glasses_battery_charging else R.string.band_percent, percent) + +@Composable +internal fun wristWords(left: Boolean): String = stringResource(if (left) R.string.band_left else R.string.band_right) + +@Composable +private fun failWords(failed: BandSetup.Status.Failed): String = when (failed.reason) { + BandSetup.Reason.NO_GLASSES -> stringResource(R.string.band_fail_no_glasses) + BandSetup.Reason.NOT_PAIRED -> stringResource(R.string.band_fail_not_paired) + BandSetup.Reason.NO_ANSWER -> stringResource(R.string.band_fail_no_answer) + BandSetup.Reason.WRONG_OWNER -> stringResource(R.string.band_fail_wrong_owner) + BandSetup.Reason.FAILED -> stringResource(R.string.band_fail, failed.detail) +} diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt index 951015af..bbd64b95 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt @@ -172,6 +172,9 @@ class SettingsViewModel @Inject constructor( val features: com.macedonga.visor.features.Features, val lastSeen: com.macedonga.visor.home.LastSeen, val hearing: com.macedonga.visor.core.assistant.Hearing, + val band: com.macedonga.visor.band.BandLink, + val bands: com.macedonga.visor.band.Bands, + val bandSetup: com.macedonga.visor.band.BandSetup, ) : ViewModel() /** @@ -242,7 +245,9 @@ fun SettingsScreen(open: (NavKey) -> Unit, model: SettingsViewModel = hiltViewMo for ((i, card) in Topic.CARDS.withIndex()) { item(key = "card-$i") { Column(Modifier.padding(top = if (i == 0) 18.dp else 14.dp)) { - for (topic in card) NavRow(topic.icon, stringResource(topic.title), summary(topic, model, state), onClick = { open(placeOf(topic)) }) + for (topic in card.filter { it.shownOn(state.product) }) { + NavRow(topic.icon, stringResource(topic.title), summary(topic, model, state), onClick = { open(placeOf(topic)) }) + } } } } @@ -556,13 +561,19 @@ private fun summary(topic: Topic, model: SettingsViewModel, state: SettingsState shown("capture_mode")?.let { stringResource(R.string.sum_controls_press, it) }, shown("tap_and_hold_mapping")?.let { stringResource(R.string.sum_controls_hold, it) }, ).joinToString(" · ") + Topic.BAND -> { + val band by model.band.state.collectAsStateWithLifecycle() + val pair by model.connection.pair.collectAsStateWithLifecycle() + if (model.bands.forGlasses(pair) == null) stringResource(R.string.band_none) + else listOfNotNull(band.battery?.let { batteryWords(it, band.charging) }, band.leftHanded?.let { wristWords(it) }).joinToString(" · ") + } Topic.SOUND -> listOfNotNull( shown("notification_volume")?.let { stringResource(R.string.sum_sound_volume, it) }, stringResource(R.string.sum_headphone).takeIf { isOn("sg_headphone_mode_enabled") == true }, ).joinToString(" · ") Topic.HEARING -> { val features by model.features.state.collectAsStateWithLifecycle() - Feature.entries.filter { it.topic == Topic.HEARING && features.standings[it] == Standing.ON } + Feature.entries.filter { it.topic == Topic.HEARING && it.on(features.product) && features.standings[it] == Standing.ON } .map { stringResource(it.title) }.joinToString(" · ").ifEmpty { stringResource(R.string.sum_all_off) } } Topic.WEAR -> listOfNotNull( diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/TopicScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/TopicScreen.kt index 2f180eac..8947b1c2 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/TopicScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/TopicScreen.kt @@ -111,6 +111,7 @@ fun TopicScreen(name: String, open: (NavKey) -> Unit, model: SettingsViewModel = ) } Topic.PHOTOS -> item(key = "visor") { PhotosPart(model) } + Topic.BAND -> item(key = "visor") { BandPart(model) } else -> Unit } // Hearing is nothing but features, so they need no heading of their own there. diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt b/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt index 16f785ab..c0a7fa94 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt @@ -2,6 +2,7 @@ package com.macedonga.visor.settings import androidx.compose.ui.graphics.vector.ImageVector import com.macedonga.visor.R +import com.macedonga.visor.core.settings.productId import com.macedonga.visor.features.Feature import com.macedonga.visor.ui.VisorIcons @@ -34,6 +35,8 @@ enum class Topic(val title: Int, val icon: ImageVector, val keys: List = "spotify_tap_enabled", ), ), + // The Display's: the band itself, then what the glasses do with it. + BAND(R.string.topic_band, VisorIcons.Pulse, listOf("sg_emg_display_off", "sg_emg_instant_action", "sg_invert_wrist_roll_direction")), // Spatial audio is the feature row: the setting does nothing while Meta's switch is off. SOUND( R.string.topic_sound, VisorIcons.Speaker, @@ -72,7 +75,7 @@ enum class Topic(val title: Int, val icon: ImageVector, val keys: List = /** The main page's cards. */ val CARDS = listOf( listOf(META_AI, LANGUAGE), - listOf(NOTIFICATIONS, CONTROLS, SOUND, HEARING, WEAR), + listOf(NOTIFICATIONS, CONTROLS, BAND, SOUND, HEARING, WEAR), listOf(CAMERA, PHOTOS, LIVE), listOf(PRIVACY, PHONE), ) @@ -100,10 +103,15 @@ fun tileIcon(id: String): ImageVector = when (id) { /** Groups the firmware keeps for glasses with a display; they matter only to a model that has one. */ val DISPLAY_GROUPS = setOf("Display", "Neural band gestures") -/** Whether the connected glasses have a display: the build fingerprint names the product, "Meta//...". */ -fun hasDisplay(build: String?): Boolean { - val product = build?.split('/')?.getOrNull(1)?.lowercase() ?: return false - return DISPLAY_PRODUCTS.any { it in product } -} +/** Whether the connected glasses have a display: the build fingerprint names the device, "Meta/greatwhite/greatwhite:14/...". */ +fun hasDisplay(build: String?): Boolean = productId(build) in DISPLAY_PRODUCTS + +// Meta Ray-Ban Display. +private val DISPLAY_PRODUCTS = setOf(259) -private val DISPLAY_PRODUCTS = listOf("hypernova", "display") +/** Whether [product] has anything on this topic: the band is the Display's, and hearing is nothing but features. */ +fun Topic.shownOn(product: Int?): Boolean = when (this) { + Topic.BAND -> product in DISPLAY_PRODUCTS + Topic.HEARING -> Feature.entries.any { it.topic == Topic.HEARING && it.on(product) } + else -> true +} diff --git a/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt b/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt index dd4f7838..b90d3661 100644 --- a/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt +++ b/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt @@ -71,6 +71,7 @@ class NearbyGlasses @Inject constructor(@ApplicationContext private val context: if (advert?.product == PRODUCT_BAND) return val device = result.device val name = record.deviceName ?: runCatching { device.name }.getOrNull() ?: productName(advert?.product) + if (BAND_NAME.containsMatchIn(name)) return val pair = NearbyPair(device.address, name, result.rssi, advert?.pairing == true, device.bondState == BluetoothDevice.BOND_BONDED) synchronized(heard) { heard[device.address] = pair to SystemClock.elapsedRealtime() } publish() @@ -98,7 +99,7 @@ class NearbyGlasses @Inject constructor(@ApplicationContext private val context: @SuppressLint("MissingPermission") fun bonded(): List = runCatching { adapter?.bondedDevices.orEmpty() - .filter { d -> d.name?.let { GLASSES_NAME.containsMatchIn(it) } == true } + .filter { d -> d.name?.let { GLASSES_NAME.containsMatchIn(it) && !BAND_NAME.containsMatchIn(it) } == true } .map { NearbyPair(it.address, it.name, null, pairing = false, bonded = true) } }.getOrDefault(emptyList()) @@ -157,5 +158,8 @@ class NearbyGlasses @Inject constructor(@ApplicationContext private val context: const val BOND_MS = 90_000L const val POLL_MS = 400L val GLASSES_NAME = Regex("ray-?ban|meta|oakley|stories", RegexOption.IGNORE_CASE) + + // The Neural Band, "Meta Band 0064": set up from the Display's settings, never as glasses. + val BAND_NAME = Regex("^meta band", RegexOption.IGNORE_CASE) } } diff --git a/app/src/main/kotlin/com/macedonga/visor/ui/Icons.kt b/app/src/main/kotlin/com/macedonga/visor/ui/Icons.kt index d69d188e..467fcb70 100644 --- a/app/src/main/kotlin/com/macedonga/visor/ui/Icons.kt +++ b/app/src/main/kotlin/com/macedonga/visor/ui/Icons.kt @@ -45,6 +45,9 @@ object VisorIcons { val Case = icon("M6 8h12a4 4 0 0 1 0 8H6a4 4 0 0 1 0-8z", "M2.5 12h19") + /** The Neural Band: the strap seen edge on, the pod on top. */ + val Band = icon("M5 12.5a7 4.5 0 1 0 14 0a7 4.5 0 1 0-14 0", "M9.8 6.3h4.4a1 1 0 0 1 1 1v1.2H8.8V7.3a1 1 0 0 1 1-1z") + /** A turn back round the clock: what just happened. */ val Replay = icon("M5.2 13.5A7 7 0 1 0 7 7.1", "M6.6 3.6l.4 3.5 3.5-.4") diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 791502e6..4f112e9b 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -364,6 +364,21 @@ Visione live Privacy Accessi del telefono + Neural Band + Configura + Configura di nuovo + Configurazione… + Non configurato + Batteria + %1$d%% + Polso + Sinistro + Destro + Configura prima gli occhiali + Associa prima il bracciale a questo telefono + Il bracciale non ha risposto + Il bracciale appartiene a un altro account Meta + Configurazione non riuscita: %1$s Cosa arriva agli occhiali, e cosa dicono ad alta voce. Cosa fanno il tasto di cattura e il touchpad sull\'asta destra. Come gli occhiali capiscono di essere indossati, e come risparmiano energia. diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 7cddf8e6..69bb4a1d 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -365,6 +365,21 @@ Live view Privacy Phone access + Neural Band + Set up + Set up again + Setting up… + Not set up + Battery + %1$d%% + Wrist + Left + Right + Set up the glasses first + Pair the band with this phone first + The band did not answer + The band belongs to another Meta account + Setup failed: %1$s What reaches the glasses, and what they say out loud. What the capture button and the touchpad on the right temple do. How the glasses notice they are on, and how they save power. diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt new file mode 100644 index 00000000..c661b5f3 --- /dev/null +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt @@ -0,0 +1,68 @@ +package com.macedonga.visor.core.protocol + +/** + * The Neural Band's RPC on svc52822, as Meta AI 294.1 speaks it (com.oculus.wearableinputservice). A request goes out + * as DataX type 0x0314, whose low byte is the band's REQUEST; what the band sends carries its own type in the low byte. + */ +object Band { + const val REQUEST_TYPE = 0x0314 + const val RESPONSE = 21 + const val STREAM_UPDATE = 22 + + /** The PSM the band's GATT names; the glasses' is 128. */ + const val PSM = 129 + + class Battery(val percent: Int, val charging: Boolean) + + /** RpcResponse {1: seq, 2: code, 3: BatteryInfoResp, 6: ConfigResp, ...}, as far as Visor reads it. */ + class Response(val seq: Int, val code: Int, val battery: Battery?, val leftHanded: Boolean?) + + /** RpcRequest {1: seq, 2: BatteryInfoReq {}}. */ + fun batteryRequest(seq: Int): ByteArray = request(seq, 2, ByteArray(0)) + + /** RpcRequest {1: seq, 5: ConfigReq}: with [leftHanded] it sets the wrist (field 10), empty it reads the config. */ + fun configRequest(seq: Int, leftHanded: Boolean? = null): ByteArray = + request(seq, 5, leftHanded?.let { Wire.fieldVarint(10, if (it) 1 else 0) } ?: ByteArray(0)) + + fun response(payload: ByteArray): Response { + val f = Wire.parse(payload) + val battery = if (f.has(3)) battery(Wire.parse(f.bytes(3)).bytes(1)) else null + val left = if (f.has(6)) Wire.parse(f.bytes(6)).let { c -> if (c.has(10)) c.varint(10) != 0L else null } else null + return Response(f.varint(1).toInt(), f.varint(2).toInt(), battery, left) + } + + /** RpcStreamUpdate field 2, BatteryUpdate {1: BatteryData}. */ + fun streamBattery(payload: ByteArray): Battery? { + val f = Wire.parse(payload) + return if (f.has(2)) battery(Wire.parse(f.bytes(2)).bytes(1)) else null + } + + // BatteryData {1: level, 2: charging, 3: temperature (float), 4: mV, 5: mA, 6: RTC seconds}. + private fun battery(data: ByteArray): Battery? { + val d = Wire.parse(data) + return if (d.has(1)) Battery(d.varint(1).toInt(), d.varint(2) != 0L) else null + } + + private fun request(seq: Int, field: Int, body: ByteArray) = Wire.fieldVarint(1, seq.toLong()) + Wire.fieldBytes(field, body) +} + +/** svc150's NotifyLease {1: peerKeyHint, 2: leaseType}: the Display taking its band. Type 1 on the channel it opens. */ +class NotifyLease(val keyHint: ByteArray, val type: Int) { + /** A relay lease leaves the band's own phone link up; the others ask the phone to step back. */ + val relay: Boolean get() = type == RELAY + + companion object { + const val MESSAGE_TYPE = 1 + const val RELAY = 1 + + // The phone's answers: Meta's DeviceLeaseService sends Error 0 ("OK") once the lease is held. + const val OK = 0x0000 + const val UNKNOWN_TYPE = 0xD001 + const val BAD_REQUEST = 0xD010 + + fun decode(payload: ByteArray): NotifyLease { + val f = Wire.parse(payload) + return NotifyLease(f.bytes(1), f.varint(2).toInt()) + } + } +} diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Constants.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Constants.kt index 3839a172..572fdc0a 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Constants.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Constants.kt @@ -140,7 +140,10 @@ enum class ServiceId(val value: Int) { AIRSHIELD_AUTH(54), CONSTELLATION_AUTH(79), - /** `wireless_input_service` in the firmware's registry. */ + /** `conn_device_lease`: the Display asks the phone for its band here. */ + CONN_DEVICE_LEASE(150), + + /** `wireless_input_service` in the firmware's registry; the Neural Band's whole RPC set. */ WIRELESS_INPUT(52822), ; From 885e049cc84cec039d81a4452877332d01e42929 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 09:30:16 +0200 Subject: [PATCH 08/90] Home: glasses that answer and turn this phone's key down (the Display sends its own credential, refuses ours and never ends link setup, so svc100 is never asked) count as a refused key, and the home offers Set up again instead of calling them asleep or out of reach --- app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt | 3 +++ .../main/kotlin/com/macedonga/visor/home/HomeScreen.kt | 8 +++++++- app/src/main/res/values-it/strings.xml | 1 + app/src/main/res/values/strings.xml | 1 + .../macedonga/visor/core/connection/GlassesConnection.kt | 4 +++- .../macedonga/visor/core/connection/GlassesSessionHost.kt | 2 ++ .../com/macedonga/visor/core/connection/SessionHost.kt | 3 +++ 7 files changed, 20 insertions(+), 2 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt index 5c19932f..8b92cbf5 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt @@ -141,10 +141,13 @@ fun StatusPill( // The Display's Neural Band, when Visor has read it. band: Int? = null, bandCharging: Boolean = false, + // The glasses answer and refuse this phone's key: another phone has set them up since. + refused: Boolean = false, ) { when { !allowed -> return Pill(stringResource(R.string.home_allow_bluetooth), onAllow, modifier) paired == false -> return Pill(stringResource(R.string.home_set_up), onSetUp, modifier) + refused -> return Pill(stringResource(R.string.home_set_up_again), onSetUp, modifier) } val colors = Visor.colors val away = mood == Mood.FAR || mood == Mood.ABSENT || mood == Mood.SEARCHING || mood == Mood.ASLEEP || mood == Mood.IN_CASE diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt index 27608986..b9f9f32f 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt @@ -356,6 +356,12 @@ fun HomeScreen( if (state.link == LinkState.READY) unanswered = false else if (state.failure != null || state.link == LinkState.LOST) unanswered = true } + // Every attempt clears the verdict, so a refused key is kept until the glasses are back. + var refused by remember { mutableStateOf(false) } + LaunchedEffect(state.link, state.diagnosis) { + if (state.link == LinkState.READY) refused = false + else if (state.diagnosis == com.macedonga.visor.core.identity.Diagnosis.EVICTED) refused = true + } // The Live view pauses Bluetooth too; until it is back the glasses are as present as they were on Wi-Fi. val backFromLive = remember { stage.fromLive.also { stage.fromLive = false } } var liveSettling by remember { mutableStateOf(backFromLive) } @@ -574,7 +580,7 @@ fun HomeScreen( StatusPill( state, mood, lastBattery, paired, allowed, transferring, importProgress, waiting = if (importing) 0 else fresh.size, onImport = onImport, onAllow = { ask.launch(wanted()) }, onSetUp = onSetUp, modifier = Modifier, - band = ours?.battery, bandCharging = ours?.charging == true, + band = ours?.battery, bandCharging = ours?.charging == true, refused = refused, ) } if (pairs.isNotEmpty()) PairDots(pairs.size + 1, shownIndex, Modifier.padding(top = 12.dp)) diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 4f112e9b..fbb59aee 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -924,4 +924,5 @@ Riprova Indicazioni per %1$s Riproduci %1$s + Configura di nuovo diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 69bb4a1d..38ded91f 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -926,4 +926,5 @@ Try again Directions to %1$s Play %1$s + Set up again diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt index ff6b1eea..9e54fbce 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt @@ -267,7 +267,9 @@ class GlassesConnection( deadline = null val trustOk = host.trustComplete - val answered = canaryAnswered + // A refusal is an answer too: the Display sends its own credential, turns ours down and never ends link + // setup, so svc100 is never asked. + val answered = canaryAnswered || host.trustRefused if (bundle == null) { // Bring-up against a peer with no stored identity. There is a verdict to publish // and no bundle whose state could move. diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesSessionHost.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesSessionHost.kt index 84ce8101..4eadbf51 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesSessionHost.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesSessionHost.kt @@ -20,6 +20,8 @@ class GlassesSessionHost(private val session: GlassesSession) : SessionHost { override val trustComplete: Boolean get() = session.trustResult?.complete == true + override val trustRefused: Boolean get() = session.trustResult?.let { it.rxOk && !it.txOk } == true + override fun start(scope: CoroutineScope) = session.start(scope) override suspend fun stop() = session.stop() diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/SessionHost.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/SessionHost.kt index 24e254bb..63424fd5 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/SessionHost.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/SessionHost.kt @@ -15,6 +15,9 @@ interface SessionHost { /** True once the svc36 exchange has completed in both directions on this attempt. */ val trustComplete: Boolean + /** The glasses sent their own credential and turned this phone's down: they are there, and refuse the key. */ + val trustRefused: Boolean get() = false + fun start(scope: CoroutineScope) /** Finish, and do not return until the socket has been let go of. */ From be49d073e81b9f8d7fa4bffffd8ae67e0e7d2da1 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 09:30:16 +0200 Subject: [PATCH 09/90] Display: the teleprompter, as a card on the home: the script in the glasses' own square at the size they draw it, size, speed (as a multiple of the glasses' default) and scroll or pages, then Show on glasses, Update and Close; started as the Meta app starts it, svc28 StartApp 2000 for app 35 with the text and its look as intent data, closed with StopApp 2002 --- .../macedonga/visor/debug/DebugReceiver.kt | 13 + .../kotlin/com/macedonga/visor/VisorNav.kt | 9 + .../macedonga/visor/di/ConnectionModule.kt | 5 + .../macedonga/visor/display/DisplayShelf.kt | 81 +++++ .../macedonga/visor/display/DisplaySquare.kt | 41 +++ .../macedonga/visor/display/PrompterScreen.kt | 334 ++++++++++++++++++ .../com/macedonga/visor/home/HomeScreen.kt | 6 + app/src/main/res/values-it/strings.xml | 26 ++ app/src/main/res/values/strings.xml | 26 ++ .../visor/core/connection/Teleprompter.kt | 124 +++++++ .../com/macedonga/visor/core/protocol/Wire.kt | 6 + 11 files changed, 671 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/display/DisplaySquare.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/display/PrompterScreen.kt create mode 100644 core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Teleprompter.kt diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index b8646657..d698075a 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -58,6 +58,7 @@ class DebugReceiver : BroadcastReceiver() { @Inject lateinit var pairs: com.macedonga.visor.pairs.Pairs @Inject lateinit var looks: com.macedonga.visor.stage.Looks @Inject lateinit var phone: com.macedonga.visor.phone.PhoneActions + @Inject lateinit var prompter: com.macedonga.visor.core.connection.Teleprompter override fun onReceive(context: Context, intent: Intent) { val action = intent.action?.substringAfterLast('.').orEmpty() @@ -304,6 +305,18 @@ class DebugReceiver : BroadcastReceiver() { resultData = "asked for $id" } } + "PROMPTER" -> answerLater { + if (intent.getBooleanExtra("close", false)) prompter.close() + else prompter.show( + intent.getStringExtra("text") ?: "Testing the teleprompter from Visor.", + com.macedonga.visor.core.connection.PrompterLook(paged = intent.getBooleanExtra("paged", false), followVoice = intent.getBooleanExtra("voice", false)), + ) + kotlinx.coroutines.delay(300) + withTimeoutOrNull(8_000) { + prompter.state.first { it.phase != com.macedonga.visor.core.connection.PrompterPhase.STARTING && it.phase != com.macedonga.visor.core.connection.PrompterPhase.CLOSING } + } + prompter.state.value.let { "${it.phase} refusal=${it.refusal}" } + } "PHONE" -> answerLater { when (intent.getStringExtra("do")) { "directions" -> phone.directions(intent.getStringExtra("to").orEmpty(), intent.getStringExtra("mode").orEmpty()).toString() diff --git a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt index 4a4782e3..419da986 100644 --- a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt +++ b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt @@ -147,6 +147,9 @@ data object PhoneAccessKey : NavKey @Serializable data object NotifyAppsKey : NavKey +@Serializable +data object PrompterKey : NavKey + /** The app's graph: the home, and every screen it opens. */ @Composable fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) { @@ -201,6 +204,11 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) }, onMetaAi = { backStack.add(MetaAiKey) }, onSettings = { backStack.add(SettingsKey) }, + onDisplayTool = { tool -> + when (tool) { + com.macedonga.visor.display.DisplayTool.TELEPROMPTER -> backStack.add(PrompterKey) + } + }, ) } entry { @@ -253,6 +261,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) entry { AboutScreen() } entry { PhoneAccessScreen() } entry { NotifyAppsScreen() } + entry { com.macedonga.visor.display.PrompterScreen() } entry { MetaAiScreen( onSettings = { backStack.add(AssistantSettingsKey) }, diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 62ab5707..bdd41383 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -158,6 +158,11 @@ object ConnectionModule { @Singleton fun hearing(connection: GlassesConnection): Hearing = Hearing(connection) + @Provides + @Singleton + fun teleprompter(connection: GlassesConnection): com.macedonga.visor.core.connection.Teleprompter = + com.macedonga.visor.core.connection.Teleprompter(connection) + @Provides @Singleton fun assistantConfig(@ApplicationContext context: Context): AssistantConfigStore = AssistantConfigStore(context) diff --git a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt new file mode 100644 index 00000000..81036572 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt @@ -0,0 +1,81 @@ +package com.macedonga.visor.display + +import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.lazy.LazyRow +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.res.pluralStringResource +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.macedonga.visor.R +import com.macedonga.visor.core.connection.PrompterPhase +import com.macedonga.visor.core.connection.Teleprompter +import com.macedonga.visor.core.design.Visor +import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject + +/** What the Display's cards on the home open. */ +enum class DisplayTool { TELEPROMPTER } + +@HiltViewModel +class ShelfModel @Inject constructor(val script: PrompterScript, val prompter: Teleprompter) : ViewModel() + +/** The Display's tools on the home: a card each, the glasses' square on it showing what the tool puts there. */ +@Composable +fun DisplayShelf(onOpen: (DisplayTool) -> Unit, modifier: Modifier = Modifier, model: ShelfModel = hiltViewModel()) { + val text by model.script.text.collectAsStateWithLifecycle() + val look by model.script.look.collectAsStateWithLifecycle() + val prompter by model.prompter.state.collectAsStateWithLifecycle() + LazyRow(modifier, contentPadding = PaddingValues(horizontal = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp)) { + item(key = "prompter") { + val words = text.split(Regex("\\s+")).count { it.isNotEmpty() } + ToolCard( + stringResource(R.string.prompter_title), + when { + prompter.phase == PrompterPhase.SHOWING -> stringResource(R.string.prompter_on_glasses) + words == 0 -> stringResource(R.string.prompter_card_empty) + else -> pluralStringResource(R.plurals.prompter_words, words, words) + }, + onClick = { onOpen(DisplayTool.TELEPROMPTER) }, + ) { ScriptPreview(text, look, Modifier.fillMaxWidth(), radius = 10.dp) } + } + } +} + +@Composable +private fun ToolCard(title: String, status: String, onClick: () -> Unit, square: @Composable () -> Unit) { + Column( + Modifier + .width(CARD_WIDTH) + .clip(RoundedCornerShape(18.dp)) + .background(Visor.colors.raised) + .clickable(onClick = onClick) + .padding(8.dp), + ) { + square() + Column(Modifier.padding(start = 8.dp, end = 8.dp, top = 12.dp, bottom = 6.dp)) { + Text(title, style = Visor.type.heading, color = Visor.colors.textPrimary, maxLines = 1, overflow = TextOverflow.Ellipsis) + Spacer(Modifier.height(2.dp)) + Text(status, style = Visor.type.label, color = Visor.colors.textSecondary, maxLines = 1, overflow = TextOverflow.Ellipsis) + } + } +} + +private val CARD_WIDTH = 172.dp diff --git a/app/src/main/kotlin/com/macedonga/visor/display/DisplaySquare.kt b/app/src/main/kotlin/com/macedonga/visor/display/DisplaySquare.kt new file mode 100644 index 00000000..bf131001 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/display/DisplaySquare.kt @@ -0,0 +1,41 @@ +package com.macedonga.visor.display + +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.BoxScope +import androidx.compose.foundation.layout.BoxWithConstraints +import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalDensity +import androidx.compose.ui.unit.TextUnit +import androidx.compose.ui.unit.dp +import com.macedonga.visor.core.design.Visor + +/** + * The Display's screen as the phone shows it: a black square at its true proportions, the 600 px an app draws on + * across the square. [content] gets how many phone pixels one of theirs is. + */ +@Composable +fun DisplaySquare(modifier: Modifier = Modifier, radius: androidx.compose.ui.unit.Dp = 14.dp, content: @Composable BoxScope.(px: Float) -> Unit) { + val shape = RoundedCornerShape(radius) + BoxWithConstraints(modifier.aspectRatio(1f).clip(shape).background(Color.Black).border(1.dp, Visor.colors.hairline, shape)) { + val px = with(LocalDensity.current) { maxWidth.toPx() } / DISPLAY_PX + Box(Modifier.fillMaxSize()) { content(px) } + } +} + +/** A size in the glasses' sp (one of their pixels: they draw at density 160), at the square's scale. */ +@Composable +fun glassesSp(size: Double, px: Float): TextUnit = with(LocalDensity.current) { (size.toFloat() * px).toSp() } + +const val DISPLAY_PX = 600f + +/** [size] of the glasses' pixels at the square's scale, as a phone length. */ +@Composable +fun glassesDp(size: Float, px: Float): androidx.compose.ui.unit.Dp = with(LocalDensity.current) { (size * px).toDp() } diff --git a/app/src/main/kotlin/com/macedonga/visor/display/PrompterScreen.kt b/app/src/main/kotlin/com/macedonga/visor/display/PrompterScreen.kt new file mode 100644 index 00000000..8cb50523 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/display/PrompterScreen.kt @@ -0,0 +1,334 @@ +package com.macedonga.visor.display + +import android.content.ClipboardManager +import android.content.Context +import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.WindowInsets +import androidx.compose.foundation.layout.asPaddingValues +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.heightIn +import androidx.compose.foundation.layout.navigationBars +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.wrapContentHeight +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.text.BasicTextField +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.SolidColor +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.pluralStringResource +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.em +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.macedonga.visor.R +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.connection.PrompterLook +import com.macedonga.visor.core.connection.PrompterPhase +import com.macedonga.visor.core.connection.Teleprompter +import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.settings.ActionPill +import com.macedonga.visor.settings.PillsRow +import com.macedonga.visor.settings.Rows +import com.macedonga.visor.settings.SettingsPage +import com.macedonga.visor.settings.Spinner +import com.macedonga.visor.settings.VisorSheet +import com.macedonga.visor.settings.SheetTitle +import com.macedonga.visor.settings.row +import com.macedonga.visor.ui.rememberHaptics +import dagger.hilt.android.lifecycle.HiltViewModel +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow + +/** The one script and how it shows, kept on the phone. */ +@Singleton +class PrompterScript @Inject constructor(@ApplicationContext context: Context) { + private val prefs = context.getSharedPreferences("teleprompter", Context.MODE_PRIVATE) + private val _text = MutableStateFlow(prefs.getString(TEXT, "").orEmpty()) + val text: StateFlow = _text.asStateFlow() + private val _look = MutableStateFlow( + PrompterLook( + sizeSp = prefs.getFloat(SIZE, 35f).toDouble(), + speed = prefs.getFloat(SPEED, 32f).toDouble(), + paged = prefs.getBoolean(PAGED, false), + ), + ) + val look: StateFlow = _look.asStateFlow() + + fun write(text: String) { + _text.value = text + prefs.edit().putString(TEXT, text).apply() + } + + fun look(look: PrompterLook) { + _look.value = look + prefs.edit().putFloat(SIZE, look.sizeSp.toFloat()).putFloat(SPEED, look.speed.toFloat()) + .putBoolean(PAGED, look.paged).apply() + } + + private companion object { + const val TEXT = "text" + const val SIZE = "size" + const val SPEED = "speed" + const val PAGED = "paged" + } +} + +@HiltViewModel +class PrompterModel @Inject constructor( + val script: PrompterScript, + val prompter: Teleprompter, + val connection: GlassesConnection, +) : ViewModel() + +/** + * The teleprompter: the script as the glasses will draw it, in their square; the script itself, how big and how fast, + * and the one button that puts it on the glasses, or updates or closes it there. + */ +@Composable +fun PrompterScreen(model: PrompterModel = hiltViewModel()) { + val text by model.script.text.collectAsStateWithLifecycle() + val look by model.script.look.collectAsStateWithLifecycle() + val state by model.prompter.state.collectAsStateWithLifecycle() + val glasses by model.connection.state.collectAsStateWithLifecycle() + val ready = glasses.link == LinkState.READY + var editing by remember { mutableStateOf(false) } + val haptics = rememberHaptics() + SettingsPage( + stringResource(R.string.prompter_title), + sheet = { + PrompterBar( + ready = ready, + empty = text.isBlank(), + phase = state.phase, + changed = state.shown != text || state.look != look, + refusal = state.refusal, + onShow = { haptics.done(); model.prompter.show(text, look) }, + onClose = { haptics.toggle(); model.prompter.close() }, + ) + if (editing) ScriptSheet(text, model.script::write) { editing = false } + }, + ) { + item(key = "square") { + ScriptPreview(text, look, Modifier.padding(horizontal = 16.dp).padding(bottom = 16.dp).clickable { editing = true }) + } + item(key = "script") { Rows(row { ScriptRow(text) { editing = true } }) } + item(key = "look") { + Spacer(Modifier.height(16.dp)) + Rows( + row { + PillsRow( + stringResource(R.string.prompter_size), SIZES.map { it to sizeName(it) }, SIZES.minByOrNull { kotlin.math.abs(it - look.sizeSp) }, + { model.script.look(look.copy(sizeSp = it)) }, + ) + }, + row { + PillsRow( + stringResource(R.string.prompter_speed), SPEEDS.map { it to "%s×".format(speedName(it)) }, + SPEEDS.minByOrNull { kotlin.math.abs(it - look.speed) }, { model.script.look(look.copy(speed = it)) }, + ) + }, + row { + PillsRow( + stringResource(R.string.prompter_layout), + listOf(false to stringResource(R.string.prompter_scroll), true to stringResource(R.string.prompter_pages)), + look.paged, { model.script.look(look.copy(paged = it)) }, + ) + }, + ) + } + item(key = "foot") { Spacer(Modifier.height(BAR_ROOM)) } + } +} + +/** + * The script in the glasses' square, at the size they will draw it: scrolled, the first line sits in the middle, + * where they centre the line being read; paged, it fills from the top as their first card does. + */ +@Composable +fun ScriptPreview(text: String, look: PrompterLook, modifier: Modifier = Modifier, radius: androidx.compose.ui.unit.Dp = 14.dp) { + DisplaySquare(modifier, radius) { px -> + val inset = Modifier.padding(glassesDp(PROMPTER_INSET, px)) + val style = Visor.type.body.copy( + fontSize = glassesSp(look.sizeSp, px), + lineHeight = 1.3.em, + fontWeight = FontWeight.Medium, + color = Color.White, + ) + if (text.isBlank()) { + Text(stringResource(R.string.prompter_empty), style = style.copy(color = Visor.colors.textTertiary), modifier = Modifier.align(Alignment.Center).then(inset)) + } else if (look.paged) { + Text(text, style = style, overflow = TextOverflow.Clip, modifier = Modifier.fillMaxSize().then(inset)) + } else { + Column(Modifier.fillMaxSize().then(inset)) { + Spacer(Modifier.weight(1f)) + Text(text, style = style, overflow = TextOverflow.Clip, modifier = Modifier.weight(1f).wrapContentHeight(Alignment.Top, unbounded = true)) + } + } + } +} + +@Composable +private fun ScriptRow(text: String, onClick: () -> Unit) { + Column(Modifier.fillMaxWidth().clickable(onClick = onClick).padding(horizontal = 16.dp, vertical = 14.dp)) { + Text(stringResource(R.string.prompter_script), style = Visor.type.body, color = Visor.colors.textPrimary) + Spacer(Modifier.height(4.dp)) + if (text.isBlank()) { + Text(stringResource(R.string.prompter_empty), style = Visor.type.label, color = Visor.colors.textSecondary) + } else { + val words = remember(text) { text.split(Regex("\\s+")).count { it.isNotEmpty() } } + Text(pluralStringResource(R.plurals.prompter_words, words, words), style = Visor.type.label, color = Visor.colors.textSecondary) + } + } +} + +/** At the foot of the page, over a fade: what the glasses are doing with the script, and the button for it. */ +@Composable +private fun PrompterBar( + ready: Boolean, + empty: Boolean, + phase: PrompterPhase, + changed: Boolean, + refusal: Long?, + onShow: () -> Unit, + onClose: () -> Unit, +) { + val colors = Visor.colors + val bottom = WindowInsets.navigationBars.asPaddingValues().calculateBottomPadding() + Box(Modifier.fillMaxSize(), contentAlignment = Alignment.BottomCenter) { + Column( + Modifier + .fillMaxWidth() + .background(Brush.verticalGradient(0f to Color.Transparent, 0.3f to colors.canvas)) + .padding(start = 16.dp, end = 16.dp, top = 28.dp, bottom = bottom + 16.dp), + horizontalAlignment = Alignment.CenterHorizontally, + ) { + val line = when { + !ready -> stringResource(R.string.prompter_not_connected) + refusal != null -> refusalWords(refusal) + phase == PrompterPhase.SHOWING -> stringResource(R.string.prompter_on_glasses) + else -> null + } + if (line != null) { + Text(line, style = Visor.type.label, color = if (refusal != null && ready) colors.signal else colors.textSecondary, modifier = Modifier.padding(bottom = 12.dp)) + } + when { + !ready -> Unit + phase == PrompterPhase.STARTING || phase == PrompterPhase.CLOSING -> Box(Modifier.height(52.dp), contentAlignment = Alignment.Center) { Spinner() } + phase == PrompterPhase.SHOWING -> Row(horizontalArrangement = Arrangement.spacedBy(10.dp)) { + WidePill(stringResource(R.string.prompter_close), onClose, Modifier.weight(1f), quiet = true) + WidePill(stringResource(R.string.prompter_update), onShow, Modifier.weight(1f), enabled = changed && !empty) + } + else -> WidePill(stringResource(R.string.prompter_show), onShow, Modifier.fillMaxWidth(), enabled = !empty) + } + } + } +} + +@Composable +private fun WidePill(text: String, onClick: () -> Unit, modifier: Modifier, quiet: Boolean = false, enabled: Boolean = true) { + Box( + modifier + .height(52.dp) + .clip(RoundedCornerShape(50)) + .background(if (quiet) Visor.colors.control else Color.White.copy(alpha = if (enabled) 1f else 0.25f)) + .clickable(enabled = enabled, onClick = onClick), + contentAlignment = Alignment.Center, + ) { + Text(text, style = Visor.type.body, color = if (quiet) Visor.colors.textPrimary else Color.Black) + } +} + +/** The script, full height, with Paste and Clear; every keystroke is kept. */ +@Composable +private fun ScriptSheet(text: String, onText: (String) -> Unit, onDismiss: () -> Unit) { + val context = LocalContext.current + VisorSheet(onDismiss, full = true) { + SheetTitle(stringResource(R.string.prompter_script)) + Box( + Modifier + .padding(horizontal = 16.dp) + .fillMaxWidth() + .heightIn(min = 320.dp, max = 520.dp) + .clip(RoundedCornerShape(16.dp)) + .background(Visor.colors.raised) + .padding(16.dp), + ) { + if (text.isEmpty()) Text(stringResource(R.string.prompter_empty), style = Visor.type.body, color = Visor.colors.textTertiary) + BasicTextField( + value = text, + onValueChange = onText, + textStyle = Visor.type.body.copy(color = Visor.colors.textPrimary), + cursorBrush = SolidColor(Color.White), + modifier = Modifier.fillMaxSize(), + ) + } + Row(Modifier.fillMaxWidth().padding(horizontal = 24.dp).padding(top = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp), verticalAlignment = Alignment.CenterVertically) { + ActionPill(stringResource(R.string.ai_paste), { pasted(context)?.let { onText(if (text.isBlank()) it else text + "\n" + it) } }, quiet = true) + if (text.isNotEmpty()) ActionPill(stringResource(R.string.settings_clear), { onText("") }, quiet = true) + Spacer(Modifier.weight(1f)) + ActionPill(stringResource(R.string.editor_done), onDismiss) + } + } +} + +@Composable +private fun refusalWords(code: Long): String = stringResource( + when (code) { + Teleprompter.SNAPP_NOT_ENOUGH_BATTERY -> R.string.prompter_refused_battery + Teleprompter.SNAPP_THERMAL_THROTTLING -> R.string.prompter_refused_heat + Teleprompter.SNAPP_DENIED_BY_PEAK_POWER -> R.string.prompter_refused_power + else -> R.string.prompter_refused + }, +) + +@Composable +private fun sizeName(sp: Double): String = stringResource( + when (sp) { + SIZES[0] -> R.string.prompter_size_small + SIZES[1] -> R.string.prompter_size_medium + SIZES[2] -> R.string.prompter_size_large + else -> R.string.prompter_size_huge + }, +) + +/** As a multiple of the glasses' own default, the one thing the number means for sure. */ +private fun speedName(speed: Double): String = + java.text.NumberFormat.getNumberInstance().apply { maximumFractionDigits = 2 }.format(speed / DEFAULT_SPEED) + +private fun pasted(context: Context): String? = + context.getSystemService(ClipboardManager::class.java)?.primaryClip?.takeIf { it.itemCount > 0 }?.getItemAt(0)?.coerceToText(context)?.toString() + +// The glasses' default size is 35 sp; the Display's own body text runs 24 to 32. +private val SIZES = listOf(28.0, 35.0, 44.0, 56.0) +private const val DEFAULT_SPEED = 32.0 +private val SPEEDS = listOf(16.0, 24.0, 32.0, 48.0, 64.0) +private val BAR_ROOM = 120.dp +// The glasses' own teleprompter layout is not known; this keeps the preview's text off the square's edge. +private const val PROMPTER_INSET = 28f diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt index b9f9f32f..df3e2b5a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt @@ -217,6 +217,7 @@ fun HomeScreen( onLive: () -> Unit, onMetaAi: () -> Unit, onSettings: () -> Unit, + onDisplayTool: (com.macedonga.visor.display.DisplayTool) -> Unit = {}, model: HomeViewModel = hiltViewModel(), ) { val context = LocalContext.current @@ -519,6 +520,11 @@ fun HomeScreen( Spacer(Modifier.height(FOOT)) } } + // The Display's tools, for a Display: known by its build while connected, by its band while away. + val bandsHere = model.bands.all.value + if (!elsewhere && (com.macedonga.visor.settings.hasDisplay(state.info?.build) || bandsHere.any { it.glasses == current })) { + item(key = "display") { com.macedonga.visor.display.DisplayShelf(onDisplayTool, Modifier.padding(top = 4.dp, bottom = 8.dp).appear(entrance, 4)) } + } item(key = "places") { Places(onMedia, onLive, onMetaAi, onSettings, Modifier.appear(entrance, 5), forThisPair = !elsewhere) } } diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index fbb59aee..735ab655 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -924,5 +924,31 @@ Riprova Indicazioni per %1$s Riproduci %1$s + Teleprompter + Testo + Scrivi o incolla il testo + Nessun testo + + %d parola + %d parole + + Dimensione del testo + Piccolo + Medio + Grande + Molto grande + Velocità di scorrimento + Disposizione + Scorrimento + Pagine + Mostra sugli occhiali + Aggiorna + Chiudi + Sugli occhiali + Connetti gli occhiali per mostrarlo + Gli occhiali non l\'hanno aperto + La batteria degli occhiali è troppo scarica + Gli occhiali sono troppo caldi + Gli occhiali non hanno abbastanza energia Configura di nuovo diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 38ded91f..ba8ecf21 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -926,5 +926,31 @@ Try again Directions to %1$s Play %1$s + Teleprompter + Script + Write or paste your script + No script + + %d word + %d words + + Text size + Small + Medium + Large + Very large + Scroll speed + Layout + Scroll + Pages + Show on glasses + Update + Close + On the glasses + Connect the glasses to show it + The glasses did not open it + The glasses\' battery is too low + The glasses are too warm + The glasses are short of power Set up again diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Teleprompter.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Teleprompter.kt new file mode 100644 index 00000000..43852f6d --- /dev/null +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Teleprompter.kt @@ -0,0 +1,124 @@ +package com.macedonga.visor.core.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.LinkState +import java.util.UUID +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch + +enum class PrompterPhase { IDLE, STARTING, SHOWING, CLOSING } + +/** [refusal] is the app manager's code when the glasses would not open it; [shown] and [look] what they have. */ +data class PrompterState( + val phase: PrompterPhase = PrompterPhase.IDLE, + val refusal: Long? = null, + val shown: String? = null, + val look: PrompterLook? = null, +) + +/** How the script shows: its size in sp on the glasses, its scroll speed (the glasses' default is 32), paged or scrolled. */ +data class PrompterLook(val sizeSp: Double = 35.0, val speed: Double = 32.0, val paged: Boolean = false, val followVoice: Boolean = false) + +/** + * The Display's teleprompter, run from the phone as the Meta app runs it: svc28 StartApp (2000) for app 35 with the + * script and its look as intent data, answered by 2001; a start while it shows replaces the text. StopApp (2002) + * closes it, answered by 2003. Following the voice is the glasses' own `asr_enabled`, which the Meta app never sends. + * The glasses do not say when the wearer closes it there. + */ +class Teleprompter(private val connection: GlassesConnection) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val _state = MutableStateFlow(PrompterState()) + val state: StateFlow = _state.asStateFlow() + private var wait: Job? = null + + init { + scope.launch { + connection.serviceData.collect { data -> + if (data.serviceId != SVC_COMPANION) return@collect + val code = runCatching { Wire.parse(data.payload).varint(1) }.getOrDefault(0L) + when (data.messageType) { + // Other apps' starts are answered here too; only the one being waited for counts. + MT_START_APP_RESPONSE -> if (_state.value.phase == PrompterPhase.STARTING) { + wait?.cancel() + _state.update { + if (code == SNAPP_SUCCESS || code == SNAPP_ALREADY_STARTED) it.copy(phase = PrompterPhase.SHOWING, refusal = null) + else PrompterState(refusal = code) + } + } + MT_STOP_APP_RESPONSE -> if (_state.value.phase == PrompterPhase.CLOSING) { + wait?.cancel() + _state.value = PrompterState() + } + } + } + } + scope.launch { + connection.state.map { it.link == LinkState.READY }.distinctUntilChanged().collect { up -> if (!up) _state.value = PrompterState() } + } + } + + /** Put [text] on the glasses, or replace what they show. */ + fun show(text: String, look: PrompterLook) { + if (connection.state.value.link != LinkState.READY) return + val data = intent("text", text) + intent("layout_type", if (look.paged) LAYOUT_PAGED else LAYOUT_SCROLL) + + intent("text_size_sp", look.sizeSp) + intent("scroll_speed", look.speed) + + intent("session_id", UUID.randomUUID().toString()) + intent("app_type", APP_TYPE_NATIVE) + + (if (look.followVoice) intent("asr_enabled", 1) else ByteArray(0)) + _state.update { it.copy(phase = PrompterPhase.STARTING, refusal = null, shown = text, look = look) } + connection.submit { it.sendOnService(SVC_COMPANION, MT_START_APP, Wire.fieldVarint(1, APP_TELEPROMPTER) + data) } + // No answer: the start went out, and the glasses may still be drawing it. + settleAfter { if (it.phase == PrompterPhase.STARTING) it.copy(phase = PrompterPhase.SHOWING) else it } + } + + fun close() { + if (_state.value.phase == PrompterPhase.IDLE) return + _state.update { it.copy(phase = PrompterPhase.CLOSING) } + connection.submit { it.sendOnService(SVC_COMPANION, MT_STOP_APP, Wire.fieldVarint(1, APP_TELEPROMPTER)) } + settleAfter { if (it.phase == PrompterPhase.CLOSING) PrompterState() else it } + } + + private fun settleAfter(change: (PrompterState) -> PrompterState) { + wait?.cancel() + wait = scope.launch { + delay(ANSWER_MS) + _state.update(change) + } + } + + /** One SNIntentData: the key, then the value in its type's field (3 int, 4 double, 5 bytes). */ + private fun intent(key: String, value: Int): ByteArray = entry(key, Wire.fieldVarint(3, value.toLong())) + + private fun intent(key: String, value: Double): ByteArray = entry(key, Wire.fieldDouble(4, value)) + + private fun intent(key: String, value: String): ByteArray = entry(key, Wire.fieldBytes(5, value.encodeToByteArray())) + + private fun entry(key: String, value: ByteArray): ByteArray = Wire.fieldBytes(3, Wire.fieldBytes(1, key.encodeToByteArray()) + value) + + companion object { + const val SNAPP_SUCCESS = 1L + const val SNAPP_NOT_ENOUGH_BATTERY = 6L + const val SNAPP_THERMAL_THROTTLING = 7L + const val SNAPP_ALREADY_STARTED = 12L + const val SNAPP_DENIED_BY_PEAK_POWER = 15L + private const val SVC_COMPANION = 28 + private const val MT_START_APP = 2000 + private const val MT_START_APP_RESPONSE = 2001 + private const val MT_STOP_APP = 2002 + private const val MT_STOP_APP_RESPONSE = 2003 + private const val APP_TELEPROMPTER = 35L + private const val LAYOUT_SCROLL = 0 + private const val LAYOUT_PAGED = 1 + private const val APP_TYPE_NATIVE = 0 + private const val ANSWER_MS = 7_000L + } +} diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt index 4278ac09..cf4b41de 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt @@ -53,6 +53,12 @@ object Wire { fun fieldBytes(fieldNumber: Int, value: ByteArray): ByteArray = tag(fieldNumber, 2) + encodeVarint(value.size.toLong()) + value + /** A double as protobuf writes it: fixed64, little-endian. */ + fun fieldDouble(fieldNumber: Int, value: Double): ByteArray { + val bits = java.lang.Double.doubleToRawLongBits(value) + return tag(fieldNumber, 1) + ByteArray(8) { i -> (bits ushr (8 * i)).toByte() } + } + // --- decode ------------------------------------------------------------ /** A varint and the offset just past it. */ From c9cbd7d4d069ba8a32b524df6c495b649678adda Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 10:04:13 +0200 Subject: [PATCH 10/90] Display: Gestures, a card on the home: the six band gestures in a list (select, back, scroll, Meta AI, wake and sleep, apps), each counted three times as the band hears it; the page takes the band from the Display, turns on its gesture streams (StreamControl, the map by stream type with priority 90 and GenericC50, the old bool too) and acks each gesture (GestureAck, MWA_ANDROID, direct) as the Meta app does, then gives it back. A single tap waits 420 ms for the double the band reports right after it. Settings > Neural Band can buzz the band (Haptics vibrate, units unknown). The home's Display cards are an icon, a name and a status --- .../kotlin/com/macedonga/visor/VisorNav.kt | 5 + .../com/macedonga/visor/band/BandLink.kt | 92 +++++++-- .../macedonga/visor/display/DisplayShelf.kt | 45 +++-- .../macedonga/visor/display/GesturesScreen.kt | 180 ++++++++++++++++++ .../com/macedonga/visor/settings/BandPart.kt | 7 + app/src/main/res/values-it/strings.xml | 18 ++ app/src/main/res/values/strings.xml | 18 ++ .../com/macedonga/visor/core/protocol/Band.kt | 51 ++++- .../com/macedonga/visor/core/protocol/Wire.kt | 6 + 9 files changed, 393 insertions(+), 29 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/display/GesturesScreen.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt index 419da986..49198e1e 100644 --- a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt +++ b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt @@ -150,6 +150,9 @@ data object NotifyAppsKey : NavKey @Serializable data object PrompterKey : NavKey +@Serializable +data object GesturesKey : NavKey + /** The app's graph: the home, and every screen it opens. */ @Composable fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) { @@ -207,6 +210,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) onDisplayTool = { tool -> when (tool) { com.macedonga.visor.display.DisplayTool.TELEPROMPTER -> backStack.add(PrompterKey) + com.macedonga.visor.display.DisplayTool.GESTURES -> backStack.add(GesturesKey) } }, ) @@ -262,6 +266,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) entry { PhoneAccessScreen() } entry { NotifyAppsScreen() } entry { com.macedonga.visor.display.PrompterScreen() } + entry { com.macedonga.visor.display.GesturesScreen() } entry { MetaAiScreen( onSettings = { backStack.add(AssistantSettingsKey) }, diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt index b5e3f9df..e5d6679a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt @@ -47,11 +47,16 @@ data class BandState( val at: Long? = null, // The Display holds the band; the phone steps back, as Meta's app does. val leased: Boolean = false, + // The band's RPC protocol, from its device info: 2 and up turn streams on by a map. + val protocol: Int? = null, + // A lesson has the band's gestures coming to the phone. + val gestures: Boolean = false, ) /** * The phone's own link to the Neural Band: up while Visor runs for the band's Display, down while the Display holds a - * lease on it (svc150). It reads the battery and the wrist over svc52822 and sets the wrist. + * lease on it (svc150), unless a lesson wants the band. It reads the battery and the wrist over svc52822 and sets the + * wrist; in a lesson it turns the gesture streams on and acks each gesture, as the Meta app does. */ @Singleton class BandLink @Inject constructor( @@ -66,6 +71,9 @@ class BandLink @Inject constructor( private val seq = AtomicInteger(1) private val leased = MutableStateFlow(false) private val held = MutableStateFlow(true) + private val lesson = MutableStateFlow(false) + private val _gestures = kotlinx.coroutines.flow.MutableSharedFlow(extraBufferCapacity = 32) + val gestures: kotlinx.coroutines.flow.SharedFlow = _gestures @Volatile private var session: GlassesSession? = null private var job: Job? = null @@ -79,8 +87,10 @@ class BandLink @Inject constructor( connection.state.collect { if (it.link != LinkState.READY && leased.value) leased.value = false } } scope.launch { - combine(connection.running, connection.pair, bands.all, leased, held) { running, pair, _, lease, hold -> - bands.forGlasses(pair)?.takeIf { running && hold && !lease } + // A lesson takes the band from the Display; when it ends, the lease decides again. + val stepBack = combine(leased, lesson) { lease, learning -> lease && !learning } + combine(connection.running, connection.pair, bands.all, stepBack, held) { running, pair, _, away, hold -> + bands.forGlasses(pair)?.takeIf { running && hold && !away } }.distinctUntilChanged().collect { band -> gate.withLock { switchTo(band) } } } scope.launch { leased.collect { lease -> _state.update { it.copy(leased = lease) } } } @@ -101,15 +111,47 @@ class BandLink @Inject constructor( held.value = true } + /** Have the band's gestures come here: the band is taken from the Display for as long as it lasts. */ + fun startLesson() { + if (lesson.value) return + lesson.value = true + log.add("Band: a lesson takes the band") + if (_state.value.link == LinkState.READY) streams(true) + } + + fun endLesson() { + if (!lesson.value) return + if (_state.value.link == LinkState.READY) streams(false) + lesson.value = false + log.add("Band: the lesson gives the band back") + } + + /** A short buzz on the band; false when it is not connected. */ + fun buzz(): Boolean = send { seq -> + listOf(Band.vibrate(seq.getAndIncrement(), BUZZ_DURATION, BUZZ_HZ, BUZZ_AMPLITUDE)) + } + /** Put the band on [left] or the right wrist; read back at once. */ - fun setLeftHanded(left: Boolean): Boolean { + fun setLeftHanded(left: Boolean): Boolean = send { seq -> + listOf(Band.configRequest(seq.getAndIncrement(), left), Band.configRequest(seq.getAndIncrement())) + } + + private fun streams(on: Boolean) { + val sent = send { seq -> listOf(Band.gestureStreams(seq.getAndIncrement(), on, _state.value.protocol)) } + if (sent) { + _state.update { it.copy(gestures = on) } + log.add("Band: gestures ${if (on) "on" else "off"} (protocol ${_state.value.protocol ?: "not read"})") + } + } + + /** Requests on the band's RPC, in order; false when the band is not connected. */ + private fun send(requests: (AtomicInteger) -> List): Boolean { val running = session ?: return false if (_state.value.link != LinkState.READY) return false running.submit( SessionCommand { s -> val mux = s.connection ?: return@SessionCommand - mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.configRequest(seq.getAndIncrement(), left)) - mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.configRequest(seq.getAndIncrement())) + for (r in requests(seq)) mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, r) }, ) return true @@ -125,7 +167,7 @@ class BandLink @Inject constructor( session = null job = null if (band == null) { - _state.update { it.copy(link = LinkState.DISCONNECTED) } + _state.update { it.copy(link = LinkState.DISCONNECTED, gestures = false) } return } val key = band.key ?: store.ownerKeyPem() ?: return @@ -141,10 +183,15 @@ class BandLink @Inject constructor( opened.events.collect { event -> when (event) { is LinkStateChanged -> { - _state.update { it.copy(link = event.state) } + _state.update { it.copy(link = event.state, gestures = if (event.state == LinkState.READY) it.gestures else false) } if (event.state == LinkState.READY) { log.add("Band: link up") - ask(opened) + ask(opened, info = true) + // The gestures wait for the device info, which picks how to ask for them; not for long. + if (lesson.value) launch { + delay(INFO_WAIT_MS) + if (lesson.value && !_state.value.gestures) streams(true) + } } } is ServiceData -> if (event.serviceId == ServiceId.WIRELESS_INPUT.value) onBand(event) @@ -165,19 +212,26 @@ class BandLink @Inject constructor( } } - private fun ask(opened: GlassesSession) = opened.submit( + private fun ask(opened: GlassesSession, info: Boolean = false) = opened.submit( SessionCommand { s -> val mux = s.connection ?: return@SessionCommand + if (info) mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.deviceInfoRequest(seq.getAndIncrement())) mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.batteryRequest(seq.getAndIncrement())) mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.configRequest(seq.getAndIncrement())) }, ) private fun onBand(event: ServiceData) { + val type = event.messageType and 0xFF + if (Band.isGesture(type)) return onGesture(type, event.payload) val battery = runCatching { - when (event.messageType and 0xFF) { + when (type) { Band.RESPONSE -> Band.response(event.payload).also { r -> if (r.leftHanded != null) _state.update { it.copy(leftHanded = r.leftHanded) } + if (r.protocol != null) { + _state.update { it.copy(protocol = r.protocol) } + if (lesson.value && !_state.value.gestures) streams(true) + } }.battery Band.STREAM_UPDATE -> Band.streamBattery(event.payload) else -> null @@ -186,7 +240,23 @@ class BandLink @Inject constructor( _state.update { it.copy(battery = battery.percent, charging = battery.charging, at = System.currentTimeMillis()) } } + // Every gesture is acked, as the Meta app does; a lesson's gestures go in the log so the names can be checked. + private fun onGesture(type: Int, payload: ByteArray) { + val g = runCatching { Band.gesture(type, payload) }.getOrElse { + log.add("Band: unreadable gesture type $type: ${it.message}") + return + } + send { seq -> listOf(Band.gestureAck(seq.getAndIncrement(), g.seq)) } + if (lesson.value) log.add("Band: gesture type ${g.type}, finger ${g.finger}, action ${g.action}, derived ${g.derived}") + _gestures.tryEmit(g) + } + private companion object { const val POLL_MS = 5 * 60_000L + const val INFO_WAIT_MS = 1_500L + // Nothing in the Meta app sends a buzz, so these are a guess at milliseconds, hertz and a 0-255 strength. + const val BUZZ_DURATION = 200L + const val BUZZ_HZ = 170f + const val BUZZ_AMPLITUDE = 160 } } diff --git a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt index 81036572..2814e3aa 100644 --- a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt +++ b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt @@ -6,17 +6,19 @@ import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.Spacer -import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.width import androidx.compose.foundation.lazy.LazyRow import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Icon import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.vector.ImageVector import androidx.compose.ui.res.pluralStringResource import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.style.TextOverflow @@ -28,25 +30,31 @@ import com.macedonga.visor.R import com.macedonga.visor.core.connection.PrompterPhase import com.macedonga.visor.core.connection.Teleprompter import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.ui.VisorIcons import dagger.hilt.android.lifecycle.HiltViewModel import javax.inject.Inject /** What the Display's cards on the home open. */ -enum class DisplayTool { TELEPROMPTER } +enum class DisplayTool { TELEPROMPTER, GESTURES } @HiltViewModel -class ShelfModel @Inject constructor(val script: PrompterScript, val prompter: Teleprompter) : ViewModel() +class ShelfModel @Inject constructor( + val script: PrompterScript, + val prompter: Teleprompter, + val band: com.macedonga.visor.band.BandLink, +) : ViewModel() -/** The Display's tools on the home: a card each, the glasses' square on it showing what the tool puts there. */ +/** The Display's tools on the home: a card each, with what it holds now. */ @Composable fun DisplayShelf(onOpen: (DisplayTool) -> Unit, modifier: Modifier = Modifier, model: ShelfModel = hiltViewModel()) { val text by model.script.text.collectAsStateWithLifecycle() - val look by model.script.look.collectAsStateWithLifecycle() val prompter by model.prompter.state.collectAsStateWithLifecycle() + val band by model.band.state.collectAsStateWithLifecycle() LazyRow(modifier, contentPadding = PaddingValues(horizontal = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp)) { item(key = "prompter") { val words = text.split(Regex("\\s+")).count { it.isNotEmpty() } ToolCard( + VisorIcons.Lines, stringResource(R.string.prompter_title), when { prompter.phase == PrompterPhase.SHOWING -> stringResource(R.string.prompter_on_glasses) @@ -54,28 +62,35 @@ fun DisplayShelf(onOpen: (DisplayTool) -> Unit, modifier: Modifier = Modifier, m else -> pluralStringResource(R.plurals.prompter_words, words, words) }, onClick = { onOpen(DisplayTool.TELEPROMPTER) }, - ) { ScriptPreview(text, look, Modifier.fillMaxWidth(), radius = 10.dp) } + ) + } + item(key = "gestures") { + ToolCard( + VisorIcons.Band, + stringResource(R.string.gestures_title), + band.battery?.let { stringResource(R.string.gestures_card_battery, it) } ?: stringResource(R.string.gestures_card), + onClick = { onOpen(DisplayTool.GESTURES) }, + ) } } } @Composable -private fun ToolCard(title: String, status: String, onClick: () -> Unit, square: @Composable () -> Unit) { +private fun ToolCard(icon: ImageVector, title: String, status: String, onClick: () -> Unit) { Column( Modifier .width(CARD_WIDTH) .clip(RoundedCornerShape(18.dp)) .background(Visor.colors.raised) .clickable(onClick = onClick) - .padding(8.dp), + .padding(18.dp), ) { - square() - Column(Modifier.padding(start = 8.dp, end = 8.dp, top = 12.dp, bottom = 6.dp)) { - Text(title, style = Visor.type.heading, color = Visor.colors.textPrimary, maxLines = 1, overflow = TextOverflow.Ellipsis) - Spacer(Modifier.height(2.dp)) - Text(status, style = Visor.type.label, color = Visor.colors.textSecondary, maxLines = 1, overflow = TextOverflow.Ellipsis) - } + Icon(icon, null, Modifier.size(22.dp), tint = Visor.colors.textPrimary) + Spacer(Modifier.height(28.dp)) + Text(title, style = Visor.type.heading, color = Visor.colors.textPrimary, maxLines = 1, overflow = TextOverflow.Ellipsis) + Spacer(Modifier.height(2.dp)) + Text(status, style = Visor.type.label, color = Visor.colors.textSecondary, maxLines = 1, overflow = TextOverflow.Ellipsis) } } -private val CARD_WIDTH = 172.dp +private val CARD_WIDTH = 160.dp diff --git a/app/src/main/kotlin/com/macedonga/visor/display/GesturesScreen.kt b/app/src/main/kotlin/com/macedonga/visor/display/GesturesScreen.kt new file mode 100644 index 00000000..1b792af8 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/display/GesturesScreen.kt @@ -0,0 +1,180 @@ +package com.macedonga.visor.display + +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.material3.Icon +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateListOf +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.macedonga.visor.R +import com.macedonga.visor.band.BandLink +import com.macedonga.visor.band.Bands +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.core.protocol.Band +import com.macedonga.visor.settings.RowFrame +import com.macedonga.visor.settings.Rows +import com.macedonga.visor.settings.SettingsPage +import com.macedonga.visor.ui.VisorIcons +import com.macedonga.visor.ui.rememberHaptics +import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject +import kotlinx.coroutines.Job +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.launch + +/** One gesture to learn: what it does on the Display and how it is made. */ +enum class Lesson(val title: Int, val how: Int) { + SELECT(R.string.lesson_select, R.string.lesson_select_how), + BACK(R.string.lesson_back, R.string.lesson_back_how), + SCROLL(R.string.lesson_scroll, R.string.lesson_scroll_how), + ASSISTANT(R.string.lesson_assistant, R.string.lesson_assistant_how), + WAKE(R.string.lesson_wake, R.string.lesson_wake_how), + APPS(R.string.lesson_apps, R.string.lesson_apps_how); + + companion object { + /** + * The band's gestures as lessons, as the Display's input filter takes them: an index tap selects, a middle tap goes + * back, a thumb swipe scrolls, the thumb's double tap is Meta AI, the middle's wakes, holding the middle opens the + * apps. Each tap comes as press, release, button release and single tap; the second tap of a double adds a double + * tap right after its single tap. So a single tap waits a moment for a double, and only the final events count. + */ + suspend fun judge(gestures: Flow, onLesson: suspend (Lesson) -> Unit) = coroutineScope { + var pending: Job? = null + var quietUntil = 0L + gestures.collect { g -> + val now = System.currentTimeMillis() + when { + g.derived == DERIVED_SINGLE_TAP -> { + pending?.cancel() + if (now < quietUntil) return@collect + val lesson = when (g.finger) { + FINGER_INDEX -> SELECT + FINGER_MIDDLE -> BACK + else -> null + } ?: return@collect + pending = launch { + delay(DOUBLE_WINDOW_MS) + onLesson(lesson) + } + } + g.derived == DERIVED_DOUBLE_TAP -> { + pending?.cancel() + when (g.finger) { + FINGER_MIDDLE -> onLesson(WAKE) + FINGER_THUMB, FINGER_INDEX -> onLesson(ASSISTANT) + } + } + // A hold, and a swipe, can come as more than one event: the first counts. + now < quietUntil -> Unit + g.derived == DERIVED_HOLD && g.finger == FINGER_MIDDLE -> { + pending?.cancel() + quietUntil = now + QUIET_MS + onLesson(APPS) + } + g.action in ACTION_UP..ACTION_RIGHT || g.derived in DERIVED_UP..DERIVED_RIGHT -> { + quietUntil = now + QUIET_MS + onLesson(SCROLL) + } + } + } + } + + private const val FINGER_THUMB = 1 + private const val FINGER_INDEX = 2 + private const val FINGER_MIDDLE = 3 + private const val ACTION_UP = 6 + private const val ACTION_RIGHT = 9 + private const val DERIVED_SINGLE_TAP = 1 + private const val DERIVED_DOUBLE_TAP = 2 + private const val DERIVED_HOLD = 3 + private const val DERIVED_UP = 5 + private const val DERIVED_RIGHT = 8 + // The second tap of a double came about 200 ms after the first on the user's band. + private const val DOUBLE_WINDOW_MS = 420L + private const val QUIET_MS = 600L + } +} + +@HiltViewModel +class GesturesModel @Inject constructor(val band: BandLink, val bands: Bands, val connection: GlassesConnection) : ViewModel() + +/** + * The band's gestures in a list: what each does and how it is made, three dots that fill as the band hears it and a + * check once it has three times. The band is the phone's while the page is open. + */ +@Composable +fun GesturesScreen(model: GesturesModel = hiltViewModel()) { + val band by model.band.state.collectAsStateWithLifecycle() + val all by model.bands.all.collectAsStateWithLifecycle() + val pair by model.connection.pair.collectAsStateWithLifecycle() + val setUp = all.any { it.glasses == pair } + DisposableEffect(Unit) { + model.band.startLesson() + onDispose { model.band.endLesson() } + } + val haptics = rememberHaptics() + val counts = remember { mutableStateListOf(*Array(Lesson.entries.size) { 0 }) } + LaunchedEffect(Unit) { + Lesson.judge(model.band.gestures) { lesson -> + val i = lesson.ordinal + if (counts[i] >= GOAL) return@judge + counts[i]++ + if (counts[i] < GOAL) haptics.tick(0.7f) else haptics.done() + } + } + SettingsPage( + stringResource(R.string.gestures_title), + // A line only while practice cannot happen. + lede = when { + !setUp -> stringResource(R.string.gestures_no_band) + !band.gestures -> stringResource(R.string.gestures_waiting) + else -> null + }, + ) { + item(key = "lessons") { + Rows( + *Lesson.entries.map { lesson -> + @Composable { + RowFrame(stringResource(lesson.title), onClick = null, sub = stringResource(lesson.how)) { Progress(counts[lesson.ordinal]) } + } + }.toTypedArray(), + ) + } + } +} + +/** Three dots, one lit for each time the band heard it; a check once all are. */ +@Composable +private fun Progress(count: Int) { + if (count >= GOAL) { + Icon(VisorIcons.Check, null, Modifier.padding(start = 12.dp).size(20.dp), tint = Color.White) + return + } + Row(Modifier.padding(start = 12.dp), horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) { + repeat(GOAL) { i -> + Box(Modifier.size(7.dp).clip(CircleShape).background(if (i < count) Color.White else Visor.colors.control)) + } + } +} + +private const val GOAL = 3 diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt index 3bf703ef..0ef7c331 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt @@ -40,6 +40,13 @@ internal fun BandPart(model: SettingsViewModel) { locked = band.link != LinkState.READY, ) }, + { + ValueRow( + stringResource(R.string.band_buzz), null, + onClick = { model.band.buzz() }, + locked = band.link != LinkState.READY, + ) + }, { SetUpRow(R.string.band_set_up_again) }, ) } diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 735ab655..ba5ccf9d 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -951,4 +951,22 @@ Gli occhiali sono troppo caldi Gli occhiali non hanno abbastanza energia Configura di nuovo + Gesti + Impara i gesti + Bracciale %1$d%% + Connessione al bracciale + Configura il Neural Band nelle Impostazioni + Seleziona + Tocca insieme pollice e indice + Indietro + Tocca insieme pollice e medio + Scorri + Fai scorrere il pollice lungo il lato dell\'indice + Meta AI + Tocca due volte il pollice sul lato dell\'indice + Riattiva e sospendi + Tocca due volte insieme pollice e medio + App + Tieni uniti pollice e medio + Fai vibrare diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index ba8ecf21..161cc911 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -953,4 +953,22 @@ The glasses are too warm The glasses are short of power Set up again + Gestures + Learn the gestures + Band %1$d%% + Connecting to the band + Set up the Neural Band in Settings + Select + Tap your thumb and index finger together + Back + Tap your thumb and middle finger together + Scroll + Swipe your thumb along the side of your index finger + Meta AI + Double tap your thumb on the side of your index finger + Wake and sleep + Double tap your thumb and middle finger together + Apps + Hold your thumb and middle finger together + Vibrate diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt index c661b5f3..2731848f 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt @@ -14,12 +14,50 @@ object Band { class Battery(val percent: Int, val charging: Boolean) - /** RpcResponse {1: seq, 2: code, 3: BatteryInfoResp, 6: ConfigResp, ...}, as far as Visor reads it. */ - class Response(val seq: Int, val code: Int, val battery: Battery?, val leftHanded: Boolean?) + /** RpcResponse {1: seq, 2: code, 3: BatteryInfoResp, 4: DeviceInfoResp, 6: ConfigResp, ...}, as far as Visor reads it. */ + class Response(val seq: Int, val code: Int, val battery: Battery?, val leftHanded: Boolean?, val protocol: Int? = null) + + /** + * A gesture, `EmgImu$GestureEvent`, sent as stream type 13 or a granular type from 60 to 200: {1 seq, 3 finger, 4 action, + * 5 derived action}. Fingers: THUMB 1, INDEX 2, MIDDLE 3. Actions: PRESS 1, RELEASE 2, TAP 3, DOUBLETAP 4, CLICK 5, UP 6, + * DOWN 7, LEFT 8, RIGHT 9. Derived: SINGLE_TAP 1, DOUBLE_TAP 2, BUTTON_HOLD 3, BUTTON_RELEASE 4, the four directions 5-8. + */ + class Gesture(val type: Int, val seq: Int, val finger: Int, val action: Int, val derived: Int) /** RpcRequest {1: seq, 2: BatteryInfoReq {}}. */ fun batteryRequest(seq: Int): ByteArray = request(seq, 2, ByteArray(0)) + /** RpcRequest {1: seq, 3: DeviceInfoReq {}}: the answer names the band's protocol version (field 15). */ + fun deviceInfoRequest(seq: Int): ByteArray = request(seq, 3, ByteArray(0)) + + /** + * RpcRequest {1: seq, 4: StreamControlReq}: the gestures on or off, as the Meta app asks. From protocol 2 a map by stream + * type (field 36) with priority 90 (35) and the use case "GenericC50" (37); before it the one bool, enableGestures (3). + */ + fun gestureStreams(seq: Int, on: Boolean, protocol: Int?): ByteArray { + val map = protocol == null || protocol >= 2 + val entries = if (map) GESTURE_STREAMS.fold(ByteArray(0)) { out, type -> + out + Wire.fieldBytes(36, Wire.fieldVarint(1, type.toLong()) + Wire.fieldVarint(2, if (on) 1 else 0)) + } + Wire.fieldVarint(35, PRIORITY_GENERIC_C50) + Wire.fieldBytes(37, "GenericC50".encodeToByteArray()) else ByteArray(0) + val bool = if (!map || protocol == null) Wire.fieldVarint(3, if (on) 1 else 0) else ByteArray(0) + return request(seq, 4, bool + entries) + } + + /** RpcRequest {1: seq, 11: GestureAckReq {1 the gesture's seq, 2 source MWA_ANDROID, 3 topology DIRECT}}, one per gesture. */ + fun gestureAck(seq: Int, gesture: Int): ByteArray = + request(seq, 11, Wire.fieldVarint(1, gesture.toLong()) + Wire.fieldVarint(2, SOURCE_MWA_ANDROID) + Wire.fieldVarint(3, TOPOLOGY_DIRECT)) + + /** RpcRequest {1: seq, 17: HapticsReq {1: VibrateReq {1 duration, 2 frequency, 3 amplitude}}}; the units are not known. */ + fun vibrate(seq: Int, duration: Long, frequency: Float, amplitude: Int): ByteArray = + request(seq, 17, Wire.fieldBytes(1, Wire.fieldVarint(1, duration) + Wire.fieldFloat(2, frequency) + Wire.fieldVarint(3, amplitude.toLong()))) + + fun isGesture(type: Int): Boolean = type == 13 || type in 60..200 + + fun gesture(type: Int, payload: ByteArray): Gesture { + val f = Wire.parse(payload) + return Gesture(type, f.varint(1).toInt(), f.varint(3).toInt(), f.varint(4).toInt(), f.varint(5).toInt()) + } + /** RpcRequest {1: seq, 5: ConfigReq}: with [leftHanded] it sets the wrist (field 10), empty it reads the config. */ fun configRequest(seq: Int, leftHanded: Boolean? = null): ByteArray = request(seq, 5, leftHanded?.let { Wire.fieldVarint(10, if (it) 1 else 0) } ?: ByteArray(0)) @@ -28,7 +66,8 @@ object Band { val f = Wire.parse(payload) val battery = if (f.has(3)) battery(Wire.parse(f.bytes(3)).bytes(1)) else null val left = if (f.has(6)) Wire.parse(f.bytes(6)).let { c -> if (c.has(10)) c.varint(10) != 0L else null } else null - return Response(f.varint(1).toInt(), f.varint(2).toInt(), battery, left) + val protocol = if (f.has(4)) Wire.parse(f.bytes(4)).let { d -> if (d.has(15)) d.varint(15).toInt() else null } else null + return Response(f.varint(1).toInt(), f.varint(2).toInt(), battery, left, protocol) } /** RpcStreamUpdate field 2, BatteryUpdate {1: BatteryData}. */ @@ -44,6 +83,12 @@ object Band { } private fun request(seq: Int, field: Int, body: ByteArray) = Wire.fieldVarint(1, seq.toLong()) + Wire.fieldBytes(field, body) + + // Every gesture stream the Meta app turns on: 60-70, 100-109, 150-156 and 200. + private val GESTURE_STREAMS = (60..70) + (100..109) + (150..156) + 200 + private const val PRIORITY_GENERIC_C50 = 90L + private const val SOURCE_MWA_ANDROID = 2L + private const val TOPOLOGY_DIRECT = 0L } /** svc150's NotifyLease {1: peerKeyHint, 2: leaseType}: the Display taking its band. Type 1 on the channel it opens. */ diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt index cf4b41de..ba4188b2 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt @@ -53,6 +53,12 @@ object Wire { fun fieldBytes(fieldNumber: Int, value: ByteArray): ByteArray = tag(fieldNumber, 2) + encodeVarint(value.size.toLong()) + value + /** A float as protobuf writes it: fixed32, little-endian. */ + fun fieldFloat(fieldNumber: Int, value: Float): ByteArray { + val bits = java.lang.Float.floatToRawIntBits(value) + return tag(fieldNumber, 5) + ByteArray(4) { i -> (bits ushr (8 * i)).toByte() } + } + /** A double as protobuf writes it: fixed64, little-endian. */ fun fieldDouble(fieldNumber: Int, value: Double): ByteArray { val bits = java.lang.Double.doubleToRawLongBits(value) From 49f5cdc7f3d1b44f29a36eca48e6261941475a5e Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 10:23:44 +0200 Subject: [PATCH 11/90] Neural Band: the band takes one host at a time, so Visor leaves it to the Display while the Display is connected and takes it only for the Gestures page or one errand (the wrist, a buzz); its battery, how tight it sits and which way round it is come from the Display's own device state (svc28 fields 23, 30, 34), on the home, the Gestures card and Settings --- .../com/macedonga/visor/band/BandLink.kt | 38 ++++++++++++++----- .../macedonga/visor/display/DisplayShelf.kt | 5 ++- .../com/macedonga/visor/home/HomeScreen.kt | 3 +- .../com/macedonga/visor/settings/BandPart.kt | 31 ++++++++++----- app/src/main/res/values-it/strings.xml | 4 ++ app/src/main/res/values/strings.xml | 4 ++ .../macedonga/visor/core/connection/Status.kt | 19 ++++++++++ 7 files changed, 84 insertions(+), 20 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt index e5d6679a..2bd21136 100644 --- a/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt @@ -32,6 +32,8 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex @@ -54,8 +56,9 @@ data class BandState( ) /** - * The phone's own link to the Neural Band: up while Visor runs for the band's Display, down while the Display holds a - * lease on it (svc150), unless a lesson wants the band. It reads the battery and the wrist over svc52822 and sets the + * The phone's own link to the Neural Band: the band takes one host at a time, so the phone leaves it to the Display + * while the Display is connected or holds a lease on it (svc150), and takes it only for a lesson or one errand (the + * wrist, a buzz); the Display reports the band's battery itself. It reads the battery and the wrist over svc52822 and sets the * wrist; in a lesson it turns the gesture streams on and acks each gesture, as the Meta app does. */ @Singleton @@ -72,6 +75,7 @@ class BandLink @Inject constructor( private val leased = MutableStateFlow(false) private val held = MutableStateFlow(true) private val lesson = MutableStateFlow(false) + private val borrowing = MutableStateFlow(false) private val _gestures = kotlinx.coroutines.flow.MutableSharedFlow(extraBufferCapacity = 32) val gestures: kotlinx.coroutines.flow.SharedFlow = _gestures @@ -87,8 +91,9 @@ class BandLink @Inject constructor( connection.state.collect { if (it.link != LinkState.READY && leased.value) leased.value = false } } scope.launch { - // A lesson takes the band from the Display; when it ends, the lease decides again. - val stepBack = combine(leased, lesson) { lease, learning -> lease && !learning } + // A lesson or an errand takes the band from the Display; when it ends, the band goes back. + val displayUp = connection.state.map { it.link == LinkState.READY }.distinctUntilChanged() + val stepBack = combine(leased, displayUp, lesson, borrowing) { lease, up, learning, errand -> (lease || up) && !learning && !errand } combine(connection.running, connection.pair, bands.all, stepBack, held) { running, pair, _, away, hold -> bands.forGlasses(pair)?.takeIf { running && hold && !away } }.distinctUntilChanged().collect { band -> gate.withLock { switchTo(band) } } @@ -126,14 +131,27 @@ class BandLink @Inject constructor( log.add("Band: the lesson gives the band back") } - /** A short buzz on the band; false when it is not connected. */ - fun buzz(): Boolean = send { seq -> - listOf(Band.vibrate(seq.getAndIncrement(), BUZZ_DURATION, BUZZ_HZ, BUZZ_AMPLITUDE)) + /** A short buzz on the band, taken from the Display for it. */ + fun buzz() = borrow { seq -> listOf(Band.vibrate(seq.getAndIncrement(), BUZZ_DURATION, BUZZ_HZ, BUZZ_AMPLITUDE)) } + + /** Take the band for one errand, then give it back; at once when the phone already has it. */ + private fun borrow(requests: (AtomicInteger) -> List) { + if (send(requests)) return + scope.launch { + borrowing.value = true + val up = kotlinx.coroutines.withTimeoutOrNull(BORROW_MS) { _state.first { it.link == LinkState.READY } } != null + if (up) { + send(requests) + delay(ERRAND_MS) + } else log.add("Band: not reached for an errand in ${BORROW_MS / 1000} s") + borrowing.value = false + } } /** Put the band on [left] or the right wrist; read back at once. */ - fun setLeftHanded(left: Boolean): Boolean = send { seq -> - listOf(Band.configRequest(seq.getAndIncrement(), left), Band.configRequest(seq.getAndIncrement())) + fun setLeftHanded(left: Boolean) { + _state.update { it.copy(leftHanded = left) } + borrow { seq -> listOf(Band.configRequest(seq.getAndIncrement(), left), Band.configRequest(seq.getAndIncrement())) } } private fun streams(on: Boolean) { @@ -254,6 +272,8 @@ class BandLink @Inject constructor( private companion object { const val POLL_MS = 5 * 60_000L const val INFO_WAIT_MS = 1_500L + const val BORROW_MS = 20_000L + const val ERRAND_MS = 1_500L // Nothing in the Meta app sends a buzz, so these are a guess at milliseconds, hertz and a 0-255 strength. const val BUZZ_DURATION = 200L const val BUZZ_HZ = 170f diff --git a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt index 2814e3aa..27f4fd19 100644 --- a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt +++ b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt @@ -42,6 +42,7 @@ class ShelfModel @Inject constructor( val script: PrompterScript, val prompter: Teleprompter, val band: com.macedonga.visor.band.BandLink, + val connection: com.macedonga.visor.core.connection.GlassesConnection, ) : ViewModel() /** The Display's tools on the home: a card each, with what it holds now. */ @@ -50,6 +51,8 @@ fun DisplayShelf(onOpen: (DisplayTool) -> Unit, modifier: Modifier = Modifier, m val text by model.script.text.collectAsStateWithLifecycle() val prompter by model.prompter.state.collectAsStateWithLifecycle() val band by model.band.state.collectAsStateWithLifecycle() + val glasses by model.connection.state.collectAsStateWithLifecycle() + val bandBattery = glasses.device?.bandBattery ?: band.battery LazyRow(modifier, contentPadding = PaddingValues(horizontal = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp)) { item(key = "prompter") { val words = text.split(Regex("\\s+")).count { it.isNotEmpty() } @@ -68,7 +71,7 @@ fun DisplayShelf(onOpen: (DisplayTool) -> Unit, modifier: Modifier = Modifier, m ToolCard( VisorIcons.Band, stringResource(R.string.gestures_title), - band.battery?.let { stringResource(R.string.gestures_card_battery, it) } ?: stringResource(R.string.gestures_card), + bandBattery?.let { stringResource(R.string.gestures_card_battery, it) } ?: stringResource(R.string.gestures_card), onClick = { onOpen(DisplayTool.GESTURES) }, ) } diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt index df3e2b5a..137819cf 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt @@ -586,7 +586,8 @@ fun HomeScreen( StatusPill( state, mood, lastBattery, paired, allowed, transferring, importProgress, waiting = if (importing) 0 else fresh.size, onImport = onImport, onAllow = { ask.launch(wanted()) }, onSetUp = onSetUp, modifier = Modifier, - band = ours?.battery, bandCharging = ours?.charging == true, refused = refused, + band = state.device?.bandBattery ?: ours?.battery, + bandCharging = state.device?.bandCharging ?: (ours?.charging == true), refused = refused, ) } if (pairs.isNotEmpty()) PairDots(pairs.size + 1, shownIndex, Modifier.padding(top = 12.dp)) diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt index 0ef7c331..340ff753 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt @@ -14,6 +14,11 @@ internal fun BandPart(model: SettingsViewModel) { val pair by model.connection.pair.collectAsStateWithLifecycle() val all by model.bands.all.collectAsStateWithLifecycle() val band by model.band.state.collectAsStateWithLifecycle() + val glasses by model.connection.state.collectAsStateWithLifecycle() + // The Display reports its band; the phone's own reading is for when the Display is away. + val device = glasses.device + val battery = device?.bandBattery ?: band.battery + val charging = device?.bandCharging ?: band.charging val setup by model.bandSetup.status.collectAsStateWithLifecycle() val running = setup is BandSetup.Status.Running val problem = (setup as? BandSetup.Status.Failed)?.let { failWords(it) } @@ -30,23 +35,24 @@ internal fun BandPart(model: SettingsViewModel) { return } Rows( - { ValueRow(stringResource(R.string.band_battery), band.battery?.let { batteryWords(it, band.charging) }, onClick = null) }, + { ValueRow(stringResource(R.string.band_battery), battery?.let { batteryWords(it, charging) }, onClick = null) }, + fitWords(device?.bandFit)?.let { fit -> + row { + ValueRow( + stringResource(R.string.band_fit), fit, onClick = null, + problem = if (device?.bandOrientation == "INCORRECT") stringResource(R.string.band_wrong_way) else null, + ) + } + }, { PillsRow( stringResource(R.string.band_wrist), listOf(true to stringResource(R.string.band_left), false to stringResource(R.string.band_right)), band.leftHanded, onPick = { model.band.setLeftHanded(it) }, - locked = band.link != LinkState.READY, - ) - }, - { - ValueRow( - stringResource(R.string.band_buzz), null, - onClick = { model.band.buzz() }, - locked = band.link != LinkState.READY, ) }, + { ValueRow(stringResource(R.string.band_buzz), null, onClick = { model.band.buzz() }) }, { SetUpRow(R.string.band_set_up_again) }, ) } @@ -55,6 +61,13 @@ internal fun BandPart(model: SettingsViewModel) { internal fun batteryWords(percent: Int, charging: Boolean?): String = stringResource(if (charging == true) R.string.glasses_battery_charging else R.string.band_percent, percent) +@Composable +private fun fitWords(fit: String?): String? = when (fit) { + "LOOSE" -> stringResource(R.string.band_fit_loose) + "TIGHT" -> stringResource(R.string.band_fit_tight) + else -> null +} + @Composable internal fun wristWords(left: Boolean): String = stringResource(if (left) R.string.band_left else R.string.band_right) diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index ba5ccf9d..f54350f8 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -969,4 +969,8 @@ App Tieni uniti pollice e medio Fai vibrare + Vestibilità + Larga + Aderente + Indossato al contrario diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 161cc911..cf6c7e8e 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -971,4 +971,8 @@ Apps Hold your thumb and middle finger together Vibrate + Fit + Loose + Snug + Worn the wrong way round diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt index 62db46a1..3e06846d 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt @@ -206,6 +206,11 @@ data class DeviceState( val odc: String? = null, val overhead: Boolean? = null, val interactionKey: String? = null, + // The Display's Neural Band, as the Display reports it: battery (23), how tight it sits (30), which way round (34). + val bandBattery: Int? = null, + val bandCharging: Boolean? = null, + val bandFit: String? = null, + val bandOrientation: String? = null, ) { val worn: Boolean? get() = mount?.let { it == "DON" } } @@ -264,6 +269,10 @@ fun mergeDeviceState(base: DeviceState, update: DeviceState): DeviceState = Devi odc = update.odc ?: base.odc, overhead = update.overhead ?: base.overhead, interactionKey = update.interactionKey ?: base.interactionKey, + bandBattery = update.bandBattery ?: base.bandBattery, + bandCharging = update.bandCharging ?: base.bandCharging, + bandFit = update.bandFit ?: base.bandFit, + bandOrientation = update.bandOrientation ?: base.bandOrientation, ) fun isDeviceStateType(messageType: Int): Boolean = messageType in DEVICE_STATE_TYPES @@ -300,6 +309,9 @@ fun isDisplayBuild(build: String?): Boolean = build?.split('/')?.getOrNull(2)?.s private val DISPLAY_DEVICES = setOf("greatwhite") +private val BAND_FIT = mapOf(0L to "OFF", 1L to "LOOSE", 2L to "TIGHT", 3L to "UNKNOWN") +private val BAND_ORIENTATION = mapOf(0L to "UNKNOWN", 1L to "INCONCLUSIVE", 2L to "CORRECT", 3L to "INCORRECT") + private fun text(fields: Wire.Fields, number: Int): String = String(fields.bytes(number), Charsets.UTF_8) @@ -408,6 +420,13 @@ fun decodeDeviceStateFull(payload: ByteArray): DeviceState { if (top.has(22)) { state = state.copy(registeredApps = top.repeatedBytes(22).map { Wire.parse(it).varint(1) }) } + if (top.has(23)) { + // EmgBatteryState {1 level, 2 charge: CHARGING 1, NOT_CHARGING 2}. + val b = Wire.parse(top.bytes(23)) + state = state.copy(bandBattery = b.varint(1).toInt(), bandCharging = b.varint(2) == 1L) + } + if (top.has(30)) state = state.copy(bandFit = name(BAND_FIT, Wire.parse(top.bytes(30)).varint(1))) + if (top.has(34)) state = state.copy(bandOrientation = name(BAND_ORIENTATION, Wire.parse(top.bytes(34)).varint(1))) if (top.has(25)) { val v = Wire.parse(top.bytes(25)) state = state.copy(volume = Volume( From 3fc052940b1e572613548a48b176d14f380d18b6 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 10:29:19 +0200 Subject: [PATCH 12/90] Glasses: the tampering lux is read as the float it is, and Visor knows the other Meta product ids: Ray-Ban Display 270 and 271, and the Neural Band 1538 in setup. Topic choice rows drop hidden options unless the stored value is one of them. --- .../visor/settings/SettingControls.kt | 2 +- .../com/macedonga/visor/settings/Topics.kt | 2 +- .../macedonga/visor/setup/NearbyGlasses.kt | 6 ++-- .../macedonga/visor/core/connection/Status.kt | 4 +-- .../com/macedonga/visor/core/protocol/Wire.kt | 34 +++++++++++++++++-- 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/SettingControls.kt b/app/src/main/kotlin/com/macedonga/visor/settings/SettingControls.kt index cd9d8311..87fe3bb1 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/SettingControls.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/SettingControls.kt @@ -147,7 +147,7 @@ fun GlassesRow(key: String, engineering: Boolean = false, overline: String? = nu is Control.Choice -> { val current = value.trim() // Options the Meta app keeps for other glasses show only in Advanced. - val shown = c.options.filter { engineering || it.offeredOn(state.product) || it.value == current } + val shown = c.options.filter { engineering || it.value == current || (!it.hidden && it.offeredOn(state.product)) } if (shown.size <= 4 && shown.all { it.label.length <= 16 && it.detail == null } && shown.any { it.value == current }) { PillsRow( spec.label, shown.map { it.value to it.label }, current, onPick = write, diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt b/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt index c0a7fa94..b3c5535b 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt @@ -107,7 +107,7 @@ val DISPLAY_GROUPS = setOf("Display", "Neural band gestures") fun hasDisplay(build: String?): Boolean = productId(build) in DISPLAY_PRODUCTS // Meta Ray-Ban Display. -private val DISPLAY_PRODUCTS = setOf(259) +private val DISPLAY_PRODUCTS = setOf(259, 270, 271) /** Whether [product] has anything on this topic: the band is the Display's, and hearing is nothing but features. */ fun Topic.shownOn(product: Int?): Boolean = when (this) { diff --git a/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt b/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt index b90d3661..79664571 100644 --- a/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt +++ b/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt @@ -68,7 +68,7 @@ class NearbyGlasses @Inject constructor(@ApplicationContext private val context: val data = record.getManufacturerSpecificData(META_COMPANY) if (data == null && record.serviceUuids?.any { it.uuid == META_LE_SERVICE } != true) return val advert = data?.let(::advert) - if (advert?.product == PRODUCT_BAND) return + if (advert?.product in PRODUCT_BANDS) return val device = result.device val name = record.deviceName ?: runCatching { device.name }.getOrNull() ?: productName(advert?.product) if (BAND_NAME.containsMatchIn(name)) return @@ -146,13 +146,13 @@ class NearbyGlasses @Inject constructor(@ApplicationContext private val context: private fun productName(product: Int?) = when (product) { 257 -> "Ray-Ban Stories" - 259 -> "Meta Ray-Ban Display" + 259, 270, 271 -> "Meta Ray-Ban Display" else -> "Ray-Ban Meta" } private companion object { const val META_COMPANY = 0x01AB - const val PRODUCT_BAND = 1537 + val PRODUCT_BANDS = setOf(1537, 1538) const val FORGET_MS = 12_000L const val SWEEP_MS = 2_000L const val BOND_MS = 90_000L diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt index 3e06846d..628169a4 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt @@ -152,7 +152,7 @@ data class Led( val privacyLedActive: Boolean, ) -data class Tampering(val tampered: Boolean, val lux: Long, val indoor: Boolean) +data class Tampering(val tampered: Boolean, val lux: Float, val indoor: Boolean) data class Thermal( val health: String, @@ -381,7 +381,7 @@ fun decodeDeviceStateFull(payload: ByteArray): DeviceState { if (top.has(12)) { val t = Wire.parse(top.bytes(12)) state = state.copy(tampering = Tampering( - tampered = bool(t.varint(1)), lux = t.varint(2), indoor = bool(t.varint(3)), + tampered = bool(t.varint(1)), lux = t.float(2), indoor = bool(t.varint(3)), )) } if (top.has(13)) { diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt index ba4188b2..cdb722f7 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt @@ -113,6 +113,22 @@ object Wire { return default } + /** The first fixed32 value for [fn] as a float, or [default]. */ + fun float(fn: Int, default: Float = 0f): Float { + byField[fn]?.forEach { (wireType, value) -> + if (wireType == 5) return Float.fromBits(value as Int) + } + return default + } + + /** The first fixed64 value for [fn] as a double, or [default]. */ + fun double(fn: Int, default: Double = 0.0): Double { + byField[fn]?.forEach { (wireType, value) -> + if (wireType == 1) return Double.fromBits(value as Long) + } + return default + } + fun repeatedBytes(fn: Int): List = byField[fn].orEmpty().filter { it.first == 2 }.map { it.second as ByteArray } @@ -122,7 +138,13 @@ object Wire { fun fieldNumbers(): List = byField.keys.sorted() } - /** Parse a flat protobuf message: varint and length-delimited fields. */ + private fun littleEndian(data: ByteArray, at: Int, size: Int): Long { + var v = 0L + for (i in size - 1 downTo 0) v = (v shl 8) or (data[at + i].toLong() and 0xFF) + return v + } + + /** Parse a flat protobuf message: varint, fixed and length-delimited fields. */ fun parse(data: ByteArray): Fields { val fields = Fields() var offset = 0 @@ -156,8 +178,14 @@ object Wire { offset = if (fits) offset + take else n } - 5 -> offset += 4 // fixed32 - 1 -> offset += 8 // fixed64 + 5 -> { // fixed32, little-endian + if (offset + 4 <= n) fields.add(fieldNumber, 5, littleEndian(data, offset, 4).toInt()) + offset += 4 + } + 1 -> { // fixed64, little-endian + if (offset + 8 <= n) fields.add(fieldNumber, 1, littleEndian(data, offset, 8)) + offset += 8 + } else -> throw ProtocolFormatException( "unsupported wire type ${key.value and 0x07L} for field $fieldNumber" ) From 7059b218e22f6824e7d18d21d40fc9da8f23846f Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 10:30:22 +0200 Subject: [PATCH 13/90] Companion presence: when the associated glasses appear, the system binds GlassesPresence, a CompanionDeviceService with the background permissions the Meta app holds, and it starts the connection for the stored pair from the background, where a foreground service may not start otherwise. Visor asks the system to observe each association on every resume, which also covers the return from Android's association dialog; API 36 matches the association by id, 33 to 35 and 31 to 32 by address. Nothing is done when the glasses disappear. --- app/src/main/AndroidManifest.xml | 14 ++++++ .../com/macedonga/visor/MainActivity.kt | 9 ++++ .../visor/connection/GlassesPresence.kt | 47 +++++++++++++++++++ .../com/macedonga/visor/phone/Companion.kt | 19 ++++++++ 4 files changed, 89 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/GlassesPresence.kt diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 4a43d515..9712cebf 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -19,6 +19,10 @@ + + + + @@ -118,6 +122,16 @@ android:exported="false" android:foregroundServiceType="connectedDevice" /> + + + + + + + = Build.VERSION_CODES.BAKLAVA) { + cdm.myAssociations.forEach { cdm.startObservingDevicePresence(ObservingDevicePresenceRequest.Builder().setAssociationId(it.id).build()) } + } else { + @Suppress("DEPRECATION") + cdm.associations.forEach { cdm.startObservingDevicePresence(it) } + } + }.onFailure { log.add("Companion presence not observed: ${it.message}") } +} + private const val ASKED = "asked" From 1f60e7c5d4610bd61f1d146b6009e6331a77831f Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 10:30:42 +0200 Subject: [PATCH 14/90] Notifications on the glasses: a notification removed on the phone is removed on the glasses too, as the sent one is named by its key, provider and uuid; the glasses' app icon requests are answered with a 78 x 78 JPEG icon, or "unknown package"; the glasses' acks are logged; and chat messages say they are text, so the glasses offer smart replies. Co-Authored-By: Claude Haiku 5.5 Claude-Session: https://claude.ai/code/session_018hwZC33my6atwfE1RxLLja --- .../macedonga/visor/di/ConnectionModule.kt | 4 ++ .../visor/notify/GlassesNotification.kt | 6 +- .../macedonga/visor/notify/MirrorListener.kt | 34 ++++++++--- .../com/macedonga/visor/notify/NotifyIcons.kt | 57 +++++++++++++++++++ 4 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/notify/NotifyIcons.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index bdd41383..5f86dbb8 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -44,6 +44,8 @@ import com.macedonga.visor.core.settings.RlDriveResponder import com.macedonga.visor.core.settings.SVC_RLDRIVE import com.macedonga.visor.core.transport.FileWireLog import com.macedonga.visor.core.transport.LinkMeter +import com.macedonga.visor.notify.GlassesNotification +import com.macedonga.visor.notify.NotifyIcons import com.macedonga.visor.phone.PhoneActions import dagger.Module import dagger.Provides @@ -136,6 +138,7 @@ object ConnectionModule { hearing: javax.inject.Provider, assistant: javax.inject.Provider, lease: javax.inject.Provider, + icons: NotifyIcons, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -145,6 +148,7 @@ object ConnectionModule { SVC_ASSISTANT to Responder { message, mux -> assistant.get().respond(message, mux) }, SVC_GLASSES_PHOTO to photos, SVC_TAR to timers, + GlassesNotification.SVC to icons, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/GlassesNotification.kt b/app/src/main/kotlin/com/macedonga/visor/notify/GlassesNotification.kt index 28d24770..6efe614f 100644 --- a/app/src/main/kotlin/com/macedonga/visor/notify/GlassesNotification.kt +++ b/app/src/main/kotlin/com/macedonga/visor/notify/GlassesNotification.kt @@ -28,13 +28,14 @@ data class GlassesNotification( val flags: Int = 0, val category: String? = null, val channel: String? = null, + // The uuid this notification carries, which a removal names. + val uuid: String = UUID.randomUUID().toString(), ) { /** svc46 type 1: a single ChunkEnvelope holding the NotificationData. */ fun envelope(): ByteArray = Wire.fieldBytes(1, UUID.randomUUID().toString().toByteArray()) + Wire.fieldVarint(2, 0) + Wire.fieldVarint(3, 1) + Wire.fieldBytes(4, data()) fun data(): ByteArray { - val uuid = UUID.randomUUID().toString() val out = ByteArrayOutputStream() fun str(field: Int, s: String) { if (s.isNotEmpty()) out.write(Wire.fieldBytes(field, s.toByteArray())) @@ -57,6 +58,8 @@ data class GlassesNotification( val foa = ByteArrayOutputStream().apply { if (isGroup && groupName.isNotEmpty()) write(Wire.fieldBytes(1, groupName.toByteArray())) write(Wire.fieldBytes(2, threadId.ifEmpty { key }.toByteArray())) + // Explicit TEXT, so the Display offers smart replies: an absent type reads as none. + write(Wire.fieldVarint(9, 0)) if (isGroup) write(Wire.fieldVarint(10, 1)) if (provider == 2) write(Wire.fieldVarint(11, 1)) if (senderId.isNotEmpty()) write(Wire.fieldBytes(12, senderId.toByteArray())) @@ -97,6 +100,7 @@ data class GlassesNotification( companion object { const val SVC = 46 const val MT_UPSERT_CHUNK = 1 + const val MT_REMOVE = 1022 const val TYPE_MIRRORED = 14 } } diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt b/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt index ae00b34b..a58815ce 100644 --- a/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt +++ b/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt @@ -22,6 +22,7 @@ import kotlinx.coroutines.delay import com.macedonga.visor.core.assistant.Conversation import com.macedonga.visor.core.assistant.Messaging import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.protocol.Wire import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.core.settings.GlassesSettings import dagger.hilt.android.AndroidEntryPoint @@ -108,7 +109,7 @@ class MirrorListener : NotificationListenerService(), Messaging { val r = Ranking() PROVIDERS[sbn.packageName]?.let { provider -> message(sbn, provider, appName)?.let { m -> - deliver(sbn.key, m.envelope()) + deliver(m) replyAction(n)?.let { action -> replyable.remove(sbn.key) replyable[sbn.key] = Conversation(sbn.key, m.sender, appName) to action @@ -133,28 +134,45 @@ class MirrorListener : NotificationListenerService(), Messaging { category = n.category, channel = n.channelId, ) - deliver(sbn.key, note.envelope()) + deliver(note) } // What came while the wearer was talking with the assistant, newest per notification, and since when. - private val held = LinkedHashMap() + private val held = LinkedHashMap() private var heldSince = 0L + // The provider and uuid of the last notification sent for each key, which a removal names. + private val mirrored = LinkedHashMap>() + /** To the glasses now, or once the conversation with the assistant is over. */ - private fun deliver(key: String, envelope: ByteArray) { + private fun deliver(note: GlassesNotification) { if (talking()) { synchronized(held) { if (held.isEmpty()) heldSince = System.currentTimeMillis() - held.remove(key) - held[key] = envelope + held.remove(note.key) + held[note.key] = note } return } - send(envelope) + send(note) } - private fun send(envelope: ByteArray) = + private fun send(note: GlassesNotification) { + mirrored.remove(note.key) + mirrored[note.key] = note.provider to note.uuid + while (mirrored.size > 300) mirrored.remove(mirrored.keys.first()) + val envelope = note.envelope() connection.submit { it.sendOnService(GlassesNotification.SVC, GlassesNotification.MT_UPSERT_CHUNK, envelope) } + } + + /** A notification Visor sent is gone from the phone: the glasses take it away too. */ + override fun onNotificationRemoved(sbn: StatusBarNotification) { + // One held and not yet sent goes no further; one already on the glasses is removed. + synchronized(held) { held.remove(sbn.key) } + val (provider, uuid) = mirrored.remove(sbn.key) ?: return + val request = Wire.fieldBytes(1, sbn.key.toByteArray()) + Wire.fieldVarint(2, provider.toLong()) + Wire.fieldBytes(6, uuid.toByteArray()) + connection.submit { it.sendOnService(GlassesNotification.SVC, GlassesNotification.MT_REMOVE, request) } + } /** The glasses listening, thinking or answering, Visor's model at work, or a Live AI session. */ private fun talking(): Boolean = diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/NotifyIcons.kt b/app/src/main/kotlin/com/macedonga/visor/notify/NotifyIcons.kt new file mode 100644 index 00000000..b287b200 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/notify/NotifyIcons.kt @@ -0,0 +1,57 @@ +package com.macedonga.visor.notify + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Canvas +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.ByteArrayOutputStream +import javax.inject.Inject +import javax.inject.Provider +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +/** The glasses' svc46 requests: an app's icon for a notification (type 9), and their acks (types 2 and 1027). */ +class NotifyIcons @Inject constructor( + @ApplicationContext private val context: Context, + // A Provider, because EventLog needs the connection this responder is registered on. + private val log: Provider, +) : Responder { + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + when (message.info.messageType) { + MT_ICON_REQUEST -> { + val pkg = runCatching { String(Wire.parse(message.payload).bytes(1)) }.getOrDefault("") + val jpeg = withContext(Dispatchers.Default) { render(pkg) } + val answer = if (jpeg != null) { + Wire.fieldVarint(2, 1) + Wire.fieldBytes(3, jpeg) + } else { + Wire.fieldVarint(2, 0) + Wire.fieldBytes(4, "unknown package".toByteArray()) + } + connection.send(DataXConnection.mirror(message.info.channelId), Wire.fieldBytes(1, pkg.toByteArray()) + answer, messageType = MT_ICON_RESPONSE) + } + MT_RESPONSE, MT_RESPONSE_ACK -> log.get().add("Glasses answered a notification (type ${message.info.messageType})") + } + } + + /** The app's icon as a 78 x 78 JPEG, or null when the package has none. */ + private fun render(pkg: String): ByteArray? = runCatching { + val d = context.packageManager.getApplicationIcon(pkg) + val bmp = Bitmap.createBitmap(ICON, ICON, Bitmap.Config.ARGB_8888) + d.setBounds(0, 0, ICON, ICON) + d.draw(Canvas(bmp)) + ByteArrayOutputStream().also { bmp.compress(Bitmap.CompressFormat.JPEG, 100, it) }.toByteArray() + }.getOrNull() + + private companion object { + const val MT_RESPONSE = 2 + const val MT_ICON_REQUEST = 9 + const val MT_ICON_RESPONSE = 10 + const val MT_RESPONSE_ACK = 1027 + const val ICON = 78 + } +} From b10ed89fbb6f98c7f4efed0174711b536d3be72f Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 10:36:19 +0200 Subject: [PATCH 15/90] Presence: the glasses appearing reach only the pair Visor is for, through the service's reconnect, so a second pair coming into range does not take over the connection --- .../com/macedonga/visor/connection/GlassesPresence.kt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesPresence.kt b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesPresence.kt index bfa6ba07..923159cf 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesPresence.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesPresence.kt @@ -5,8 +5,6 @@ import android.companion.CompanionDeviceManager import android.companion.CompanionDeviceService import android.companion.DevicePresenceEvent import androidx.annotation.RequiresApi -import com.macedonga.visor.core.connection.GlassesLink -import com.macedonga.visor.core.connection.GlassesTarget import com.macedonga.visor.core.identity.IdentityStore import dagger.hilt.android.AndroidEntryPoint import javax.inject.Inject @@ -37,11 +35,11 @@ class GlassesPresence : CompanionDeviceService() { @Suppress("OVERRIDE_DEPRECATION") override fun onDeviceAppeared(address: String) = appeared(address) - /** Starts the service for the stored pair at [address], and for no other glasses. */ + /** Reaches the pair Visor is for when it is the one at [address]; another pair stays as it is. */ private fun appeared(address: String?) { if (address == null) return - val bundle = runBlocking(Dispatchers.IO) { store.all() } - .firstOrNull { it.glassesAddress.equals(address, ignoreCase = true) } ?: return - GlassesService.start(this, GlassesTarget(GlassesLink.Bluetooth(bundle.glassesAddress), bundle.serial)) + val current = runBlocking(Dispatchers.IO) { store.currentSerial()?.let { serial -> store.all().firstOrNull { it.serial == serial } } } ?: return + if (!current.glassesAddress.equals(address, ignoreCase = true)) return + GlassesService.reconnect(this) } } From 2f9c9045a3d1532a6480c8300460032bcf4642c0 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 10:33:49 +0200 Subject: [PATCH 16/90] Wi-Fi networks: the Display's glasses keep Wi-Fi networks on svc47, so a Display's Glasses page lists the saved ones, removes them, and adds one with the phone's current network name prefilled; the glasses check a new network and say whether the password was wrong. --- .../kotlin/com/macedonga/visor/VisorNav.kt | 4 + .../macedonga/visor/di/ConnectionModule.kt | 5 + .../com/macedonga/visor/log/EventLog.kt | 5 + .../macedonga/visor/settings/GlassesPage.kt | 2 + .../visor/settings/WifiNetworksScreen.kt | 155 ++++++++++++++++++ app/src/main/res/values-it/strings_wifi.xml | 15 ++ app/src/main/res/values/strings_wifi.xml | 15 ++ .../visor/core/connection/GlassesWifi.kt | 120 ++++++++++++++ 8 files changed, 321 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/settings/WifiNetworksScreen.kt create mode 100644 app/src/main/res/values-it/strings_wifi.xml create mode 100644 app/src/main/res/values/strings_wifi.xml create mode 100644 core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt index 49198e1e..7ae5ac05 100644 --- a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt +++ b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt @@ -153,6 +153,9 @@ data object PrompterKey : NavKey @Serializable data object GesturesKey : NavKey +@Serializable +data object WifiNetworksKey : NavKey + /** The app's graph: the home, and every screen it opens. */ @Composable fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) { @@ -267,6 +270,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) entry { NotifyAppsScreen() } entry { com.macedonga.visor.display.PrompterScreen() } entry { com.macedonga.visor.display.GesturesScreen() } + entry { com.macedonga.visor.settings.WifiNetworksScreen() } entry { MetaAiScreen( onSettings = { backStack.add(AssistantSettingsKey) }, diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 5f86dbb8..73c91fd3 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -23,6 +23,7 @@ import com.macedonga.visor.core.assistant.Voice import com.macedonga.visor.core.assistant.mcp.McpHub import com.macedonga.visor.core.assistant.mcp.McpStore import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.connection.GlassesWifi import com.macedonga.visor.core.connection.RealSessionHostFactory import com.macedonga.visor.core.connection.SessionHostFactory import com.macedonga.visor.core.connection.TransportFactory @@ -270,4 +271,8 @@ object ConnectionModule { @Singleton fun glassesSettings(connection: GlassesConnection, drive: RlDriveResponder): GlassesSettings = GlassesSettings(connection, drive) + + @Provides + @Singleton + fun glassesWifi(connection: GlassesConnection): GlassesWifi = GlassesWifi(connection) } diff --git a/app/src/main/kotlin/com/macedonga/visor/log/EventLog.kt b/app/src/main/kotlin/com/macedonga/visor/log/EventLog.kt index a27aa138..7fd1abb8 100644 --- a/app/src/main/kotlin/com/macedonga/visor/log/EventLog.kt +++ b/app/src/main/kotlin/com/macedonga/visor/log/EventLog.kt @@ -8,6 +8,7 @@ import com.macedonga.visor.core.assistant.Assistant import com.macedonga.visor.core.assistant.Hearing import com.macedonga.visor.core.assistant.Timers import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.connection.GlassesWifi import com.macedonga.visor.core.connection.GlassesState import com.macedonga.visor.core.media.LiveStream import com.macedonga.visor.core.media.MediaImport @@ -37,6 +38,7 @@ class EventLog @Inject constructor( keeper: LeKeeper, media: MediaImport, settings: GlassesSettings, + wifi: GlassesWifi, live: LiveStream, assistant: Assistant, tools: com.macedonga.visor.core.assistant.mcp.McpHub, @@ -80,6 +82,9 @@ class EventLog @Inject constructor( scope.launch { settings.notes.collect { add("Settings: $it") } } + scope.launch { + wifi.notes.collect { add("Wi-Fi networks: $it") } + } scope.launch { live.notes.collect { add("Live: $it") } } diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt index cb624139..332a9ebe 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt @@ -28,6 +28,7 @@ import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.lifecycle.viewModelScope import androidx.navigation3.runtime.NavKey import com.macedonga.visor.OnboardingKey +import com.macedonga.visor.WifiNetworksKey import com.macedonga.visor.R import com.macedonga.visor.connection.GlassesService import com.macedonga.visor.connection.LinkKind @@ -217,6 +218,7 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( } } }.toTypedArray(), + if (hasDisplay(info?.build)) row { ValueRow(stringResource(R.string.wifi_networks), null, onClick = { open(WifiNetworksKey) }, locked = !ready) } else null, ) } } diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/WifiNetworksScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/WifiNetworksScreen.kt new file mode 100644 index 00000000..7475508e --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/settings/WifiNetworksScreen.kt @@ -0,0 +1,155 @@ +package com.macedonga.visor.settings + +import android.content.Context +import android.net.wifi.WifiManager +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.lifecycle.viewModelScope +import com.macedonga.visor.R +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.connection.GlassesWifi +import com.macedonga.visor.core.session.LinkState +import dagger.hilt.android.lifecycle.HiltViewModel +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch + +enum class WifiResult { SAVED, WRONG_PASSWORD, REMOVED, FAILED } + +@HiltViewModel +class WifiNetworksModel @Inject constructor( + @ApplicationContext private val context: Context, + val connection: GlassesConnection, + private val wifi: GlassesWifi, +) : ViewModel() { + + // Null until the glasses answer. + private val _networks = MutableStateFlow?>(null) + val networks: StateFlow?> = _networks.asStateFlow() + + private val _busy = MutableStateFlow(false) + val busy: StateFlow = _busy.asStateFlow() + + private val _result = MutableStateFlow(null) + val result: StateFlow = _result.asStateFlow() + + /** The network the phone is on, when Android gives its name. */ + fun phoneNetwork(): String? { + val info = context.getSystemService(WifiManager::class.java)?.connectionInfo ?: return null + return info.ssid?.trim('"')?.takeIf { it.isNotBlank() && it != "" } + } + + fun refresh() = task { + _networks.value = wifi.saved() ?: _networks.value + } + + fun save(ssid: String, passphrase: String) = task { + _result.value = when (wifi.save(ssid, passphrase, open = false)) { + 0 -> WifiResult.SAVED + 6 -> WifiResult.WRONG_PASSWORD + else -> WifiResult.FAILED + } + wifi.saved()?.let { _networks.value = it } + } + + fun remove(ssid: String) = task { + _result.value = if (wifi.remove(ssid) == 0) WifiResult.REMOVED else WifiResult.FAILED + wifi.saved()?.let { _networks.value = it } + } + + private fun task(work: suspend () -> Unit) { + _busy.value = true + viewModelScope.launch { + try { + work() + } finally { + _busy.value = false + } + } + } +} + +/** The Wi-Fi networks a Display's glasses keep, with the means to add and remove them. */ +@Composable +fun WifiNetworksScreen(model: WifiNetworksModel = hiltViewModel()) { + val glasses by model.connection.state.collectAsStateWithLifecycle() + val ready = glasses.link == LinkState.READY + val networks by model.networks.collectAsStateWithLifecycle() + val busy by model.busy.collectAsStateWithLifecycle() + val result by model.result.collectAsStateWithLifecycle() + var adding by remember { mutableStateOf(false) } + LaunchedEffect(ready) { if (ready) model.refresh() } + + SettingsPage(stringResource(R.string.wifi_networks_title), light = 0f) { + item(key = "networks") { + Column { + val saved = networks.orEmpty() + Rows( + *saved.map { ssid -> + @Composable { + RowFrame(ssid, onClick = null, locked = !ready) { + ActionPill( + stringResource(R.string.wifi_remove), { model.remove(ssid) }, + quiet = true, modifier = Modifier.padding(start = 12.dp), enabled = ready && !busy, + ) + } + } + }.toTypedArray(), + { ValueRow(stringResource(R.string.wifi_add), null, onClick = { adding = true }, locked = !ready, busy = busy) }, + ) + result?.let { r -> + Footnote( + stringResource( + when (r) { + WifiResult.SAVED -> R.string.wifi_saved + WifiResult.WRONG_PASSWORD -> R.string.wifi_wrong_password + WifiResult.REMOVED -> R.string.wifi_removed + WifiResult.FAILED -> R.string.wifi_failed + }, + ), + ) + } + if (networks != null && saved.isEmpty()) Footnote(stringResource(R.string.wifi_none)) + if (networks == null && ready && !busy) Footnote(stringResource(R.string.wifi_unread)) + } + } + } + + if (adding) { + val phone = remember { model.phoneNetwork().orEmpty() } + var name by remember { mutableStateOf(phone) } + var password by remember { mutableStateOf("") } + VisorSheet({ adding = false }) { + SheetTitle(stringResource(R.string.wifi_add)) + SheetField(name, { name = it }, hint = stringResource(R.string.wifi_network_name), focus = name.isEmpty()) + Spacer(Modifier.height(10.dp)) + SheetField(password, { password = it }, hint = stringResource(R.string.wifi_password), secret = true, focus = name.isNotEmpty()) + Row(Modifier.fillMaxWidth().padding(horizontal = SIDE).padding(top = 16.dp), horizontalArrangement = Arrangement.End) { + ActionPill(stringResource(R.string.settings_save), { + model.save(name.trim(), password) + adding = false + }, enabled = name.isNotBlank() && password.isNotEmpty()) + } + } + } +} diff --git a/app/src/main/res/values-it/strings_wifi.xml b/app/src/main/res/values-it/strings_wifi.xml new file mode 100644 index 00000000..69370622 --- /dev/null +++ b/app/src/main/res/values-it/strings_wifi.xml @@ -0,0 +1,15 @@ + + + Reti Wi-Fi + Reti Wi-Fi + Aggiungi rete + Rimuovi + Nome della rete + Password + Nessuna rete salvata + Non riesco a leggere le reti + Salvata + Password sbagliata + Rimossa + Non riuscito + diff --git a/app/src/main/res/values/strings_wifi.xml b/app/src/main/res/values/strings_wifi.xml new file mode 100644 index 00000000..c6aab527 --- /dev/null +++ b/app/src/main/res/values/strings_wifi.xml @@ -0,0 +1,15 @@ + + + Wi-Fi networks + Wi-Fi networks + Add network + Remove + Network name + Password + No saved networks + Could not read the networks + Saved + Wrong password + Removed + Failed + diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt new file mode 100644 index 00000000..516e6ede --- /dev/null +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt @@ -0,0 +1,120 @@ +package com.macedonga.visor.core.connection + +import com.macedonga.visor.core.protocol.Wire +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.onSubscription +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withTimeoutOrNull + +/** svc47, cloud configuration: the Wi-Fi networks the glasses keep. Its answers come on [GlassesConnection.serviceData]. */ +const val SVC_CLOUD_CONFIG = 47 +private const val MT_REMOVE_NETWORK = 3 +private const val MT_REMOVE_NETWORK_RESULT = 4 +private const val MT_QUERY_NETWORKS = 5 +private const val MT_QUERY_NETWORKS_RESULT = 6 +private const val MT_SAVE_NETWORK_LIST = 18 +private const val MT_SAVE_NETWORK_RESULT = 2 + +private const val SVC_APPS = 28 +private const val MT_START_APP = 2000 +private const val MT_START_APP_RESPONSE = 2001 +private const val APP_CLOUD_CONFIGURATION = 6L + +private const val SECURITY_OPEN = 1L +private const val SECURITY_WPA_PSK = 3L +private const val RANDOMIZATION_PUBLIC = 1L +private const val REQUEST_CONNECT = 2L + +private const val ANSWER_MS = 15_000L + +/** Lists, saves and removes the Wi-Fi networks the glasses keep. One request at a time. */ +class GlassesWifi(private val connection: GlassesConnection) { + + private val lock = Mutex() + + // The glasses' cloud configuration app, started before the first request of a connection. + @Volatile + private var started = false + + private val _notes = MutableSharedFlow(extraBufferCapacity = 64) + + /** What was asked and answered, for the event log. */ + val notes: SharedFlow = _notes.asSharedFlow() + + /** The saved network names, or null without an answer. */ + suspend fun saved(): List? = lock.withLock { + val answer = ask(MT_QUERY_NETWORKS, ByteArray(0), MT_QUERY_NETWORKS_RESULT) + if (answer == null) null else readList(answer) + } + + private fun readList(answer: ByteArray): List? { + val fields = Wire.parse(answer) + if (fields.varint(1) != 0L) { + note("list refused, status ${fields.varint(1)}") + return null + } + return fields.repeatedBytes(2).mapIndexedNotNull { i, entry -> + val network = Wire.parse(entry) + if (i == 0) note("first network has fields ${network.fieldNumbers()}") + if (network.has(1)) network.bytes(1).decodeToString().takeIf { it.isNotBlank() } else null + } + } + + /** Save a WPA2 network and have the glasses check it. Its status, 0 for saved; null without an answer. */ + suspend fun save(ssid: String, passphrase: String, open: Boolean): Int? = lock.withLock { + val network = Wire.fieldBytes(1, ssid.toByteArray()) + + (if (open) ByteArray(0) else Wire.fieldBytes(2, passphrase.toByteArray())) + + Wire.fieldVarint(3, if (open) SECURITY_OPEN else SECURITY_WPA_PSK) + + Wire.fieldVarint(5, 1) + + Wire.fieldVarint(6, RANDOMIZATION_PUBLIC) + + Wire.fieldVarint(9, REQUEST_CONNECT) + val answer = ask(MT_SAVE_NETWORK_LIST, Wire.fieldBytes(1, network), MT_SAVE_NETWORK_RESULT) + answer?.let { Wire.parse(it).varint(1).toInt().also { status -> note("save $ssid: status $status") } } + } + + /** Forget a saved network. Its status, 0 for removed; null without an answer. */ + suspend fun remove(ssid: String): Int? = lock.withLock { + val answer = ask(MT_REMOVE_NETWORK, Wire.fieldBytes(1, ssid.toByteArray()), MT_REMOVE_NETWORK_RESULT) + answer?.let { Wire.parse(it).varint(1).toInt().also { status -> note("remove $ssid: status $status") } } + } + + /** Sends one svc47 message and returns the answer's payload, or null after [ANSWER_MS]. */ + private suspend fun ask(type: Int, payload: ByteArray, answer: Int): ByteArray? { + if (!started && !startApp()) return null + val reply = withTimeoutOrNull(ANSWER_MS) { + connection.serviceData + .onSubscription { connection.submit { it.sendOnService(SVC_CLOUD_CONFIG, type, payload) } } + .first { it.serviceId == SVC_CLOUD_CONFIG && it.messageType == answer } + .payload + } + if (reply == null) { + // The app may have stopped with the link; the next request starts it again. + started = false + note("no answer to type $type") + } + return reply + } + + /** Starts the cloud configuration app on svc28, as the Meta app does first. True when it runs. */ + private suspend fun startApp(): Boolean { + val code = withTimeoutOrNull(ANSWER_MS) { + connection.serviceData + .onSubscription { + connection.submit { it.sendOnService(SVC_APPS, MT_START_APP, Wire.fieldVarint(1, APP_CLOUD_CONFIGURATION)) } + } + .first { it.serviceId == SVC_APPS && it.messageType == MT_START_APP_RESPONSE } + .let { Wire.parse(it.payload).varint(1) } + } + started = code == 1L + if (!started) note("cloud configuration did not start, code ${code ?: "none"}") + return started + } + + private fun note(text: String) { + _notes.tryEmit(text) + } +} From 38dfd2387711bb56414e6ced4c0cfb7556ae5be4 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 10:36:38 +0200 Subject: [PATCH 17/90] Display: the Display's own screen on the phone, as a Mirror card on the Display shelf. Visor sets the glasses' hn_demos.enable_hn_screencast switch if it is off, opens the screencast on svc23464 with the Meta app's encoder defaults, and decodes the H.264 frames from svc23463 into a square black page with Start and Stop. Leaving the page stops the stream; the card says idle or live. --- .../kotlin/com/macedonga/visor/VisorNav.kt | 6 + .../macedonga/visor/di/ConnectionModule.kt | 5 + .../macedonga/visor/display/DisplayShelf.kt | 12 +- .../macedonga/visor/media/MirrorDecoder.kt | 60 ++++++++ .../com/macedonga/visor/media/MirrorScreen.kt | 132 ++++++++++++++++ app/src/main/res/values-it/strings_mirror.xml | 17 ++ app/src/main/res/values/strings_mirror.xml | 17 ++ .../macedonga/visor/core/media/Screencast.kt | 145 ++++++++++++++++++ 8 files changed, 393 insertions(+), 1 deletion(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/media/MirrorDecoder.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/media/MirrorScreen.kt create mode 100644 app/src/main/res/values-it/strings_mirror.xml create mode 100644 app/src/main/res/values/strings_mirror.xml create mode 100644 core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt index 7ae5ac05..da388cf1 100644 --- a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt +++ b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt @@ -37,6 +37,7 @@ import com.macedonga.visor.log.EventLogScreen import com.macedonga.visor.login.NativeLoginPage import com.macedonga.visor.media.LiveScreen import com.macedonga.visor.media.LiveSettingsScreen +import com.macedonga.visor.media.MirrorScreen import com.macedonga.visor.metaai.LiveSessionScreen import com.macedonga.visor.metaai.MemoriesScreen import com.macedonga.visor.metaai.MetaAiScreen @@ -90,6 +91,9 @@ data object MetaAiKey : NavKey @Serializable data object LiveKey : NavKey +@Serializable +data object MirrorKey : NavKey + @Serializable data object FlagsKey : NavKey @@ -214,6 +218,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) when (tool) { com.macedonga.visor.display.DisplayTool.TELEPROMPTER -> backStack.add(PrompterKey) com.macedonga.visor.display.DisplayTool.GESTURES -> backStack.add(GesturesKey) + com.macedonga.visor.display.DisplayTool.MIRROR -> backStack.add(MirrorKey) } }, ) @@ -297,6 +302,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) ) } entry { LiveSettingsScreen() } + entry { MirrorScreen() } entry { com.macedonga.visor.language.LanguageScreen() } entry { FlagsScreen() } entry { ConversationFocusScreen() } diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 73c91fd3..af3850e9 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -32,6 +32,7 @@ import com.macedonga.visor.core.identity.KeystoreSealer import com.macedonga.visor.core.media.AirshipReceiver import com.macedonga.visor.core.media.GlassesConfig import com.macedonga.visor.core.media.MediaImport +import com.macedonga.visor.core.media.Screencast import com.macedonga.visor.core.media.LiveStream import com.macedonga.visor.core.media.LiveOutputs import com.macedonga.visor.core.media.LiveSettingsStore @@ -267,6 +268,10 @@ object ConnectionModule { media: MediaImport, ): GlassesConfig = GlassesConfig(context, connection, airship, media) + @Provides + @Singleton + fun screencast(connection: GlassesConnection, config: GlassesConfig): Screencast = Screencast(connection, config) + @Provides @Singleton fun glassesSettings(connection: GlassesConnection, drive: RlDriveResponder): GlassesSettings = diff --git a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt index 27f4fd19..73d7dae1 100644 --- a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt +++ b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt @@ -35,7 +35,7 @@ import dagger.hilt.android.lifecycle.HiltViewModel import javax.inject.Inject /** What the Display's cards on the home open. */ -enum class DisplayTool { TELEPROMPTER, GESTURES } +enum class DisplayTool { TELEPROMPTER, GESTURES, MIRROR } @HiltViewModel class ShelfModel @Inject constructor( @@ -43,6 +43,7 @@ class ShelfModel @Inject constructor( val prompter: Teleprompter, val band: com.macedonga.visor.band.BandLink, val connection: com.macedonga.visor.core.connection.GlassesConnection, + val screencast: com.macedonga.visor.core.media.Screencast, ) : ViewModel() /** The Display's tools on the home: a card each, with what it holds now. */ @@ -75,6 +76,15 @@ fun DisplayShelf(onOpen: (DisplayTool) -> Unit, modifier: Modifier = Modifier, m onClick = { onOpen(DisplayTool.GESTURES) }, ) } + item(key = "mirror") { + val mirror by model.screencast.state.collectAsStateWithLifecycle() + ToolCard( + VisorIcons.Live, + stringResource(R.string.mirror_title), + if (mirror.phase == com.macedonga.visor.core.media.ScreencastPhase.LIVE) stringResource(R.string.mirror_live) else stringResource(R.string.mirror_idle), + onClick = { onOpen(DisplayTool.MIRROR) }, + ) + } } } diff --git a/app/src/main/kotlin/com/macedonga/visor/media/MirrorDecoder.kt b/app/src/main/kotlin/com/macedonga/visor/media/MirrorDecoder.kt new file mode 100644 index 00000000..962f0b90 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/media/MirrorDecoder.kt @@ -0,0 +1,60 @@ +package com.macedonga.visor.media + +import android.media.MediaCodec +import android.media.MediaFormat +import android.view.Surface +import com.macedonga.visor.core.media.ScreencastFrame + +/** H.264 decoding of the Display's screen onto a Surface. One thread only; starts on a key frame. */ +class MirrorDecoder(private val surface: Surface) { + private var codec: MediaCodec? = null + private var lastPtsUs = -1L + private val info = MediaCodec.BufferInfo() + + fun feed(frame: ScreencastFrame) { + val c = codec ?: (if (frame.key) start() else null) ?: return + try { + val i = c.dequeueInputBuffer(INPUT_WAIT_US) + if (i >= 0) { + c.getInputBuffer(i)!!.apply { clear(); put(frame.data) } + lastPtsUs = maxOf(frame.timestampMs * 1000, lastPtsUs + 1) + c.queueInputBuffer(i, 0, frame.data.size, lastPtsUs, 0) + } + drain(c) + } catch (e: RuntimeException) { + // A codec error: the next key frame starts a new decoder. + release() + } + } + + fun release() { + codec?.let { runCatching { it.stop(); it.release() } } + codec = null + } + + private fun start(): MediaCodec? = try { + MediaCodec.createDecoderByType(MIME).apply { + configure(MediaFormat.createVideoFormat(MIME, MAX_W, MAX_H), surface, null, 0) + setVideoScalingMode(MediaCodec.VIDEO_SCALING_MODE_SCALE_TO_FIT) + start() + }.also { codec = it } + } catch (e: Exception) { + null + } + + private fun drain(c: MediaCodec) { + while (true) { + val o = c.dequeueOutputBuffer(info, 0) + if (o == MediaCodec.INFO_TRY_AGAIN_LATER) return + if (o >= 0) c.releaseOutputBuffer(o, true) + } + } + + private companion object { + const val MIME = "video/avc" + // The size the Display picks is read from the stream's parameter sets; this is only the most it may pick. + const val MAX_W = 1920 + const val MAX_H = 1080 + const val INPUT_WAIT_US = 10_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/media/MirrorScreen.kt b/app/src/main/kotlin/com/macedonga/visor/media/MirrorScreen.kt new file mode 100644 index 00000000..bc54c9f9 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/media/MirrorScreen.kt @@ -0,0 +1,132 @@ +package com.macedonga.visor.media + +import android.view.Surface +import android.view.SurfaceHolder +import android.view.SurfaceView +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.compose.ui.viewinterop.AndroidView +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.lifecycle.viewModelScope +import com.macedonga.visor.R +import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.core.media.Screencast +import com.macedonga.visor.core.media.ScreencastFault +import com.macedonga.visor.core.media.ScreencastPhase +import com.macedonga.visor.core.media.ScreencastState +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.settings.ActionPill +import com.macedonga.visor.settings.SettingsPage +import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.launch + +@HiltViewModel +class MirrorModel @Inject constructor(val screencast: Screencast, private val log: EventLog) : ViewModel() { + // The decoder is touched on this one thread only. + private val decoding = CoroutineScope(SupervisorJob() + Dispatchers.Default.limitedParallelism(1)) + private var decoder: MirrorDecoder? = null + + private val feed = decoding.launch { screencast.frames.collect { decoder?.feed(it) } } + + init { + viewModelScope.launch { + screencast.state.collect { s -> + when (s.phase) { + ScreencastPhase.FAILED -> log.add("Mirror failed: ${s.fault}") + ScreencastPhase.LIVE -> log.add("Mirror live") + else -> Unit + } + } + } + } + + /** The surface the picture goes to, or null when it is gone. */ + fun attach(surface: Surface?) { + decoding.launch { + decoder?.release() + decoder = surface?.let { MirrorDecoder(it) } + } + } + + override fun onCleared() { + decoding.launch { decoder?.release(); decoder = null } + feed.cancel() + super.onCleared() + } +} + +/** The Display's screen, on the phone. Closing the page stops the stream. */ +@Composable +fun MirrorScreen(model: MirrorModel = hiltViewModel()) { + val state by model.screencast.state.collectAsStateWithLifecycle() + val active = state.phase == ScreencastPhase.STARTING || state.phase == ScreencastPhase.LIVE + DisposableEffect(model) { onDispose { model.screencast.stop() } } + + SettingsPage(title = stringResource(R.string.mirror_title)) { + item(key = "picture") { + AndroidView( + modifier = Modifier.fillMaxWidth().aspectRatio(1f).background(Color.Black), + factory = { context -> + SurfaceView(context).apply { + holder.addCallback(object : SurfaceHolder.Callback { + override fun surfaceCreated(holder: SurfaceHolder) = model.attach(holder.surface) + override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) = Unit + override fun surfaceDestroyed(holder: SurfaceHolder) = model.attach(null) + }) + } + }, + ) + } + item(key = "controls") { + Row(Modifier.padding(horizontal = 16.dp, vertical = 12.dp), horizontalArrangement = Arrangement.spacedBy(8.dp)) { + ActionPill(stringResource(R.string.mirror_start), onClick = { model.screencast.start() }, enabled = !active) + ActionPill(stringResource(R.string.mirror_stop), onClick = { model.screencast.stop() }, quiet = true, enabled = active) + } + } + item(key = "state") { + Text( + phraseOf(state), + style = Visor.type.label, + color = Visor.colors.textSecondary, + modifier = Modifier.padding(horizontal = 16.dp), + ) + } + } +} + +@Composable +private fun phraseOf(s: ScreencastState): String = when (s.phase) { + ScreencastPhase.IDLE -> stringResource(R.string.mirror_idle) + ScreencastPhase.STARTING -> stringResource(R.string.mirror_starting) + ScreencastPhase.LIVE -> stringResource(R.string.mirror_live) + ScreencastPhase.FAILED -> when (s.fault) { + ScreencastFault.NOT_CONNECTED -> stringResource(R.string.mirror_not_connected) + ScreencastFault.NOT_READ -> stringResource(R.string.mirror_not_read) + ScreencastFault.NO_FLAG -> stringResource(R.string.mirror_no_flag) + ScreencastFault.FLAG_NOT_SENT -> stringResource(R.string.mirror_flag_not_sent, s.detail) + ScreencastFault.NO_ANSWER -> stringResource(R.string.mirror_no_answer) + ScreencastFault.ASLEEP -> stringResource(R.string.mirror_asleep) + ScreencastFault.GLASSES_TIMEOUT -> stringResource(R.string.mirror_timeout) + ScreencastFault.UNKNOWN, null -> stringResource(R.string.mirror_unknown) + } +} diff --git a/app/src/main/res/values-it/strings_mirror.xml b/app/src/main/res/values-it/strings_mirror.xml new file mode 100644 index 00000000..1f777e90 --- /dev/null +++ b/app/src/main/res/values-it/strings_mirror.xml @@ -0,0 +1,17 @@ + + + Specchio + Avvia + Ferma + Inattivo + Avvio… + In diretta + Connettiti prima agli occhiali. + Leggi prima la configurazione degli occhiali. + Questi occhiali non hanno l\'interruttore dello schermo. + Gli occhiali non hanno accettato l\'interruttore: %1$s + Gli occhiali non hanno risposto. + Lo schermo degli occhiali è spento. + Gli occhiali sono andati in timeout. + Gli occhiali hanno rifiutato lo schermo condiviso. + diff --git a/app/src/main/res/values/strings_mirror.xml b/app/src/main/res/values/strings_mirror.xml new file mode 100644 index 00000000..fdb1f0d7 --- /dev/null +++ b/app/src/main/res/values/strings_mirror.xml @@ -0,0 +1,17 @@ + + + Mirror + Start + Stop + Idle + Starting… + Live + Connect to the glasses first. + Read the glasses\' configuration first. + These glasses have no screencast switch. + The glasses did not take the switch: %1$s + The glasses did not answer. + The glasses\' screen is asleep. + The glasses timed out. + The glasses refused the screencast. + diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt new file mode 100644 index 00000000..0851d08e --- /dev/null +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt @@ -0,0 +1,145 @@ +package com.macedonga.visor.core.media + +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.ServiceData +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull + +const val SVC_HN_SCREENCAST_MANAGER = 23464 +const val SVC_HN_SCREENCAST = 23463 + +// The MobileConfig switch the glasses check before they run the screencast service. +const val SCREENCAST_FLAG = "hn_demos.enable_hn_screencast" + +enum class ScreencastPhase { IDLE, STARTING, LIVE, FAILED } + +enum class ScreencastFault { NOT_CONNECTED, NOT_READ, NO_FLAG, FLAG_NOT_SENT, NO_ANSWER, ASLEEP, GLASSES_TIMEOUT, UNKNOWN } + +data class ScreencastState(val phase: ScreencastPhase = ScreencastPhase.IDLE, val fault: ScreencastFault? = null, val detail: String = "") + +/** One whole H.264 frame, its parameter sets in band. */ +class ScreencastFrame(val timestampMs: Long, val key: Boolean, val data: ByteArray) + +/** The Display's own screen, streamed to the phone on svc23463 and driven on svc23464. */ +class Screencast(private val connection: GlassesConnection, private val config: GlassesConfig) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val _state = MutableStateFlow(ScreencastState()) + val state: StateFlow = _state.asStateFlow() + + private val _frames = MutableSharedFlow(extraBufferCapacity = 8, onBufferOverflow = BufferOverflow.DROP_OLDEST) + val frames: SharedFlow = _frames.asSharedFlow() + + private var job: Job? = null + private var answer: CompletableDeferred>? = null + + // The frame being assembled: its time, key flag and the chunks still owed. + private var timestamp = 0L + private var key = false + private var chunksLeft = 0 + private var frame = ByteArrayOutputStream() + + init { + scope.launch { connection.serviceData.collect { onData(it) } } + } + + /** Switch the glasses' screencast on and wait for their answer; the state says how it went. */ + @Synchronized + fun start() { + if (job?.isActive == true) return + job = scope.launch { run() } + } + + /** Stop the screencast on the glasses and forget the state. */ + @Synchronized + fun stop() { + val live = _state.value.phase == ScreencastPhase.STARTING || _state.value.phase == ScreencastPhase.LIVE + job?.cancel() + job = null + _state.value = ScreencastState() + if (live) connection.submit { it.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONTROL, byteArrayOf(3)) } + } + + private suspend fun run() { + fun fail(fault: ScreencastFault, detail: String = "") { + _state.value = ScreencastState(ScreencastPhase.FAILED, fault, detail) + } + _state.value = ScreencastState(ScreencastPhase.STARTING) + if (connection.state.value.link != LinkState.READY) return fail(ScreencastFault.NOT_CONNECTED) + val cfg = config.state.value + if (cfg.params.isEmpty()) return fail(ScreencastFault.NOT_READ) + if (SCREENCAST_FLAG !in cfg.params) return fail(ScreencastFault.NO_FLAG) + if (cfg.values[SCREENCAST_FLAG] != true) { + config.send(mapOf(SCREENCAST_FLAG to true))?.let { return fail(ScreencastFault.FLAG_NOT_SENT, it) } + } + + val reply = CompletableDeferred>() + answer = reply + connection.submit { it.sendOnService(SVC_HN_SCREENCAST, MT_CONTROL, byteArrayOf(0)) } + connection.submit { it.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONTROL, byteArrayOf(0)) } + connection.submit { it.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONFIGURE, configure()) } + val got = withTimeoutOrNull(ANSWER_MS) { reply.await() } + answer = null + when { + got == null -> fail(ScreencastFault.NO_ANSWER) + got.first -> _state.value = ScreencastState(ScreencastPhase.LIVE) + got.second == 2 -> fail(ScreencastFault.ASLEEP) + got.second == 3 -> fail(ScreencastFault.GLASSES_TIMEOUT) + else -> fail(ScreencastFault.UNKNOWN) + } + } + + private fun onData(m: ServiceData) { + when (m.serviceId) { + SVC_HN_SCREENCAST_MANAGER -> { + val reply = runCatching { Wire.parse(m.payload) }.getOrNull() ?: return + answer?.complete(Pair(reply.varint(1) != 0L, reply.varint(2).toInt())) + } + SVC_HN_SCREENCAST -> chunk(m.payload) + } + } + + // A 32-byte message opens a frame; every other one is the next chunk of the open frame. + private fun chunk(p: ByteArray) { + if (p.size == HEADER_BYTES) { + val b = ByteBuffer.wrap(p).order(ByteOrder.LITTLE_ENDIAN) + timestamp = b.getLong(0) + key = b.getInt(8) and KEY_FLAG != 0 + chunksLeft = b.getInt(24) + frame = ByteArrayOutputStream() + return + } + if (chunksLeft <= 0) return + frame.write(p) + if (--chunksLeft == 0) _frames.tryEmit(ScreencastFrame(timestamp, key, frame.toByteArray())) + } + + // Five little-endian int32s: bitrate, I-frame interval, 2^profile, 2^level, framerate. These are the Meta app's defaults. + private fun configure(): ByteArray = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN) + .putInt(100_000).putInt(1_000_000).putInt(1).putInt(1).putInt(5) + .array() + + private companion object { + const val MT_CONTROL = 129 + const val MT_CONFIGURE = 1 + const val HEADER_BYTES = 32 + const val KEY_FLAG = 1 + const val ANSWER_MS = 15_000L + } +} From 7e091eee0f95d7a5ac719cd60a65e0f2a54199f2 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 10:37:31 +0200 Subject: [PATCH 18/90] Home: take out this branch's own take on a refused key, for #25's --- app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt | 3 --- .../main/kotlin/com/macedonga/visor/home/HomeScreen.kt | 8 +------- app/src/main/res/values-it/strings.xml | 1 - app/src/main/res/values/strings.xml | 1 - .../macedonga/visor/core/connection/GlassesConnection.kt | 4 +--- .../macedonga/visor/core/connection/GlassesSessionHost.kt | 2 -- .../com/macedonga/visor/core/connection/SessionHost.kt | 3 --- 7 files changed, 2 insertions(+), 20 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt index 8b92cbf5..5c19932f 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt @@ -141,13 +141,10 @@ fun StatusPill( // The Display's Neural Band, when Visor has read it. band: Int? = null, bandCharging: Boolean = false, - // The glasses answer and refuse this phone's key: another phone has set them up since. - refused: Boolean = false, ) { when { !allowed -> return Pill(stringResource(R.string.home_allow_bluetooth), onAllow, modifier) paired == false -> return Pill(stringResource(R.string.home_set_up), onSetUp, modifier) - refused -> return Pill(stringResource(R.string.home_set_up_again), onSetUp, modifier) } val colors = Visor.colors val away = mood == Mood.FAR || mood == Mood.ABSENT || mood == Mood.SEARCHING || mood == Mood.ASLEEP || mood == Mood.IN_CASE diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt index 137819cf..e7e8e5d6 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt @@ -357,12 +357,6 @@ fun HomeScreen( if (state.link == LinkState.READY) unanswered = false else if (state.failure != null || state.link == LinkState.LOST) unanswered = true } - // Every attempt clears the verdict, so a refused key is kept until the glasses are back. - var refused by remember { mutableStateOf(false) } - LaunchedEffect(state.link, state.diagnosis) { - if (state.link == LinkState.READY) refused = false - else if (state.diagnosis == com.macedonga.visor.core.identity.Diagnosis.EVICTED) refused = true - } // The Live view pauses Bluetooth too; until it is back the glasses are as present as they were on Wi-Fi. val backFromLive = remember { stage.fromLive.also { stage.fromLive = false } } var liveSettling by remember { mutableStateOf(backFromLive) } @@ -587,7 +581,7 @@ fun HomeScreen( state, mood, lastBattery, paired, allowed, transferring, importProgress, waiting = if (importing) 0 else fresh.size, onImport = onImport, onAllow = { ask.launch(wanted()) }, onSetUp = onSetUp, modifier = Modifier, band = state.device?.bandBattery ?: ours?.battery, - bandCharging = state.device?.bandCharging ?: (ours?.charging == true), refused = refused, + bandCharging = state.device?.bandCharging ?: (ours?.charging == true), ) } if (pairs.isNotEmpty()) PairDots(pairs.size + 1, shownIndex, Modifier.padding(top = 12.dp)) diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index f54350f8..b9d140a4 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -950,7 +950,6 @@ La batteria degli occhiali è troppo scarica Gli occhiali sono troppo caldi Gli occhiali non hanno abbastanza energia - Configura di nuovo Gesti Impara i gesti Bracciale %1$d%% diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index cf6c7e8e..b5dfd143 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -952,7 +952,6 @@ The glasses\' battery is too low The glasses are too warm The glasses are short of power - Set up again Gestures Learn the gestures Band %1$d%% diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt index 9e54fbce..ff6b1eea 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt @@ -267,9 +267,7 @@ class GlassesConnection( deadline = null val trustOk = host.trustComplete - // A refusal is an answer too: the Display sends its own credential, turns ours down and never ends link - // setup, so svc100 is never asked. - val answered = canaryAnswered || host.trustRefused + val answered = canaryAnswered if (bundle == null) { // Bring-up against a peer with no stored identity. There is a verdict to publish // and no bundle whose state could move. diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesSessionHost.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesSessionHost.kt index 4eadbf51..84ce8101 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesSessionHost.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesSessionHost.kt @@ -20,8 +20,6 @@ class GlassesSessionHost(private val session: GlassesSession) : SessionHost { override val trustComplete: Boolean get() = session.trustResult?.complete == true - override val trustRefused: Boolean get() = session.trustResult?.let { it.rxOk && !it.txOk } == true - override fun start(scope: CoroutineScope) = session.start(scope) override suspend fun stop() = session.stop() diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/SessionHost.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/SessionHost.kt index 63424fd5..24e254bb 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/SessionHost.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/SessionHost.kt @@ -15,9 +15,6 @@ interface SessionHost { /** True once the svc36 exchange has completed in both directions on this attempt. */ val trustComplete: Boolean - /** The glasses sent their own credential and turned this phone's down: they are there, and refuse the key. */ - val trustRefused: Boolean get() = false - fun start(scope: CoroutineScope) /** Finish, and do not return until the socket has been let go of. */ From 74cddf55497ca7c1c299250df1eb310dd9f3107c Mon Sep 17 00:00:00 2001 From: vsiioky Date: Tue, 20 Oct 2026 00:22:48 +0500 Subject: [PATCH 19/90] Trust: glasses that refuse the key are EVICTED before the link is up Glasses that no longer know this phone's key still send their own svc36 credential, refuse ours and then withhold EndLinkSetup. The link never reaches READY, the canary is never asked, and every attempt ended as LINK_DOWN: "nothing answered", "The glasses are not answering". Their credential arriving is an answer, so a refused key with it is now EVICTED. The verdict is kept through the reconnect ladder until a link is READY, so the notification keeps saying the key is refused. - The trust result is reset at each attempt, so a verdict never reads the last attempt's exchange. - The bundle state is compared with what is stored, not start()'s copy: a refused key is judged on every retry, and a pair forgotten in the middle of one must not be saved back. - The canary deadline no longer cancels itself through judge(), which dropped a verdict that had a state to save. - The glasses' refusal status goes into the event log. --- .../core/connection/GlassesConnection.kt | 26 ++++++++++++++----- .../core/connection/GlassesSessionHost.kt | 2 ++ .../visor/core/connection/SessionHost.kt | 3 +++ .../visor/core/identity/Diagnosis.kt | 10 +++++-- .../macedonga/visor/core/identity/Trust.kt | 5 ++-- .../macedonga/visor/core/identity/Verify.kt | 7 ++++- .../visor/core/session/GlassesSession.kt | 5 +++- .../visor/core/session/TrustExchange.kt | 9 +++++-- 8 files changed, 53 insertions(+), 14 deletions(-) diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt index ff6b1eea..08666b4b 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt @@ -1,5 +1,6 @@ package com.macedonga.visor.core.connection +import com.macedonga.visor.core.identity.Diagnosis import com.macedonga.visor.core.identity.IdentityBundle import com.macedonga.visor.core.identity.IdentityStore import com.macedonga.visor.core.identity.SessionProbe @@ -178,6 +179,8 @@ class GlassesConnection( // reconnect, so that is where the slate is cleared. reason = if (fresh) "" else event.reason, failure = if (fresh) null else _state.value.failure, + // A refused key outlives the retries (newAttempt keeps it); a link that comes up is judged afresh. + diagnosis = if (event.state == LinkState.READY) null else _state.value.diagnosis, ) when (event.state) { LinkState.READY -> onReady(host, bundle, scope) @@ -236,7 +239,12 @@ class GlassesConnection( judged.set(false) deadline?.cancel() deadline = null - _state.value = _state.value.copy(diagnosis = null, device = null, info = null) + // A refused key stays said through the reconnect ladder: every try ends the same way until READY. + _state.value = _state.value.copy( + diagnosis = _state.value.diagnosis.takeIf { it == Diagnosis.EVICTED }, + device = null, + info = null, + ) } /** Ask svc100, then start the clock on the answer. */ @@ -254,6 +262,8 @@ class GlassesConnection( } deadline = scope.launch { delay(CANARY_TIMEOUT_MILLIS) + // judge cancels the deadline, and this is it: let go first, or its save is cancelled under it. + deadline = null judge(host, bundle) } } @@ -268,22 +278,26 @@ class GlassesConnection( val trustOk = host.trustComplete val answered = canaryAnswered + val theirs = host.credentialReceived if (bundle == null) { // Bring-up against a peer with no stored identity. There is a verdict to publish // and no bundle whose state could move. - _state.value = _state.value.copy(diagnosis = diagnose(trustOk, answered)) + _state.value = _state.value.copy(diagnosis = diagnose(trustOk, answered, theirs)) return } val checked = verifyBundle(bundle) { object : SessionProbe { override suspend fun runTrust(): Boolean = trustOk override suspend fun readCanary(): Boolean = answered + override suspend fun credentialReceived(): Boolean = theirs } } - if (checked.bundle.state != bundle.state) { - // A file write, off the caller's dispatcher for the same reason the store reads - // in start() are: see the class doc. - withContext(ioDispatcher) { store.save(checked.bundle) } + // A file write, off the caller's dispatcher for the same reason the store reads + // in start() are: see the class doc. Against what is stored now, not start()'s copy: a + // refused key is judged on every retry, and a pair forgotten meanwhile must stay gone. + withContext(ioDispatcher) { + val stored = store.load(bundle.serial) ?: return@withContext + if (stored.state != checked.bundle.state) store.save(stored.withState(checked.bundle.state)) } _state.value = _state.value.copy(diagnosis = checked.diagnosis) } diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesSessionHost.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesSessionHost.kt index 84ce8101..110ab588 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesSessionHost.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesSessionHost.kt @@ -20,6 +20,8 @@ class GlassesSessionHost(private val session: GlassesSession) : SessionHost { override val trustComplete: Boolean get() = session.trustResult?.complete == true + override val credentialReceived: Boolean get() = session.trustResult?.rxOk == true + override fun start(scope: CoroutineScope) = session.start(scope) override suspend fun stop() = session.stop() diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/SessionHost.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/SessionHost.kt index 24e254bb..d6c2df43 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/SessionHost.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/SessionHost.kt @@ -15,6 +15,9 @@ interface SessionHost { /** True once the svc36 exchange has completed in both directions on this attempt. */ val trustComplete: Boolean + /** True once the glasses have sent their own svc36 credential on this attempt, even if they refused ours. */ + val credentialReceived: Boolean + fun start(scope: CoroutineScope) /** Finish, and do not return until the socket has been let go of. */ diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/Diagnosis.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/Diagnosis.kt index 53c55208..73775580 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/Diagnosis.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/Diagnosis.kt @@ -5,7 +5,10 @@ enum class Diagnosis { /** Trust completed and the canary answered. Nothing is wrong. */ HEALTHY, - /** The canary answered and trust did not. The glasses no longer know this key. */ + /** + * The glasses answered, the canary or their own credential, and did not accept ours. They no longer know + * this key. + */ EVICTED, /** Nothing answered. This says nothing about the key, because nothing was asked. */ @@ -16,8 +19,11 @@ enum class Diagnosis { } /** `diagnose`. */ -fun diagnose(trustOk: Boolean, canaryAnswered: Boolean): Diagnosis { +fun diagnose(trustOk: Boolean, canaryAnswered: Boolean, credentialReceived: Boolean = false): Diagnosis { if (!canaryAnswered) { + // Glasses that refuse the key also withhold their EndLinkSetup, so the canary is never asked. Their own + // credential arriving is the answer then: identity is working, and ours is what it turned down. + if (!trustOk && credentialReceived) return Diagnosis.EVICTED // Nothing below identity is working, so nothing can be concluded about the key. return if (!trustOk) Diagnosis.LINK_DOWN else Diagnosis.UNKNOWN } diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/Trust.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/Trust.kt index a7ae3098..a2e79800 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/Trust.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/Trust.kt @@ -107,6 +107,7 @@ class IdentityTrustExchange internal constructor( var txOk = false var rxOk = false + var refusal: Int? = null // Anything this loop does not consume belongs to the caller. val deferred = mutableListOf() try { @@ -117,7 +118,7 @@ class IdentityTrustExchange internal constructor( when { info.flags.hasError && info.channelId == ourStatusChannel -> { // Consumed either way, as the reference's if/elif chain does. - if (info.errorId == IdentityStatus.SUCCESS.value) txOk = true + if (info.errorId == IdentityStatus.SUCCESS.value) txOk = true else refusal = info.errorId } (info.channelId and 0x8000) != 0 && @@ -138,7 +139,7 @@ class IdentityTrustExchange internal constructor( connection.pushback(deferred) } - return TrustResult(txOk = txOk, rxOk = rxOk) + return TrustResult(txOk = txOk, rxOk = rxOk, refusal = refusal) } } diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/Verify.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/Verify.kt index 36a88e46..1df5e6cb 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/Verify.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/Verify.kt @@ -10,6 +10,9 @@ interface SessionProbe { /** Ask svc100, `peer_build_info`, and say whether anything came back. */ suspend fun readCanary(): Boolean + + /** Whether the glasses sent their own svc36 credential, whatever they made of ours. */ + suspend fun credentialReceived(): Boolean = false } class VerificationResult( @@ -26,10 +29,12 @@ suspend fun verifyBundle( ): VerificationResult { val trustOk: Boolean val canaryAnswered: Boolean + val credentialReceived: Boolean try { val probe = openSession() trustOk = probe.runTrust() canaryAnswered = probe.readCanary() + credentialReceived = probe.credentialReceived() } catch (cancellation: CancellationException) { // The caller gave up, which is not a verdict about anything. throw cancellation @@ -39,7 +44,7 @@ suspend fun verifyBundle( return VerificationResult(Diagnosis.LINK_DOWN, false, false, bundle) } - val verdict = diagnose(trustOk = trustOk, canaryAnswered = canaryAnswered) + val verdict = diagnose(trustOk = trustOk, canaryAnswered = canaryAnswered, credentialReceived = credentialReceived) return VerificationResult( diagnosis = verdict, trustOk = trustOk, diff --git a/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt b/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt index 4fe2eeea..083c7c0c 100644 --- a/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt +++ b/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt @@ -128,6 +128,8 @@ class GlassesSession( /** One connection attempt. */ private suspend fun runOnce(): Boolean { var reachedReady = false + // Per attempt: a verdict read after this one fails must not see the last one's exchange. + trustResult = null publish(LinkStateChanged(LinkState.CONNECTING)) try { transport.connect() @@ -341,8 +343,9 @@ class GlassesSession( /** Say which direction of the trust exchange failed. */ private fun trustFault(result: TrustResult): String { + val status = result.refusal?.let { " (status 0x%04x)".format(it) }.orEmpty() val faults = buildList { - if (!result.txOk) add("the glasses did not accept our credential") + if (!result.txOk) add("the glasses did not accept our credential$status") if (!result.rxOk) add("the glasses sent no credential of their own") } return "trust incomplete: ${faults.joinToString(" and ")} " + diff --git a/core/session/src/main/kotlin/com/macedonga/visor/core/session/TrustExchange.kt b/core/session/src/main/kotlin/com/macedonga/visor/core/session/TrustExchange.kt index 43bcef8d..a9a485b2 100644 --- a/core/session/src/main/kotlin/com/macedonga/visor/core/session/TrustExchange.kt +++ b/core/session/src/main/kotlin/com/macedonga/visor/core/session/TrustExchange.kt @@ -3,11 +3,16 @@ package com.macedonga.visor.core.session import com.macedonga.visor.core.protocol.HandshakeBinding /** Which direction of the svc36 trust exchange succeeded. From `identity/trust.py`. */ -class TrustResult(val txOk: Boolean, val rxOk: Boolean) { +class TrustResult( + val txOk: Boolean, + val rxOk: Boolean, + /** The status the glasses answered our credential with, when it was not SUCCESS. Null when they said nothing. */ + val refusal: Int? = null, +) { val complete: Boolean get() = txOk && rxOk - override fun toString(): String = "TrustResult(txOk=$txOk, rxOk=$rxOk)" + override fun toString(): String = "TrustResult(txOk=$txOk, rxOk=$rxOk, refusal=$refusal)" } /** The svc36 exchange the session runs between the handshake and EndLinkSetup. */ From 832d06a8388600a69597d06c01f5b9e043fecdc2 Mon Sep 17 00:00:00 2001 From: vsiioky Date: Tue, 20 Oct 2026 00:22:48 +0500 Subject: [PATCH 20/90] Home: a refused key says so and opens Change owner, not "Asleep" Glasses nearby that keep refusing the key looked asleep on the home, the same pill as a pair folded on the table, with nothing to do about it. While the verdict is EVICTED and the link is not up, the pill on the home and in Settings reads "Key refused. Set them up again", and a tap opens the login for Change owner. --- app/src/main/kotlin/com/macedonga/visor/VisorNav.kt | 1 + app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt | 6 ++++++ app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt | 3 ++- .../kotlin/com/macedonga/visor/settings/SettingsScreen.kt | 2 +- app/src/main/res/values-it/strings.xml | 1 + app/src/main/res/values/strings.xml | 1 + 6 files changed, 12 insertions(+), 2 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt index da388cf1..70bba47d 100644 --- a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt +++ b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt @@ -207,6 +207,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) entry { HomeScreen( onSetUp = { backStack.add(OnboardingKey()) }, + onChangeOwner = { backStack.add(OnboardingKey(login = true)) }, onMedia = { backStack.add(MediaKey) }, onLive = { stage.homeToTop = true diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt index 5c19932f..f7a96aad 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt @@ -67,8 +67,10 @@ import com.macedonga.visor.core.identity.language.GlassesLook import com.macedonga.visor.core.connection.CaseBattery import com.macedonga.visor.core.connection.GlassesState import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.core.identity.Diagnosis import com.macedonga.visor.core.media.Capture import com.macedonga.visor.core.media.Transfer +import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.stage.LocalStage import com.macedonga.visor.stage.Mood import com.macedonga.visor.ui.VisorIcons @@ -137,6 +139,7 @@ fun StatusPill( onImport: () -> Unit, onAllow: () -> Unit, onSetUp: () -> Unit, + onChangeOwner: () -> Unit, modifier: Modifier, // The Display's Neural Band, when Visor has read it. band: Int? = null, @@ -145,6 +148,9 @@ fun StatusPill( when { !allowed -> return Pill(stringResource(R.string.home_allow_bluetooth), onAllow, modifier) paired == false -> return Pill(stringResource(R.string.home_set_up), onSetUp, modifier) + // Nearby and refusing the key would otherwise look asleep, with nothing to do about it. + state.diagnosis == Diagnosis.EVICTED && state.link != LinkState.READY -> + return Pill(stringResource(R.string.home_key_refused), onChangeOwner, modifier) } val colors = Visor.colors val away = mood == Mood.FAR || mood == Mood.ABSENT || mood == Mood.SEARCHING || mood == Mood.ASLEEP || mood == Mood.IN_CASE diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt index e7e8e5d6..210a3792 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt @@ -213,6 +213,7 @@ suspend fun ownerFirstName(context: Context): String? = withContext(Dispatchers. @Composable fun HomeScreen( onSetUp: () -> Unit, + onChangeOwner: () -> Unit, onMedia: () -> Unit, onLive: () -> Unit, onMetaAi: () -> Unit, @@ -579,7 +580,7 @@ fun HomeScreen( val ours = band.takeIf { b -> b.battery != null && bands.any { it.serial == b.serial && it.glasses == current } } StatusPill( state, mood, lastBattery, paired, allowed, transferring, importProgress, waiting = if (importing) 0 else fresh.size, onImport = onImport, - onAllow = { ask.launch(wanted()) }, onSetUp = onSetUp, modifier = Modifier, + onAllow = { ask.launch(wanted()) }, onSetUp = onSetUp, onChangeOwner = onChangeOwner, modifier = Modifier, band = state.device?.bandBattery ?: ours?.battery, bandCharging = state.device?.bandCharging ?: (ours?.charging == true), ) diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt index bbd64b95..ae7248b7 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt @@ -295,7 +295,7 @@ fun SettingsScreen(open: (NavKey) -> Unit, model: SettingsViewModel = hiltViewMo val remembered by model.lastSeen.battery.collectAsStateWithLifecycle() StatusPill( glasses, mood, remembered, paired = glasses.target?.let { true }, allowed = true, importing = false, importProgress = 0f, waiting = 0, - onImport = {}, onAllow = {}, onSetUp = { open(OnboardingKey()) }, + onImport = {}, onAllow = {}, onSetUp = { open(OnboardingKey()) }, onChangeOwner = { open(OnboardingKey(login = true)) }, modifier = Modifier.align(Alignment.BottomCenter).padding(bottom = 26.dp).graphicsLayer { alpha = (1f - (scrollFold * 2.5f).coerceIn(0f, 1f)) * stage.shown(spot) }, ) } diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index b9d140a4..b283787b 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -19,6 +19,7 @@ Custodia Consenti il Bluetooth Configura gli occhiali + Chiave rifiutata. Configurali di nuovo %d foto o video nuovo %d foto e video nuovi diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index b5dfd143..7764de01 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -20,6 +20,7 @@ Case Allow Bluetooth Set up glasses + Key refused. Set them up again %d new photo or video %d new photos and videos From da47387bad720fda7ef5fe515c75e68d9eb1b7bf Mon Sep 17 00:00:00 2001 From: vsiioky Date: Tue, 20 Oct 2026 13:36:31 +0500 Subject: [PATCH 21/90] Home: a refused key with the Meta AI app installed names it In the field case the Meta AI app had taken the glasses back, and it takes them again after Change owner unless it is gone first: it is relaunched as the glasses' companion device service and reconnects. With it installed the refused pill now says Meta AI may have taken them, and opens a sheet with both steps in order: uninstall it (the system's app page), then set the glasses up again. --- .../com/macedonga/visor/home/HomeFeed.kt | 52 ++++++++++++++++++- .../macedonga/visor/settings/PhoneAccess.kt | 4 +- app/src/main/res/values-it/strings.xml | 5 ++ app/src/main/res/values/strings.xml | 5 ++ 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt index f7a96aad..672fe27d 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt @@ -1,5 +1,7 @@ package com.macedonga.visor.home +import android.content.Context +import android.content.pm.PackageManager import android.graphics.ImageDecoder import android.text.format.DateUtils import androidx.compose.animation.core.Animatable @@ -38,6 +40,7 @@ import kotlinx.coroutines.flow.first import androidx.compose.runtime.snapshotFlow import androidx.compose.runtime.setValue import androidx.compose.runtime.mutableIntStateOf +import androidx.compose.runtime.mutableStateOf import androidx.compose.ui.unit.Dp import androidx.compose.ui.draw.clipToBounds import androidx.compose.ui.draw.blur @@ -62,6 +65,7 @@ import androidx.compose.ui.res.pluralStringResource import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp +import androidx.lifecycle.compose.LifecycleResumeEffect import com.macedonga.visor.R import com.macedonga.visor.core.identity.language.GlassesLook import com.macedonga.visor.core.connection.CaseBattery @@ -71,6 +75,11 @@ import com.macedonga.visor.core.identity.Diagnosis import com.macedonga.visor.core.media.Capture import com.macedonga.visor.core.media.Transfer import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.settings.ActionPill +import com.macedonga.visor.settings.SIDE +import com.macedonga.visor.settings.SheetTitle +import com.macedonga.visor.settings.VisorSheet +import com.macedonga.visor.settings.openAppPage import com.macedonga.visor.stage.LocalStage import com.macedonga.visor.stage.Mood import com.macedonga.visor.ui.VisorIcons @@ -150,7 +159,7 @@ fun StatusPill( paired == false -> return Pill(stringResource(R.string.home_set_up), onSetUp, modifier) // Nearby and refusing the key would otherwise look asleep, with nothing to do about it. state.diagnosis == Diagnosis.EVICTED && state.link != LinkState.READY -> - return Pill(stringResource(R.string.home_key_refused), onChangeOwner, modifier) + return RefusedPill(onChangeOwner, modifier) } val colors = Visor.colors val away = mood == Mood.FAR || mood == Mood.ABSENT || mood == Mood.SEARCHING || mood == Mood.ASLEEP || mood == Mood.IN_CASE @@ -639,6 +648,47 @@ fun Pill(text: String, onClick: () -> Unit, modifier: Modifier = Modifier) { } } +/** The Meta AI app, the other app the glasses take as their owner. */ +private const val META_AI_PACKAGE = "com.facebook.stella" + +private fun metaAiInstalled(context: Context): Boolean = try { + context.packageManager.getPackageInfo(META_AI_PACKAGE, 0) + true +} catch (_: PackageManager.NameNotFoundException) { + false +} + +/** + * A refused key. With the Meta AI app on the phone that is the likely why: it took the glasses back, and it would take + * them again after setup unless it goes first. So the pill names it, and its sheet offers both steps in order. + */ +@Composable +private fun RefusedPill(onChangeOwner: () -> Unit, modifier: Modifier) { + val context = LocalContext.current + var metaAi by remember { mutableStateOf(metaAiInstalled(context)) } + // Checked again on the way back, since uninstalling it happens in the system's own page. + LifecycleResumeEffect(Unit) { + metaAi = metaAiInstalled(context) + onPauseOrDispose { } + } + // Closed again whenever Meta AI comes or goes, so a reinstall never opens it by itself. + var explaining by remember(metaAi) { mutableStateOf(false) } + if (!metaAi) return Pill(stringResource(R.string.home_key_refused), onChangeOwner, modifier) + Pill(stringResource(R.string.home_key_refused_meta_ai), { explaining = true }, modifier) + if (explaining) { + VisorSheet({ explaining = false }) { + SheetTitle(stringResource(R.string.key_refused_meta_ai_title), stringResource(R.string.key_refused_meta_ai_text)) + Row(Modifier.padding(horizontal = SIDE), horizontalArrangement = Arrangement.spacedBy(10.dp)) { + ActionPill(stringResource(R.string.key_refused_uninstall_meta_ai), { openAppPage(context, META_AI_PACKAGE) }) + ActionPill(stringResource(R.string.key_refused_set_up_again), { + explaining = false + onChangeOwner() + }, quiet = true) + } + } + } +} + private fun Modifier.matchParentSizeCompat(): Modifier = fillMaxWidth().fillMaxHeight() fun moodWords(mood: Mood): Int = when (mood) { diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt b/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt index 06518a47..46970bb3 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt @@ -142,8 +142,8 @@ fun PhoneAccessScreen(model: PhoneAccessModel = hiltViewModel()) { } } -internal fun openAppPage(context: Context) { - context.startActivity(Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS, Uri.fromParts("package", context.packageName, null)).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)) +internal fun openAppPage(context: Context, packageName: String = context.packageName) { + context.startActivity(Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS, Uri.fromParts("package", packageName, null)).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)) } internal fun openListenerPage(context: Context) { diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index b283787b..7325f116 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -20,6 +20,11 @@ Consenti il Bluetooth Configura gli occhiali Chiave rifiutata. Configurali di nuovo + Chiave rifiutata. Forse li ha presi l\'app Meta AI + Forse l\'app Meta AI si è ripresa gli occhiali + Gli occhiali accettano un solo proprietario alla volta. L\'app Meta AI su questo telefono potrebbe essere tornata a esserlo, quindi non accettano più la chiave di Visor. Se è così, prima disinstallala, altrimenti li riprende, poi configura di nuovo gli occhiali. + Disinstalla Meta AI + Configura di nuovo %d foto o video nuovo %d foto e video nuovi diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 7764de01..890fbc99 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -21,6 +21,11 @@ Allow Bluetooth Set up glasses Key refused. Set them up again + Key refused. Meta AI may have taken them + Meta AI may have taken the glasses back + Glasses take one owner app at a time. The Meta AI app on this phone may have made itself their owner again, so they no longer accept Visor\'s key. If so, uninstall it first, or it takes them again, then set the glasses up again. + Uninstall Meta AI + Set up again %d new photo or video %d new photos and videos From 144f3a87b5029f91179e921d9d11fbe081045f65 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 10:49:28 +0200 Subject: [PATCH 22/90] Refused key: glasses that turn the key down and still end link setup are not ready (they refuse every service), so the verdict holds instead of the home saying Ready; the refused message is the status pill's capsule with the error dot; and setting the pair up again takes those glasses as soon as they are heard in pairing mode, with nothing to pick --- .../kotlin/com/macedonga/visor/VisorNav.kt | 5 ++-- .../com/macedonga/visor/home/HomeFeed.kt | 25 ++++++++++++++++--- .../com/macedonga/visor/home/HomeScreen.kt | 2 +- .../macedonga/visor/settings/GlassesPage.kt | 2 +- .../visor/settings/SettingsScreen.kt | 2 +- .../com/macedonga/visor/setup/Onboarding.kt | 14 ++++++++--- .../core/connection/GlassesConnection.kt | 18 ++++++++----- 7 files changed, 50 insertions(+), 18 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt index 70bba47d..6c4f71f9 100644 --- a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt +++ b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt @@ -68,7 +68,7 @@ data object DeviceKey : NavKey /** The setup: from the welcome on the first run, otherwise straight to what is left to do. */ @Serializable -data class OnboardingKey(val welcome: Boolean = false, val login: Boolean = false) : NavKey +data class OnboardingKey(val welcome: Boolean = false, val login: Boolean = false, val address: String? = null) : NavKey @Serializable data object MetaLoginPageKey : NavKey @@ -207,7 +207,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) entry { HomeScreen( onSetUp = { backStack.add(OnboardingKey()) }, - onChangeOwner = { backStack.add(OnboardingKey(login = true)) }, + onChangeOwner = { backStack.add(OnboardingKey(login = true, address = it)) }, onMedia = { backStack.add(MediaKey) }, onLive = { stage.homeToTop = true @@ -238,6 +238,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) OnboardingScreen( welcome = key.welcome, login = key.login, + address = key.address, onLogin = { backStack.add(MetaLoginPageKey) }, onEventLog = { backStack.add(EventLogKey) }, onDone = { target -> diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt index 672fe27d..5201f826 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt @@ -148,7 +148,7 @@ fun StatusPill( onImport: () -> Unit, onAllow: () -> Unit, onSetUp: () -> Unit, - onChangeOwner: () -> Unit, + onChangeOwner: (address: String?) -> Unit, modifier: Modifier, // The Display's Neural Band, when Visor has read it. band: Int? = null, @@ -159,7 +159,7 @@ fun StatusPill( paired == false -> return Pill(stringResource(R.string.home_set_up), onSetUp, modifier) // Nearby and refusing the key would otherwise look asleep, with nothing to do about it. state.diagnosis == Diagnosis.EVICTED && state.link != LinkState.READY -> - return RefusedPill(onChangeOwner, modifier) + return RefusedPill({ onChangeOwner((state.target?.link as? com.macedonga.visor.core.connection.GlassesLink.Bluetooth)?.address) }, modifier) } val colors = Visor.colors val away = mood == Mood.FAR || mood == Mood.ABSENT || mood == Mood.SEARCHING || mood == Mood.ASLEEP || mood == Mood.IN_CASE @@ -673,8 +673,8 @@ private fun RefusedPill(onChangeOwner: () -> Unit, modifier: Modifier) { } // Closed again whenever Meta AI comes or goes, so a reinstall never opens it by itself. var explaining by remember(metaAi) { mutableStateOf(false) } - if (!metaAi) return Pill(stringResource(R.string.home_key_refused), onChangeOwner, modifier) - Pill(stringResource(R.string.home_key_refused_meta_ai), { explaining = true }, modifier) + if (!metaAi) return RefusedCapsule(stringResource(R.string.home_key_refused), onChangeOwner, modifier) + RefusedCapsule(stringResource(R.string.home_key_refused_meta_ai), { explaining = true }, modifier) if (explaining) { VisorSheet({ explaining = false }) { SheetTitle(stringResource(R.string.key_refused_meta_ai_title), stringResource(R.string.key_refused_meta_ai_text)) @@ -689,6 +689,23 @@ private fun RefusedPill(onChangeOwner: () -> Unit, modifier: Modifier) { } } +/** The status pill's capsule, saying what is wrong with the signal dot an error takes, and opening what fixes it. */ +@Composable +private fun RefusedCapsule(text: String, onClick: () -> Unit, modifier: Modifier) { + Row( + modifier + .clip(RoundedCornerShape(50)) + .background(Visor.colors.surface) + .clickable(onClick = onClick) + .padding(horizontal = 16.dp, vertical = 9.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Box(Modifier.size(7.dp).clip(RoundedCornerShape(50)).background(Visor.colors.signal)) + Spacer(Modifier.size(8.dp)) + Text(text, style = Visor.type.label, color = Visor.colors.textPrimary) + } +} + private fun Modifier.matchParentSizeCompat(): Modifier = fillMaxWidth().fillMaxHeight() fun moodWords(mood: Mood): Int = when (mood) { diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt index 210a3792..b9b8648b 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt @@ -213,7 +213,7 @@ suspend fun ownerFirstName(context: Context): String? = withContext(Dispatchers. @Composable fun HomeScreen( onSetUp: () -> Unit, - onChangeOwner: () -> Unit, + onChangeOwner: (address: String?) -> Unit, onMedia: () -> Unit, onLive: () -> Unit, onMetaAi: () -> Unit, diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt index 332a9ebe..7f0e3efd 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt @@ -236,7 +236,7 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( } } }, - { ValueRow(stringResource(R.string.glasses_change_owner), null, onClick = { open(OnboardingKey(login = true)) }, sub = stringResource(R.string.glasses_change_owner_sub)) }, + { ValueRow(stringResource(R.string.glasses_change_owner), null, onClick = { open(OnboardingKey(login = true, address = (glasses.target?.link as? GlassesLink.Bluetooth)?.address)) }, sub = stringResource(R.string.glasses_change_owner_sub)) }, ) } } diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt index ae7248b7..e3bef08e 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt @@ -295,7 +295,7 @@ fun SettingsScreen(open: (NavKey) -> Unit, model: SettingsViewModel = hiltViewMo val remembered by model.lastSeen.battery.collectAsStateWithLifecycle() StatusPill( glasses, mood, remembered, paired = glasses.target?.let { true }, allowed = true, importing = false, importProgress = 0f, waiting = 0, - onImport = {}, onAllow = {}, onSetUp = { open(OnboardingKey()) }, onChangeOwner = { open(OnboardingKey(login = true)) }, + onImport = {}, onAllow = {}, onSetUp = { open(OnboardingKey()) }, onChangeOwner = { open(OnboardingKey(login = true, address = it)) }, modifier = Modifier.align(Alignment.BottomCenter).padding(bottom = 26.dp).graphicsLayer { alpha = (1f - (scrollFold * 2.5f).coerceIn(0f, 1f)) * stage.shown(spot) }, ) } diff --git a/app/src/main/kotlin/com/macedonga/visor/setup/Onboarding.kt b/app/src/main/kotlin/com/macedonga/visor/setup/Onboarding.kt index df3190f7..56af543d 100644 --- a/app/src/main/kotlin/com/macedonga/visor/setup/Onboarding.kt +++ b/app/src/main/kotlin/com/macedonga/visor/setup/Onboarding.kt @@ -162,9 +162,12 @@ class OnboardingViewModel @Inject constructor( var loginOpened by mutableStateOf(false) private var watching: Job? = null private var firstRun = false + // Setting a known pair up again: it is taken as soon as it is heard in pairing mode. + private var wanted: String? = null /** A fresh setup: from the welcome on the first run, otherwise from the first page with something to do. */ - fun begin(welcome: Boolean, login: Boolean) { + fun begin(welcome: Boolean, login: Boolean, address: String? = null) { + wanted = address history.clear() asked = emptySet() loginOpened = false @@ -217,7 +220,11 @@ class OnboardingViewModel @Inject constructor( if (bluetoothOff) bonded = withContext(Dispatchers.IO) { nearby.bonded() } bluetoothOff = false try { - nearby.scan().collect { heard.value = it } + nearby.scan().collect { found -> + heard.value = found + val known = wanted?.let { a -> found.firstOrNull { it.pairing && it.address.equals(a, ignoreCase = true) } } + if (known != null && chosen == null && page == Page.PAIRING) choose(known) + } } catch (stop: CancellationException) { throw stop } catch (failed: Exception) { @@ -274,6 +281,7 @@ class OnboardingViewModel @Inject constructor( fun OnboardingScreen( welcome: Boolean, login: Boolean, + address: String? = null, onLogin: () -> Unit, onEventLog: () -> Unit, onDone: (GlassesTarget) -> Unit, @@ -282,7 +290,7 @@ fun OnboardingScreen( ) { // Saved with the entry, so coming back from the login page goes on where it was. rememberSaveable { - model.begin(welcome, login) + model.begin(welcome, login, address) true } val debugPage by StageDebug.onboardingPage.collectAsStateWithLifecycle() diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt index 08666b4b..306cfa49 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt @@ -173,20 +173,26 @@ class GlassesConnection( is LinkStateChanged -> { val fresh = event.state == LinkState.CONNECTING if (fresh) newAttempt() + // Glasses can turn the key down and still end link setup, then refuse every service: that is no link. + val refused = event.state == LinkState.READY && !host.trustComplete && host.credentialReceived _state.value = _state.value.copy( - link = event.state, + link = if (refused) LinkState.LOST else event.state, // The session publishes CONNECTING at the top of every attempt, including a // reconnect, so that is where the slate is cleared. reason = if (fresh) "" else event.reason, failure = if (fresh) null else _state.value.failure, // A refused key outlives the retries (newAttempt keeps it); a link that comes up is judged afresh. - diagnosis = if (event.state == LinkState.READY) null else _state.value.diagnosis, + diagnosis = when { + refused -> Diagnosis.EVICTED + event.state == LinkState.READY -> null + else -> _state.value.diagnosis + }, ) - when (event.state) { - LinkState.READY -> onReady(host, bundle, scope) + when { + refused -> judge(host, bundle) + event.state == LinkState.READY -> onReady(host, bundle, scope) // An attempt that ended without a verdict gets one now. - LinkState.LOST, LinkState.DISCONNECTED -> judge(host, bundle) - else -> Unit + event.state == LinkState.LOST || event.state == LinkState.DISCONNECTED -> judge(host, bundle) } } From a588fe388e2c9c60429ed4516b3a163b538da41e Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 10:49:28 +0200 Subject: [PATCH 23/90] Presence: the permission Android wants to observe the companion's presence, REQUEST_OBSERVE_COMPANION_DEVICE_PRESENCE, without which the observing was refused --- app/src/main/AndroidManifest.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 9712cebf..aec42ac7 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -23,6 +23,7 @@ + From 3c79f5ee524cce8e3e4372c069566c96c5472488 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 10:54:16 +0200 Subject: [PATCH 24/90] Setup again: a known pair this phone is bonded to skips the pairing page and goes straight to linking, and one heard nearby is taken whether or not it is in pairing mode --- .../com/macedonga/visor/setup/Onboarding.kt | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/setup/Onboarding.kt b/app/src/main/kotlin/com/macedonga/visor/setup/Onboarding.kt index 56af543d..7836ec12 100644 --- a/app/src/main/kotlin/com/macedonga/visor/setup/Onboarding.kt +++ b/app/src/main/kotlin/com/macedonga/visor/setup/Onboarding.kt @@ -162,7 +162,7 @@ class OnboardingViewModel @Inject constructor( var loginOpened by mutableStateOf(false) private var watching: Job? = null private var firstRun = false - // Setting a known pair up again: it is taken as soon as it is heard in pairing mode. + // Setting a known pair up again: it is taken as soon as it is found, bonded or heard, with nothing to pick. private var wanted: String? = null /** A fresh setup: from the welcome on the first run, otherwise from the first page with something to do. */ @@ -187,7 +187,17 @@ class OnboardingViewModel @Inject constructor( fun go(to: Page) { if (to == page) return + // A known pair this phone is bonded to needs no pairing page: straight to linking it. + val known = if (to == Page.PAIRING && chosen == null) wanted?.let { a -> + runCatching { nearby.bonded() }.getOrDefault(emptyList()).firstOrNull { it.address.equals(a, ignoreCase = true) } + } else null history += page + if (known != null) { + chosen = known + page = Page.LINKING + link() + return + } page = to } @@ -210,6 +220,7 @@ class OnboardingViewModel @Inject constructor( watching = viewModelScope.launch { bonded = withContext(Dispatchers.IO) { nearby.bonded() } setUpHere = withContext(Dispatchers.IO) { runCatching { store.all().map { it.glassesAddress.uppercase() }.toSet() }.getOrDefault(emptySet()) } + takeWanted(bonded) while (true) { if (!nearby.enabled) { bluetoothOff = true @@ -222,8 +233,7 @@ class OnboardingViewModel @Inject constructor( try { nearby.scan().collect { found -> heard.value = found - val known = wanted?.let { a -> found.firstOrNull { it.pairing && it.address.equals(a, ignoreCase = true) } } - if (known != null && chosen == null && page == Page.PAIRING) choose(known) + takeWanted(found) } } catch (stop: CancellationException) { throw stop @@ -234,6 +244,11 @@ class OnboardingViewModel @Inject constructor( } } + private fun takeWanted(found: List) { + val known = wanted?.let { a -> found.firstOrNull { it.address.equals(a, ignoreCase = true) } } ?: return + if (chosen == null && page == Page.PAIRING) choose(known) + } + fun stopWatching() { watching?.cancel() watching = null From b8f82530653f2ceae619e896350bca388054279f Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 15:02:00 +0200 Subject: [PATCH 25/90] Display: Captions, a card on the home. Visor starts the Display's live captions or translation as the Meta app does (svc28 StartApp for app 31 with the action and the two languages) and lists each turn as the glasses send it back on svc72, in the text they show. Captions run only with Meta's pack for the language: the page offers it with its size, downloads it and installs it through the glasses' OTA service (svc29 start, install request, the zip on airship ota, then the feature change that turns it on), as the Meta app does; a pack pushed on airship language_resource arrives and turns nothing on. --- .../macedonga/visor/debug/DebugReceiver.kt | 34 +++ .../kotlin/com/macedonga/visor/VisorNav.kt | 5 + .../macedonga/visor/di/ConnectionModule.kt | 5 + .../macedonga/visor/display/CaptionPacks.kt | 223 ++++++++++++++ .../macedonga/visor/display/CaptionsScreen.kt | 284 ++++++++++++++++++ .../macedonga/visor/display/DisplayShelf.kt | 14 +- .../macedonga/visor/display/PrompterScreen.kt | 4 +- .../visor/language/LanguageInstaller.kt | 71 +---- .../macedonga/visor/language/PackDownload.kt | 67 +++++ .../main/res/values-it/strings_captions.xml | 21 ++ app/src/main/res/values/strings_captions.xml | 21 ++ .../visor/core/connection/Captions.kt | 170 +++++++++++ .../core/identity/language/LanguagePacks.kt | 6 + .../com/macedonga/visor/core/media/Airship.kt | 3 +- .../macedonga/visor/core/media/LiveStream.kt | 5 + 15 files changed, 864 insertions(+), 69 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/display/CaptionPacks.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/display/CaptionsScreen.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/language/PackDownload.kt create mode 100644 app/src/main/res/values-it/strings_captions.xml create mode 100644 app/src/main/res/values/strings_captions.xml create mode 100644 core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Captions.kt diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index d698075a..03bcf691 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -59,6 +59,9 @@ class DebugReceiver : BroadcastReceiver() { @Inject lateinit var looks: com.macedonga.visor.stage.Looks @Inject lateinit var phone: com.macedonga.visor.phone.PhoneActions @Inject lateinit var prompter: com.macedonga.visor.core.connection.Teleprompter + @Inject lateinit var captions: com.macedonga.visor.core.connection.Captions + @Inject lateinit var languages: com.macedonga.visor.language.LanguageInstaller + @Inject lateinit var captionPacks: com.macedonga.visor.display.CaptionPacks override fun onReceive(context: Context, intent: Intent) { val action = intent.action?.substringAfterLast('.').orEmpty() @@ -317,6 +320,37 @@ class DebugReceiver : BroadcastReceiver() { } prompter.state.value.let { "${it.phase} refusal=${it.refusal}" } } + // --es features a,b: what Meta has for these constellation features on this pair. + "PACKS" -> answerLater { + val features = intent.getStringExtra("features").orEmpty().split(',').map { it.trim() }.filter { it.isNotEmpty() } + runCatching { languages.offersFor(features) }.fold( + { offers -> offers.joinToString("\n") { "${it.feature} ${it.resource} ${it.version} ${it.type} app ${it.appId} ${it.size} ${it.url}" }.ifEmpty { "none" } }, + { "failed: ${it.message}" }, + ) + } + // --es feature live_captions_en: Meta's pack installed through the glasses' OTA service. + "CAPTION_PACK" -> answerLater { + captionPacks.install(intent.getStringExtra("feature") ?: "live_captions_en") + kotlinx.coroutines.delay(500) + withTimeoutOrNull(600_000) { captionPacks.work.first { it !is com.macedonga.visor.display.PackWork.Working } } + captionPacks.work.value.toString() + } + "CAPTIONS" -> answerLater { + val self = intent.getStringExtra("lang") ?: "en" + when { + intent.getBooleanExtra("stop", false) -> captions.stop() + intent.getBooleanExtra("show", false) -> Unit + else -> captions.start( + if (intent.hasExtra("other")) com.macedonga.visor.core.connection.CaptionsMode.TRANSLATION else com.macedonga.visor.core.connection.CaptionsMode.CAPTIONS, + self, intent.getStringExtra("other") ?: self, + ) + } + kotlinx.coroutines.delay(300) + withTimeoutOrNull(8_000) { + captions.state.first { it.phase == com.macedonga.visor.core.connection.CaptionsPhase.LIVE || it.phase == com.macedonga.visor.core.connection.CaptionsPhase.IDLE } + } + captions.state.value.let { s -> "${s.phase} ${s.mode} refusal=${s.refusal}\n" + s.lines.joinToString("\n") { "${if (it.done) "" else "… "}${it.text}${it.source?.let { src -> " ($src)" }.orEmpty()}" } } + } "PHONE" -> answerLater { when (intent.getStringExtra("do")) { "directions" -> phone.directions(intent.getStringExtra("to").orEmpty(), intent.getStringExtra("mode").orEmpty()).toString() diff --git a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt index 6c4f71f9..188f6e3a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt +++ b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt @@ -157,6 +157,9 @@ data object PrompterKey : NavKey @Serializable data object GesturesKey : NavKey +@Serializable +data object CaptionsKey : NavKey + @Serializable data object WifiNetworksKey : NavKey @@ -218,6 +221,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) onDisplayTool = { tool -> when (tool) { com.macedonga.visor.display.DisplayTool.TELEPROMPTER -> backStack.add(PrompterKey) + com.macedonga.visor.display.DisplayTool.CAPTIONS -> backStack.add(CaptionsKey) com.macedonga.visor.display.DisplayTool.GESTURES -> backStack.add(GesturesKey) com.macedonga.visor.display.DisplayTool.MIRROR -> backStack.add(MirrorKey) } @@ -277,6 +281,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) entry { NotifyAppsScreen() } entry { com.macedonga.visor.display.PrompterScreen() } entry { com.macedonga.visor.display.GesturesScreen() } + entry { com.macedonga.visor.display.CaptionsScreen() } entry { com.macedonga.visor.settings.WifiNetworksScreen() } entry { MetaAiScreen( diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index af3850e9..0955fb63 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -169,6 +169,11 @@ object ConnectionModule { fun teleprompter(connection: GlassesConnection): com.macedonga.visor.core.connection.Teleprompter = com.macedonga.visor.core.connection.Teleprompter(connection) + @Provides + @Singleton + fun captions(connection: GlassesConnection): com.macedonga.visor.core.connection.Captions = + com.macedonga.visor.core.connection.Captions(connection) + @Provides @Singleton fun assistantConfig(@ApplicationContext context: Context): AssistantConfigStore = AssistantConfigStore(context) diff --git a/app/src/main/kotlin/com/macedonga/visor/display/CaptionPacks.kt b/app/src/main/kotlin/com/macedonga/visor/display/CaptionPacks.kt new file mode 100644 index 00000000..fac0ed4f --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/display/CaptionPacks.kt @@ -0,0 +1,223 @@ +package com.macedonga.visor.display + +import android.content.Context +import com.macedonga.visor.core.connection.CaptionsMode +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.language.PackOffer +import com.macedonga.visor.core.media.AirshipReceiver +import com.macedonga.visor.core.media.LiveStream +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.ServiceData +import com.macedonga.visor.language.LanguageInstaller +import com.macedonga.visor.language.downloadPack +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.pairs.pairPrefs +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.File +import java.nio.ByteBuffer +import java.util.UUID +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull + +enum class PackStep { DOWNLOADING, SENDING, INSTALLING } + +sealed interface PackWork { + data object Idle : PackWork + data class Working(val feature: String, val step: PackStep, val done: Long = 0, val total: Long = 0) : PackWork + data class Failed(val feature: String, val reason: String) : PackWork +} + +/** + * The Display's captions and translation run only with Meta's pack for the language, a constellation feature + * (`air.live_captions_en`, `air.translation_en_it`) the glasses' feature manager must have on. Installed as the Meta + * app installs it, through the glasses' OTA service (svc29): StartApp for the OTA app, the install request naming the + * pack, the zip on airship `ota` under the id the request gave it, then a feature change that turns it on. The + * glasses do not say which they have, so this phone remembers what it installed. + */ +@Singleton +class CaptionPacks @Inject constructor( + @ApplicationContext private val context: Context, + private val languages: LanguageInstaller, + private val connection: GlassesConnection, + private val live: LiveStream, + private val airship: AirshipReceiver, + private val log: EventLog, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + @Volatile + private var prefs = pairPrefs(context, "caption_packs", connection.pair.value) + private val dir = File(context.filesDir, "packs") + + private val _installed = MutableStateFlow(prefs.getStringSet(INSTALLED, null).orEmpty()) + val installed: StateFlow> = _installed.asStateFlow() + private val _offers = MutableStateFlow>(emptyMap()) + val offers: StateFlow> = _offers.asStateFlow() + private val _work = MutableStateFlow(PackWork.Idle) + val work: StateFlow = _work.asStateFlow() + private var job: Job? = null + + init { + scope.launch { + connection.pair.filterNotNull().distinctUntilChanged().drop(1).collect { serial -> + prefs = pairPrefs(context, "caption_packs", serial) + _installed.value = prefs.getStringSet(INSTALLED, null).orEmpty() + _offers.value = emptyMap() + } + } + } + + /** Ask Meta about [feature]'s pack, for its size. */ + fun look(feature: String) { + if (feature in _offers.value) return + scope.launch { + runCatching { languages.offersFor(listOf(FEATURE + feature)) }.getOrNull()?.firstOrNull() + ?.let { offer -> _offers.update { it + (feature to offer) } } + } + } + + fun install(feature: String) { + if (job?.isActive == true) return + job = scope.launch { + try { + _work.value = PackWork.Working(feature, PackStep.DOWNLOADING) + val offer = _offers.value[feature] ?: languages.offersFor(listOf(FEATURE + feature)).firstOrNull() + ?: error("Meta has no pack for these glasses") + dir.mkdirs() + val zip = File(dir, offer.resource + ".zip") + if (!zip.exists()) { + if (dir.usableSpace < offer.size + SPARE_BYTES) error("the phone needs ${(offer.size + SPARE_BYTES) / 1_000_000} MB free") + log.add("Captions: downloading ${offer.resource} ${offer.version}") + downloadPack(offer, zip) { _work.value = PackWork.Working(feature, PackStep.DOWNLOADING, it, offer.size) } + } + if (connection.state.value.link != LinkState.READY) error("the glasses are not connected") + log.add("Captions: installing ${zip.name} on the glasses over Wi-Fi") + val why = live.overWifi { mux -> runCatching { installOn(mux, feature, offer, zip) }.exceptionOrNull()?.let { it.message ?: it::class.java.simpleName } } + if (why != null) error(why) + zip.delete() + _installed.update { it + feature } + prefs.edit().putStringSet(INSTALLED, _installed.value).apply() + log.add("Captions: ${offer.resource} is on the glasses") + _work.value = PackWork.Idle + } catch (error: kotlinx.coroutines.CancellationException) { + _work.value = PackWork.Idle + throw error + } catch (error: Throwable) { + log.add("Captions: $feature failed: ${error.message}") + _work.value = PackWork.Failed(feature, error.message ?: error::class.java.simpleName) + } + } + } + + fun cancel() { + job?.cancel() + } + + private suspend fun installOn(mux: DataXConnection, feature: String, offer: PackOffer, zip: File) = coroutineScope { + val fromGlasses = Channel(Channel.UNLIMITED) + val listening = launch { live.wifiData.collect { if (it.serviceId == SVC_OTA) fromGlasses.send(it) } } + suspend fun next(type: Int, ms: Long): ServiceData? = withTimeoutOrNull(ms) { + var m: ServiceData + do { + m = fromGlasses.receive() + if (m.messageType == MT_ERROR) error("the glasses stopped the install: ${errorWords(m.payload)}") + } while (m.messageType != type) + m + } + try { + val artifact = UUID.randomUUID() + val raw = ByteBuffer.allocate(16).putLong(artifact.mostSignificantBits).putLong(artifact.leastSignificantBits).array() + val session = UUID.randomUUID().toString() + val key = Wire.fieldBytes(1, str(UUID.randomUUID().toString())) + Wire.fieldBytes(2, str(session)) + Wire.fieldBytes(3, str(artifact.toString())) + + // The OTA app first: its workflow waits for the install request once started. + val megabytes = (offer.size / 1_000_000 + 1) * 2 + val start = Wire.fieldVarint(1, APP_OTA) + Wire.fieldBytes(2, Wire.fieldBytes(1, Wire.fieldVarint(1, megabytes))) + + intent("ota_type", Wire.fieldBytes(5, str("resource"))) + intent("attempt_id", Wire.fieldBytes(5, str(artifact.toString()))) + + intent("app_transfer_auto", Wire.fieldVarint(2, 1)) + mux.sendOnService(SVC_OTA, MT_START_APP, start) + val started = next(MT_START_APP_RESPONSE, ANSWER_MS) ?: error("the glasses' OTA service did not answer") + val code = runCatching { Wire.parse(started.payload).varint(1) }.getOrDefault(0) + // Already started: its workflow is up, and still takes the request. + if (code != SNAPP_SUCCESS && code != SNAPP_ALREADY_STARTED) error("the glasses would not start an install (code $code)") + + val pack = Wire.fieldVarint(1, APP_RESOURCE) + Wire.fieldBytes(2, raw) + Wire.fieldBytes(3, str(offer.version)) + + Wire.fieldBytes(4, str(offer.appId)) + Wire.fieldBytes(6, str(offer.feature)) + Wire.fieldBytes(7, str("$artifact.zip")) + val request = Wire.fieldBytes(1, key) + Wire.fieldBytes(2, raw) + Wire.fieldVarint(5, zip.length()) + + Wire.fieldBytes(8, pack) + Wire.fieldVarint(9, 1) + mux.sendOnService(SVC_OTA, MT_INSTALL, request) + // They ask for the file once their receiver is ready for it; earlier it is dropped. + next(MT_FILE_STATUS, ANSWER_MS) ?: error("the glasses did not ask for the file") + + _work.value = PackWork.Working(feature, PackStep.SENDING, 0, zip.length()) + airship.send(mux, NAMESPACE, "$artifact.zip", zip, uuid = raw) { + _work.value = PackWork.Working(feature, PackStep.SENDING, it, zip.length()) + }?.let { error(it) } + + _work.value = PackWork.Working(feature, PackStep.INSTALLING) + next(MT_INSTALL_COMPLETE, INSTALL_MS) ?: error("the glasses did not finish installing it") + + val change = Wire.fieldBytes(1, Wire.fieldBytes(2, str(session))) + + Wire.fieldBytes(2, Wire.fieldVarint(1, APP_RESOURCE) + Wire.fieldBytes(2, str(offer.feature)) + Wire.fieldVarint(3, ENABLE)) + mux.sendOnService(SVC_OTA, MT_FEATURE_CHANGE, change) + val changed = next(MT_FEATURE_CHANGE_RESPONSE, ANSWER_MS) + log.add("Captions: feature change answered " + (changed?.payload?.joinToString("") { "%02x".format(it) } ?: "nothing")) + } finally { + listening.cancel() + } + } + + private fun errorWords(payload: ByteArray): String = runCatching { + val f = Wire.parse(payload) + val text = f.bytes(5).decodeToString() + "code ${f.varint(3)}" + if (text.isNotBlank()) ", $text" else "" + }.getOrDefault(payload.joinToString("") { "%02x".format(it) }) + + private fun str(s: String) = s.encodeToByteArray() + + private fun intent(key: String, value: ByteArray) = Wire.fieldBytes(3, Wire.fieldBytes(1, str(key)) + value) + + companion object { + /** The feature a choice needs: one language for captions; for translation, theirs into the wearer's. */ + fun featureFor(mode: CaptionsMode, self: String, other: String): String = + if (mode == CaptionsMode.CAPTIONS) "live_captions_$self" else "translation_${other}_$self" + + private const val FEATURE = "com.meta.constellation.feature.air." + private const val NAMESPACE = "ota" + private const val INSTALLED = "installed" + private const val SPARE_BYTES = 50_000_000L + private const val SVC_OTA = 29 + private const val MT_INSTALL = 1 + private const val MT_ERROR = 3 + private const val MT_FILE_STATUS = 4 + private const val MT_INSTALL_COMPLETE = 7 + private const val MT_FEATURE_CHANGE = 8 + private const val MT_FEATURE_CHANGE_RESPONSE = 9 + private const val MT_START_APP = 2000 + private const val MT_START_APP_RESPONSE = 2001 + private const val APP_OTA = 1L + // RESOURCE_PACK, as the Meta app numbers the package it installs. + private const val APP_RESOURCE = 1L + private const val ENABLE = 1L + private const val SNAPP_SUCCESS = 1L + private const val SNAPP_ALREADY_STARTED = 12L + private const val ANSWER_MS = 15_000L + private const val INSTALL_MS = 240_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/display/CaptionsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/display/CaptionsScreen.kt new file mode 100644 index 00000000..c2397398 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/display/CaptionsScreen.kt @@ -0,0 +1,284 @@ +package com.macedonga.visor.display + +import android.content.Context +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.WindowInsets +import androidx.compose.foundation.layout.asPaddingValues +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.navigationBars +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.lazy.rememberLazyListState +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.macedonga.visor.R +import com.macedonga.visor.core.connection.Captions +import com.macedonga.visor.core.connection.CaptionsMode +import com.macedonga.visor.core.connection.CaptionsPhase +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.settings.ChoiceSheet +import com.macedonga.visor.settings.PillsRow +import com.macedonga.visor.settings.Rows +import com.macedonga.visor.settings.SectionLabel +import com.macedonga.visor.settings.SettingsPage +import com.macedonga.visor.settings.Spinner +import com.macedonga.visor.settings.ValueRow +import com.macedonga.visor.settings.row +import com.macedonga.visor.ui.rememberHaptics +import dagger.hilt.android.lifecycle.HiltViewModel +import dagger.hilt.android.qualifiers.ApplicationContext +import java.util.Locale +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch + +/** Captions or translation, and the languages, as last chosen. */ +data class CaptionsChoice(val mode: CaptionsMode = CaptionsMode.CAPTIONS, val self: String = "en", val other: String = "en") { + /** The other side's language: for captions there is one. */ + val heard: String get() = if (mode == CaptionsMode.CAPTIONS) self else other +} + +@Singleton +class CaptionsChoices @Inject constructor(@ApplicationContext context: Context) { + private val prefs = context.getSharedPreferences("captions", Context.MODE_PRIVATE) + private val _choice = MutableStateFlow( + CaptionsChoice( + runCatching { CaptionsMode.valueOf(prefs.getString(MODE, null).orEmpty()) }.getOrDefault(CaptionsMode.CAPTIONS), + prefs.getString(SELF, null) ?: "en", + prefs.getString(OTHER, null) ?: "en", + ), + ) + val choice: StateFlow = _choice.asStateFlow() + + fun choose(choice: CaptionsChoice) { + _choice.value = choice + prefs.edit().putString(MODE, choice.mode.name).putString(SELF, choice.self).putString(OTHER, choice.other).apply() + } + + private companion object { + const val MODE = "mode" + const val SELF = "self" + const val OTHER = "other" + } +} + +@HiltViewModel +class CaptionsModel @Inject constructor( + val choices: CaptionsChoices, + val captions: Captions, + val connection: GlassesConnection, + val packs: CaptionPacks, +) : ViewModel() { + /** Keep [choice]; while the glasses caption, they stop and start again with it. */ + fun choose(choice: CaptionsChoice) { + choices.choose(choice) + if (captions.state.value.phase != CaptionsPhase.LIVE) return + viewModelScope.launch { + captions.stop() + captions.state.first { it.phase == CaptionsPhase.IDLE } + captions.start(choice.mode, choice.self, choice.heard) + } + } +} + +/** Captions and translation on the Display: what and in which languages, then what the glasses hear, turn by turn. */ +@Composable +fun CaptionsScreen(model: CaptionsModel = hiltViewModel()) { + val choice by model.choices.choice.collectAsStateWithLifecycle() + val state by model.captions.state.collectAsStateWithLifecycle() + val glasses by model.connection.state.collectAsStateWithLifecycle() + val ready = glasses.link == LinkState.READY + val feature = CaptionPacks.featureFor(choice.mode, choice.self, choice.heard) + val installed by model.packs.installed.collectAsStateWithLifecycle() + val offers by model.packs.offers.collectAsStateWithLifecycle() + val work by model.packs.work.collectAsStateWithLifecycle() + LaunchedEffect(feature, ready) { if (ready) model.packs.look(feature) } + var picking by remember { mutableStateOf(null) } + val haptics = rememberHaptics() + val list = rememberLazyListState() + val last = state.lines.lastOrNull() + // Follow the newest turn, unless the transcript was scrolled back to read. + LaunchedEffect(state.lines.size, last?.text) { + val info = list.layoutInfo + val end = info.visibleItemsInfo.lastOrNull()?.index ?: return@LaunchedEffect + if (end >= info.totalItemsCount - 3) list.animateScrollToItem(info.totalItemsCount - 1) + } + SettingsPage( + stringResource(R.string.captions_title), + list = list, + sheet = { + CaptionsBar( + ready = ready, + phase = state.phase, + refusal = state.refusal, + pack = PackView(feature in installed, offers[feature]?.size, work.takeIf { it !is PackWork.Idle }), + onGet = { haptics.done(); model.packs.install(feature) }, + onCancel = { model.packs.cancel() }, + onStart = { haptics.done(); model.captions.start(choice.mode, choice.self, choice.heard) }, + onStop = { haptics.toggle(); model.captions.stop() }, + ) + picking?.let { side -> + ChoiceSheet( + stringResource(if (side == Side.SELF) if (choice.mode == CaptionsMode.CAPTIONS) R.string.captions_language else R.string.captions_self else R.string.captions_other), + languageOptions(), + if (side == Side.SELF) choice.self else choice.other, + { model.choose(if (side == Side.SELF) choice.copy(self = it) else choice.copy(other = it)) }, + { picking = null }, + ) + } + }, + ) { + item(key = "choice") { + Rows( + row { + PillsRow( + stringResource(R.string.captions_mode), + listOf(CaptionsMode.CAPTIONS to stringResource(R.string.captions_mode_captions), CaptionsMode.TRANSLATION to stringResource(R.string.captions_mode_translation)), + choice.mode, { model.choose(choice.copy(mode = it)) }, + ) + }, + row { + ValueRow( + stringResource(if (choice.mode == CaptionsMode.CAPTIONS) R.string.captions_language else R.string.captions_self), + languageName(choice.self), { picking = Side.SELF }, + ) + }, + if (choice.mode == CaptionsMode.TRANSLATION) row { + ValueRow(stringResource(R.string.captions_other), languageName(choice.other), { picking = Side.OTHER }) + } else null, + ) + } + if (state.lines.isNotEmpty() || state.phase == CaptionsPhase.LIVE) { + item(key = "heard") { SectionLabel(stringResource(R.string.captions_heard)) } + } + if (state.lines.isEmpty() && state.phase == CaptionsPhase.LIVE) { + item(key = "listening") { Turn(stringResource(R.string.captions_listening), null, current = false) } + } + items(state.lines, key = { "line-" + it.id }) { line -> Turn(line.text, line.source, current = line === last && state.phase == CaptionsPhase.LIVE) } + item(key = "foot") { Spacer(Modifier.height(BAR_ROOM)) } + } +} + +private enum class Side { SELF, OTHER } + +/** One turn: the newest one bright while the glasses listen, what was said under a translation. */ +@Composable +private fun Turn(text: String, source: String?, current: Boolean) { + val colors = Visor.colors + Column(Modifier.fillMaxWidth().padding(horizontal = 32.dp, vertical = 8.dp)) { + Text(text, style = Visor.type.body, color = if (current) colors.textPrimary else colors.textSecondary) + if (source != null) Text(source, style = Visor.type.label, color = colors.textTertiary, modifier = Modifier.padding(top = 2.dp)) + } +} + +/** At the foot of the page, over a fade: whether the glasses are captioning, and Start or Stop. */ +@Composable +private fun CaptionsBar( + ready: Boolean, + phase: CaptionsPhase, + refusal: Long?, + pack: PackView, + onGet: () -> Unit, + onCancel: () -> Unit, + onStart: () -> Unit, + onStop: () -> Unit, +) { + val colors = Visor.colors + val bottom = WindowInsets.navigationBars.asPaddingValues().calculateBottomPadding() + Box(Modifier.fillMaxSize(), contentAlignment = Alignment.BottomCenter) { + Column( + Modifier + .fillMaxWidth() + .background(Brush.verticalGradient(0f to Color.Transparent, 0.3f to colors.canvas)) + .padding(start = 16.dp, end = 16.dp, top = 28.dp, bottom = bottom + 16.dp), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.Center, + ) { + val work = pack.work + val line = when { + work is PackWork.Working -> stringResource( + when (work.step) { + PackStep.DOWNLOADING -> R.string.language_downloading + PackStep.SENDING -> R.string.language_sending + PackStep.INSTALLING -> R.string.captions_installing + }, + ) + if (work.total > 0) " · ${megabytes(work.done)} / ${megabytes(work.total)}" else "" + work is PackWork.Failed -> stringResource(R.string.captions_pack_failed, work.reason) + !ready -> stringResource(R.string.captions_not_connected) + refusal != null && phase == CaptionsPhase.IDLE -> refusalWords(refusal) + phase == CaptionsPhase.LIVE -> stringResource(R.string.captions_live) + else -> null + } + if (line != null) { + val bad = work is PackWork.Failed || refusal != null && ready && phase == CaptionsPhase.IDLE + Text(line, style = Visor.type.label, color = if (bad) colors.signal else colors.textSecondary, modifier = Modifier.padding(bottom = 12.dp)) + } + when { + work is PackWork.Working -> WidePill(stringResource(R.string.language_cancel), onCancel, Modifier.fillMaxWidth(), quiet = true) + !ready -> Unit + phase == CaptionsPhase.IDLE && !pack.installed -> Row(horizontalArrangement = Arrangement.spacedBy(10.dp)) { + WidePill(stringResource(R.string.captions_start_anyway), onStart, Modifier.weight(1f), quiet = true) + WidePill( + pack.size?.let { stringResource(R.string.captions_get_size, megabytes(it)) } ?: stringResource(R.string.language_download), + onGet, Modifier.weight(1f), + ) + } + phase == CaptionsPhase.STARTING || phase == CaptionsPhase.STOPPING -> Box(Modifier.height(52.dp), contentAlignment = Alignment.Center) { Spinner() } + phase == CaptionsPhase.LIVE -> WidePill(stringResource(R.string.captions_stop), onStop, Modifier.fillMaxWidth(), quiet = true) + else -> WidePill(stringResource(R.string.captions_start), onStart, Modifier.fillMaxWidth()) + } + } + } +} + +/** Whether this phone sent the choice's pack, how big Meta's is, and a download or send under way. */ +private data class PackView(val installed: Boolean, val size: Long?, val work: PackWork?) + +private fun megabytes(bytes: Long) = "${(bytes + 500_000) / 1_000_000} MB" + +/** The card's line: the language, or the two of a translation. */ +@Composable +fun captionsSummary(choice: CaptionsChoice): String = + if (choice.mode == CaptionsMode.CAPTIONS) languageName(choice.self) + else stringResource(R.string.captions_pair, languageName(choice.other), languageName(choice.self)) + +private fun languageName(code: String): String = + Locale.forLanguageTag(code).let { it.getDisplayLanguage(Locale.getDefault()).replaceFirstChar { c -> c.titlecase(Locale.getDefault()) } } + +private fun languageOptions(): List> = + LANGUAGES.map { Triple(it, languageName(it), null) }.sortedBy { it.second } + +// The Display's own list, from its Captions app; packs beyond English come from Meta. +private val LANGUAGES = listOf( + "en", "es", "it", "fr", "de", "pt", "zh", "ja", "ko", "hi", "te", "bn", "nl", "tl", "gu", "pa", "kn", "ml", "da", + "fi", "sv", "ru", "vi", "tr", "he", "pl", "hu", "cs", "hr", "el", "ms", "ta", "mr", "ar", "id", "th", "ur", +) +private val BAR_ROOM = 120.dp diff --git a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt index 73d7dae1..1684caec 100644 --- a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt +++ b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt @@ -35,7 +35,7 @@ import dagger.hilt.android.lifecycle.HiltViewModel import javax.inject.Inject /** What the Display's cards on the home open. */ -enum class DisplayTool { TELEPROMPTER, GESTURES, MIRROR } +enum class DisplayTool { TELEPROMPTER, CAPTIONS, GESTURES, MIRROR } @HiltViewModel class ShelfModel @Inject constructor( @@ -44,6 +44,8 @@ class ShelfModel @Inject constructor( val band: com.macedonga.visor.band.BandLink, val connection: com.macedonga.visor.core.connection.GlassesConnection, val screencast: com.macedonga.visor.core.media.Screencast, + val captions: com.macedonga.visor.core.connection.Captions, + val captionsChoices: CaptionsChoices, ) : ViewModel() /** The Display's tools on the home: a card each, with what it holds now. */ @@ -68,6 +70,16 @@ fun DisplayShelf(onOpen: (DisplayTool) -> Unit, modifier: Modifier = Modifier, m onClick = { onOpen(DisplayTool.TELEPROMPTER) }, ) } + item(key = "captions") { + val captions by model.captions.state.collectAsStateWithLifecycle() + val choice by model.captionsChoices.choice.collectAsStateWithLifecycle() + ToolCard( + VisorIcons.Captions, + stringResource(R.string.captions_title), + if (captions.phase == com.macedonga.visor.core.connection.CaptionsPhase.LIVE) stringResource(R.string.captions_live) else captionsSummary(choice), + onClick = { onOpen(DisplayTool.CAPTIONS) }, + ) + } item(key = "gestures") { ToolCard( VisorIcons.Band, diff --git a/app/src/main/kotlin/com/macedonga/visor/display/PrompterScreen.kt b/app/src/main/kotlin/com/macedonga/visor/display/PrompterScreen.kt index 8cb50523..358aa7b4 100644 --- a/app/src/main/kotlin/com/macedonga/visor/display/PrompterScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/display/PrompterScreen.kt @@ -252,7 +252,7 @@ private fun PrompterBar( } @Composable -private fun WidePill(text: String, onClick: () -> Unit, modifier: Modifier, quiet: Boolean = false, enabled: Boolean = true) { +internal fun WidePill(text: String, onClick: () -> Unit, modifier: Modifier, quiet: Boolean = false, enabled: Boolean = true) { Box( modifier .height(52.dp) @@ -299,7 +299,7 @@ private fun ScriptSheet(text: String, onText: (String) -> Unit, onDismiss: () -> } @Composable -private fun refusalWords(code: Long): String = stringResource( +internal fun refusalWords(code: Long): String = stringResource( when (code) { Teleprompter.SNAPP_NOT_ENOUGH_BATTERY -> R.string.prompter_refused_battery Teleprompter.SNAPP_THERMAL_THROTTLING -> R.string.prompter_refused_heat diff --git a/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt b/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt index e346c393..73b094d7 100644 --- a/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt +++ b/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt @@ -14,18 +14,12 @@ import com.macedonga.visor.login.MetaLogin import com.macedonga.visor.pairs.pairPrefs import dagger.hilt.android.qualifiers.ApplicationContext import java.io.File -import java.io.FileOutputStream -import java.net.HttpURLConnection -import java.net.URL -import java.security.MessageDigest import javax.inject.Inject import javax.inject.Singleton -import kotlin.coroutines.coroutineContext import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.ensureActive import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow @@ -34,7 +28,6 @@ import kotlinx.coroutines.flow.drop import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext /** Where an install is. [done] and [total] are bytes for the download and the transfer. */ sealed interface LanguageWork { @@ -120,7 +113,7 @@ class LanguageInstaller @Inject constructor( if (!zip.exists()) { if (dir.usableSpace < offer.size + SPARE_BYTES) error("the phone needs ${mb(offer.size + SPARE_BYTES)} free") log.add("Language: downloading ${offer.resource} ${offer.version}, ${mb(offer.size)}") - download(offer, zip) { done -> _work.value = LanguageWork.Working(language, LanguageWork.Step.DOWNLOADING, done, offer.size) } + downloadPack(offer, zip) { done -> _work.value = LanguageWork.Working(language, LanguageWork.Step.DOWNLOADING, done, offer.size) } } if (connection.state.value.link != LinkState.READY) error("the glasses are not connected") step(language, LanguageWork.Step.SENDING, 0, zip.length()) @@ -169,7 +162,10 @@ class LanguageInstaller @Inject constructor( prefs.edit().putStringSet(KEY_INSTALLED, _installed.value - "en_US").apply() } - private fun ask(packs: List): List { + private fun ask(packs: List): List = offersFor(packs.map { LANGUAGE_FEATURE + it }) + + /** What Meta has for these glasses under the constellation [features], named in full. */ + fun offersFor(features: List): List { val info = connection.state.value.info ?: error("connect the glasses once, so they say which they are") // Meta/hammerhead/greathammerhead:14/UKQ1.250303.001/72387580077200100:user/release-keys val parts = info.build.split('/') @@ -177,62 +173,7 @@ class LanguageInstaller @Inject constructor( val type = parts.getOrNull(4)?.substringAfter(':', "user").orEmpty() if (firmware.isEmpty()) error("the glasses' build is unknown") val token = login.source.get() - return fetchPackOffers(token, info.serial, firmware, packs.map { LANGUAGE_FEATURE + it }, "ota.${parts.getOrElse(1) { "hammerhead" }}.$type") - } - - /** The zip, resumed where an earlier try stopped, checked against Meta's SHA-256. */ - private suspend fun download(offer: PackOffer, file: File, onProgress: (Long) -> Unit) = withContext(Dispatchers.IO) { - val part = File(file.path + ".part") - var have = if (part.exists()) part.length() else 0L - val c = URL(offer.url).openConnection() as HttpURLConnection - try { - c.connectTimeout = 30_000 - c.readTimeout = 60_000 - if (have > 0) c.setRequestProperty("Range", "bytes=$have-") - when (c.responseCode) { - 206 -> Unit - 200 -> have = 0 - else -> error("the download answered ${c.responseCode}") - } - FileOutputStream(part, have > 0).use { out -> - c.inputStream.use { input -> - val buffer = ByteArray(256 * 1024) - var lastReport = 0L - while (true) { - coroutineContext.ensureActive() - val n = input.read(buffer) - if (n < 0) break - out.write(buffer, 0, n) - have += n - if (have - lastReport > 1_000_000) { - lastReport = have - onProgress(have) - } - } - } - } - } finally { - c.disconnect() - } - onProgress(have) - if (offer.sha256.isNotEmpty() && sha256(part) != offer.sha256.lowercase()) { - part.delete() - error("the download did not match Meta's checksum; try again") - } - check(part.renameTo(file)) { "could not keep the download" } - } - - private fun sha256(file: File): String { - val digest = MessageDigest.getInstance("SHA-256") - file.inputStream().use { input -> - val buffer = ByteArray(1024 * 1024) - while (true) { - val n = input.read(buffer) - if (n < 0) break - digest.update(buffer, 0, n) - } - } - return digest.digest().joinToString("") { "%02x".format(it) } + return fetchPackOffers(token, info.serial, firmware, features, "ota.${parts.getOrElse(1) { "hammerhead" }}.$type") } private companion object { diff --git a/app/src/main/kotlin/com/macedonga/visor/language/PackDownload.kt b/app/src/main/kotlin/com/macedonga/visor/language/PackDownload.kt new file mode 100644 index 00000000..5172b198 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/language/PackDownload.kt @@ -0,0 +1,67 @@ +package com.macedonga.visor.language + +import com.macedonga.visor.core.identity.language.PackOffer +import java.io.File +import java.io.FileOutputStream +import java.net.HttpURLConnection +import java.net.URL +import java.security.MessageDigest +import kotlin.coroutines.coroutineContext +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.withContext + +/** The zip, resumed where an earlier try stopped, checked against Meta's SHA-256. */ +internal suspend fun downloadPack(offer: PackOffer, file: File, onProgress: (Long) -> Unit) = withContext(Dispatchers.IO) { + val part = File(file.path + ".part") + var have = if (part.exists()) part.length() else 0L + val c = URL(offer.url).openConnection() as HttpURLConnection + try { + c.connectTimeout = 30_000 + c.readTimeout = 60_000 + if (have > 0) c.setRequestProperty("Range", "bytes=$have-") + when (c.responseCode) { + 206 -> Unit + 200 -> have = 0 + else -> error("the download answered ${c.responseCode}") + } + FileOutputStream(part, have > 0).use { out -> + c.inputStream.use { input -> + val buffer = ByteArray(256 * 1024) + var lastReport = 0L + while (true) { + coroutineContext.ensureActive() + val n = input.read(buffer) + if (n < 0) break + out.write(buffer, 0, n) + have += n + if (have - lastReport > 1_000_000) { + lastReport = have + onProgress(have) + } + } + } + } + } finally { + c.disconnect() + } + onProgress(have) + if (offer.sha256.isNotEmpty() && sha256(part) != offer.sha256.lowercase()) { + part.delete() + error("the download did not match Meta's checksum; try again") + } + check(part.renameTo(file)) { "could not keep the download" } +} + +private fun sha256(file: File): String { + val digest = MessageDigest.getInstance("SHA-256") + file.inputStream().use { input -> + val buffer = ByteArray(1024 * 1024) + while (true) { + val n = input.read(buffer) + if (n < 0) break + digest.update(buffer, 0, n) + } + } + return digest.digest().joinToString("") { "%02x".format(it) } +} diff --git a/app/src/main/res/values-it/strings_captions.xml b/app/src/main/res/values-it/strings_captions.xml new file mode 100644 index 00000000..efb1ba28 --- /dev/null +++ b/app/src/main/res/values-it/strings_captions.xml @@ -0,0 +1,21 @@ + + + Sottotitoli + Modalità + Sottotitoli + Traduzione + Lingua + Tu parli + Gli altri parlano + Da %1$s a %2$s + Ascoltato + In ascolto… + Avvia sugli occhiali + Ferma + Sugli occhiali + Connetti gli occhiali per avviare + Avvia + Scarica · %1$s + Non installato: %1$s + Installazione sugli occhiali + diff --git a/app/src/main/res/values/strings_captions.xml b/app/src/main/res/values/strings_captions.xml new file mode 100644 index 00000000..ddc6974c --- /dev/null +++ b/app/src/main/res/values/strings_captions.xml @@ -0,0 +1,21 @@ + + + Captions + Mode + Captions + Translation + Language + You speak + They speak + %1$s to %2$s + Heard + Listening… + Start on glasses + Stop + On the glasses + Connect the glasses to start + Start + Download · %1$s + Did not install: %1$s + Installing on the glasses + diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Captions.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Captions.kt new file mode 100644 index 00000000..e11c2f97 --- /dev/null +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Captions.kt @@ -0,0 +1,170 @@ +package com.macedonga.visor.core.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.LinkState +import java.util.UUID +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch + +enum class CaptionsPhase { IDLE, STARTING, LIVE, STOPPING } + +enum class CaptionsMode { CAPTIONS, TRANSLATION } + +/** One speech turn as the glasses last sent it: [text] to read, [source] what was said when it is a translation. */ +data class CaptionLine(val id: String, val text: String, val source: String?, val done: Boolean) + +/** [refusal] is the app manager's code when the glasses would not start it. */ +data class CaptionsState( + val phase: CaptionsPhase = CaptionsPhase.IDLE, + val mode: CaptionsMode = CaptionsMode.CAPTIONS, + val refusal: Long? = null, + val lines: List = emptyList(), +) + +/** + * The Display's live captions and translation, started as the Meta app starts them: svc28 StartApp (2000) for app 31, + * TRANSLATION, with `action.lst_lc.action` and the two languages as intent data, closed with StopApp (2002) and the + * matching stop action. The glasses recognise the speech themselves and send each turn back on svc72 as a + * TwoWayTranslation; they say so when the wearer stops it there. + */ +class Captions(private val connection: GlassesConnection) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val _state = MutableStateFlow(CaptionsState()) + val state: StateFlow = _state.asStateFlow() + private var wait: Job? = null + + init { + scope.launch { + connection.serviceData.collect { data -> + when (data.serviceId) { + SVC_COMPANION -> fromAppManager(data.messageType, runCatching { Wire.parse(data.payload).varint(1) }.getOrDefault(0L)) + SVC_TRANSLATION -> runCatching { fromSession(data.messageType, Wire.parse(data.payload)) } + } + } + } + scope.launch { + connection.state.map { it.link == LinkState.READY }.distinctUntilChanged().collect { up -> + if (!up) _state.update { it.copy(phase = CaptionsPhase.IDLE) } + } + } + } + + /** Start [mode] for a wearer speaking [self] with others speaking [other], codes such as "en"; the same for captions. */ + fun start(mode: CaptionsMode, self: String, other: String) { + if (connection.state.value.link != LinkState.READY) return + val data = intent("action.lst_lc.self_lang", self) + intent("action.lst_lc.other_lang", other) + + intent("action.lst_lc.request_id", UUID.randomUUID().toString()) + intent("action.lst_lc.action", action(mode, "start")) + _state.value = CaptionsState(CaptionsPhase.STARTING, mode) + connection.submit { it.sendOnService(SVC_COMPANION, MT_START_APP, Wire.fieldVarint(1, APP_TRANSLATION) + data) } + settleAfter { if (it.phase == CaptionsPhase.STARTING) it.copy(phase = CaptionsPhase.LIVE) else it } + } + + fun stop() { + val now = _state.value + if (now.phase == CaptionsPhase.IDLE) return + _state.update { it.copy(phase = CaptionsPhase.STOPPING) } + val data = intent("action.lst_lc.action", action(now.mode, "stop")) + intent("action.lst_lc.request_id", UUID.randomUUID().toString()) + connection.submit { it.sendOnService(SVC_COMPANION, MT_STOP_APP, Wire.fieldVarint(1, APP_TRANSLATION) + data) } + settleAfter { if (it.phase == CaptionsPhase.STOPPING) it.copy(phase = CaptionsPhase.IDLE) else it } + } + + // Other apps' starts and stops are answered here too; only the one being waited for counts. + private fun fromAppManager(type: Int, code: Long) { + val phase = _state.value.phase + if (type == MT_START_APP_RESPONSE && phase == CaptionsPhase.STARTING) { + wait?.cancel() + _state.update { + if (code == Teleprompter.SNAPP_SUCCESS || code == Teleprompter.SNAPP_ALREADY_STARTED) it.copy(phase = CaptionsPhase.LIVE) + else it.copy(phase = CaptionsPhase.IDLE, refusal = code) + } + } else if (type == MT_STOP_APP_RESPONSE && phase == CaptionsPhase.STOPPING) { + wait?.cancel() + _state.update { it.copy(phase = CaptionsPhase.IDLE) } + } + } + + private fun fromSession(type: Int, fields: Wire.Fields) { + when (type) { + MT_TWO_WAY_TRANSLATION -> turn(fields, _state.value.mode)?.let { line -> + _state.update { now -> + val at = now.lines.indexOfLast { it.id == line.id } + val lines = if (at >= 0) now.lines.toMutableList().also { it[at] = line } else (now.lines + line).takeLast(MAX_LINES) + now.copy(phase = if (now.phase == CaptionsPhase.IDLE) CaptionsPhase.LIVE else now.phase, lines = lines) + } + } + // Also how a session the wearer started on the glasses shows here. Its category is 1 for captions as well. + MT_INVITE_TO_JOIN -> _state.update { + if (it.phase == CaptionsPhase.IDLE) CaptionsState(CaptionsPhase.LIVE) else it.copy(phase = CaptionsPhase.LIVE) + } + MT_SESSION_STATUS -> if (fields.varint(1) == STATUS_IDLE) ended() + MT_END_FROM_FRAME, MT_STOPPING_FROM_FRAME -> ended() + } + } + + // A start makes the glasses end the session before it first; that end is not this one's. + private fun ended() { + if (_state.value.phase == CaptionsPhase.STARTING) return + wait?.cancel() + _state.update { it.copy(phase = CaptionsPhase.IDLE) } + } + + /** + * Fields 12 and 11 are what was heard, final then partial, as it is recognised; 10 and 9 (and 4 and 5) what the + * Display shows, formatted or translated, with "..." while that catches up. A partial goes on with its own space. + */ + private fun turn(f: Wire.Fields, mode: CaptionsMode): CaptionLine? { + fun s(fn: Int) = f.bytes(fn).decodeToString() + fun shown(done: Int, partial: Int) = (s(done) + s(partial).takeUnless { it.trim() == PENDING }.orEmpty()).trim() + val heard = (s(12) + s(11)).trim() + val shown = shown(10, 9).ifEmpty { shown(4, 5) }.ifEmpty { s(2).trim() } + // As the glasses show it; what was heard stands in until their text comes. + val text = shown.ifEmpty { if (mode == CaptionsMode.CAPTIONS) heard else PENDING } + if (text.isEmpty() || text == PENDING && heard.isEmpty()) return null + val id = s(1).ifEmpty { "t${f.varint(6)}" } + val done = f.varint(13) != 0L || s(11).isEmpty() && s(9).isEmpty() && s(5).isEmpty() + return CaptionLine(id, text, heard.takeIf { mode == CaptionsMode.TRANSLATION && it.isNotEmpty() && it != text }, done) + } + + private fun settleAfter(change: (CaptionsState) -> CaptionsState) { + wait?.cancel() + wait = scope.launch { + delay(ANSWER_MS) + _state.update(change) + } + } + + private fun action(mode: CaptionsMode, verb: String) = + if (mode == CaptionsMode.TRANSLATION) "action.speech_translation.$verb" else "action.speech_caption.$verb" + + private fun intent(key: String, value: String): ByteArray = + Wire.fieldBytes(3, Wire.fieldBytes(1, key.encodeToByteArray()) + Wire.fieldBytes(5, value.encodeToByteArray())) + + private companion object { + const val SVC_COMPANION = 28 + const val SVC_TRANSLATION = 72 + const val MT_START_APP = 2000 + const val MT_START_APP_RESPONSE = 2001 + const val MT_STOP_APP = 2002 + const val MT_STOP_APP_RESPONSE = 2003 + const val MT_TWO_WAY_TRANSLATION = 1 + const val MT_INVITE_TO_JOIN = 2 + const val MT_END_FROM_FRAME = 4 + const val MT_SESSION_STATUS = 17 + const val MT_STOPPING_FROM_FRAME = 24 + const val STATUS_IDLE = 0L + const val APP_TRANSLATION = 31L + const val MAX_LINES = 300 + const val PENDING = "..." + const val ANSWER_MS = 7_000L + } +} diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt index f6e6834d..da57cabf 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt @@ -28,6 +28,10 @@ data class PackOffer( val url: String, val size: Long, val sha256: String, + /** What the glasses' OTA install names it by, e.g. `7506485332707669`. */ + val appId: String = "", + /** e.g. `RESOURCE_PACK`. */ + val type: String = "", ) const val LANGUAGE_FEATURE = "com.meta.constellation.feature.air.language_" @@ -105,6 +109,8 @@ fun fetchPackOffers( url = o.text("file_uri") ?: return@mapNotNull null, size = (o["file_size"] as? JsonPrimitive)?.longOrNull ?: o.text("file_size")?.toLongOrNull() ?: 0, sha256 = o.text("file_checksum").orEmpty(), + appId = o.text("app_id").orEmpty(), + type = o.text("resource_type").orEmpty(), ) } } diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Airship.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Airship.kt index 8bf5132d..d4b2b805 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Airship.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Airship.kt @@ -171,10 +171,11 @@ class AirshipReceiver(private val spool: File) : Responder { baseName: String, file: java.io.File, chunk: Int = 1024, + uuid: ByteArray? = null, onProgress: (Long) -> Unit = {}, ): String? = java.io.RandomAccessFile(file, "r").use { raf -> - send(connection, namespace, baseName, file.length(), chunk, onProgress) { at, n -> ByteArray(n).also { raf.seek(at); raf.readFully(it) } } + send(connection, namespace, baseName, file.length(), chunk, onProgress, uuid) { at, n -> ByteArray(n).also { raf.seek(at); raf.readFully(it) } } } private suspend fun send( diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt index 3ec546d5..a11ba999 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt @@ -352,6 +352,10 @@ class LiveStream( private val _wifiSession = MutableStateFlow(null) val wifiSession: StateFlow = _wifiSession.asStateFlow() + // What the glasses send on the Wi-Fi session, for work run there with overWifi. + private val _wifiData = kotlinx.coroutines.flow.MutableSharedFlow(extraBufferCapacity = 256) + val wifiData: kotlinx.coroutines.flow.SharedFlow = _wifiData + /** The picture now as a JPEG, decoded from the replay ring; null when no Live runs. */ suspend fun frameNow(): ByteArray? = if (_running.value) outputs.replay.lastPicture()?.let { grabJpeg(it, quality = 88) } else null @@ -848,6 +852,7 @@ class LiveStream( } is ServiceData -> { messageAt = System.currentTimeMillis() + _wifiData.tryEmit(event) onWifiMessage(event) } is ProtocolError -> say("Wi-Fi: ${event.detail}") From 05fca1634295f1bb61d0b7151eff18afc46d8398 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 15:14:52 +0200 Subject: [PATCH 26/90] Neural Band: Visor takes the band only for a lesson or an errand. It took it whenever the Display's link to the phone was down, so a Display restarting found its band held by the phone and came back without it. --- .../main/kotlin/com/macedonga/visor/band/BandLink.kt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt index 2bd21136..f3f9de3a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt @@ -33,7 +33,6 @@ import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.first -import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex @@ -56,8 +55,8 @@ data class BandState( ) /** - * The phone's own link to the Neural Band: the band takes one host at a time, so the phone leaves it to the Display - * while the Display is connected or holds a lease on it (svc150), and takes it only for a lesson or one errand (the + * The phone's own link to the Neural Band: the band takes one host at a time, so the phone leaves it to the Display, + * which may hold a lease on it (svc150), and takes it only for a lesson or one errand (the * wrist, a buzz); the Display reports the band's battery itself. It reads the battery and the wrist over svc52822 and sets the * wrist; in a lesson it turns the gesture streams on and acks each gesture, as the Meta app does. */ @@ -91,9 +90,9 @@ class BandLink @Inject constructor( connection.state.collect { if (it.link != LinkState.READY && leased.value) leased.value = false } } scope.launch { - // A lesson or an errand takes the band from the Display; when it ends, the band goes back. - val displayUp = connection.state.map { it.link == LinkState.READY }.distinctUntilChanged() - val stepBack = combine(leased, displayUp, lesson, borrowing) { lease, up, learning, errand -> (lease || up) && !learning && !errand } + // A lesson or an errand takes the band from the Display; when it ends, the band goes back. Never otherwise: + // the Display reaches the band on its own, and a phone holding it while the Display restarts keeps it off. + val stepBack = combine(lesson, borrowing) { learning, errand -> !learning && !errand } combine(connection.running, connection.pair, bands.all, stepBack, held) { running, pair, _, away, hold -> bands.forGlasses(pair)?.takeIf { running && hold && !away } }.distinctUntilChanged().collect { band -> gate.withLock { switchTo(band) } } From 13dd691d640cc7921e229a2bd87383956d935a44 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 15:40:20 +0200 Subject: [PATCH 27/90] Display: Mirror leaves the home until the Display's screencast answers. It now runs as the Meta app does, over Wi-Fi, the manager opened before the settings and the frames channel opened only after their answer, and says when the glasses need a restart to start the service; the Display still does not answer the settings. --- .../macedonga/visor/debug/DebugReceiver.kt | 12 +++ .../macedonga/visor/di/ConnectionModule.kt | 3 +- .../macedonga/visor/display/DisplayShelf.kt | 10 --- .../com/macedonga/visor/media/MirrorScreen.kt | 2 + app/src/main/res/values-it/strings_mirror.xml | 2 + app/src/main/res/values/strings_mirror.xml | 2 + .../macedonga/visor/core/media/LiveStream.kt | 2 + .../macedonga/visor/core/media/Screencast.kt | 87 ++++++++++++++----- 8 files changed, 88 insertions(+), 32 deletions(-) diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index 03bcf691..7aef2cf7 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -62,6 +62,7 @@ class DebugReceiver : BroadcastReceiver() { @Inject lateinit var captions: com.macedonga.visor.core.connection.Captions @Inject lateinit var languages: com.macedonga.visor.language.LanguageInstaller @Inject lateinit var captionPacks: com.macedonga.visor.display.CaptionPacks + @Inject lateinit var screencast: com.macedonga.visor.core.media.Screencast override fun onReceive(context: Context, intent: Intent) { val action = intent.action?.substringAfterLast('.').orEmpty() @@ -328,6 +329,17 @@ class DebugReceiver : BroadcastReceiver() { { "failed: ${it.message}" }, ) } + // --es key --ez value true|false: one bool written to the glasses. + "MC_SET" -> answerLater { + config.send(mapOf(intent.getStringExtra("key").orEmpty() to intent.getBooleanExtra("value", false))) ?: "sent" + } + // --ez stop true to stop; the answer is the state once it settles. + "MIRROR" -> answerLater { + if (intent.getBooleanExtra("stop", false)) screencast.stop() else screencast.start() + kotlinx.coroutines.delay(500) + withTimeoutOrNull(40_000) { screencast.state.first { it.phase != com.macedonga.visor.core.media.ScreencastPhase.STARTING } } + screencast.state.value.toString() + } // --es feature live_captions_en: Meta's pack installed through the glasses' OTA service. "CAPTION_PACK" -> answerLater { captionPacks.install(intent.getStringExtra("feature") ?: "live_captions_en") diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 0955fb63..74914de2 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -275,7 +275,8 @@ object ConnectionModule { @Provides @Singleton - fun screencast(connection: GlassesConnection, config: GlassesConfig): Screencast = Screencast(connection, config) + fun screencast(connection: GlassesConnection, config: GlassesConfig, live: com.macedonga.visor.core.media.LiveStream): Screencast = + Screencast(connection, config, live) @Provides @Singleton diff --git a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt index 1684caec..cce3b32b 100644 --- a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt +++ b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt @@ -43,7 +43,6 @@ class ShelfModel @Inject constructor( val prompter: Teleprompter, val band: com.macedonga.visor.band.BandLink, val connection: com.macedonga.visor.core.connection.GlassesConnection, - val screencast: com.macedonga.visor.core.media.Screencast, val captions: com.macedonga.visor.core.connection.Captions, val captionsChoices: CaptionsChoices, ) : ViewModel() @@ -88,15 +87,6 @@ fun DisplayShelf(onOpen: (DisplayTool) -> Unit, modifier: Modifier = Modifier, m onClick = { onOpen(DisplayTool.GESTURES) }, ) } - item(key = "mirror") { - val mirror by model.screencast.state.collectAsStateWithLifecycle() - ToolCard( - VisorIcons.Live, - stringResource(R.string.mirror_title), - if (mirror.phase == com.macedonga.visor.core.media.ScreencastPhase.LIVE) stringResource(R.string.mirror_live) else stringResource(R.string.mirror_idle), - onClick = { onOpen(DisplayTool.MIRROR) }, - ) - } } } diff --git a/app/src/main/kotlin/com/macedonga/visor/media/MirrorScreen.kt b/app/src/main/kotlin/com/macedonga/visor/media/MirrorScreen.kt index bc54c9f9..b72f8e38 100644 --- a/app/src/main/kotlin/com/macedonga/visor/media/MirrorScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/media/MirrorScreen.kt @@ -124,6 +124,8 @@ private fun phraseOf(s: ScreencastState): String = when (s.phase) { ScreencastFault.NOT_READ -> stringResource(R.string.mirror_not_read) ScreencastFault.NO_FLAG -> stringResource(R.string.mirror_no_flag) ScreencastFault.FLAG_NOT_SENT -> stringResource(R.string.mirror_flag_not_sent, s.detail) + ScreencastFault.NEEDS_RESTART -> stringResource(R.string.mirror_needs_restart) + ScreencastFault.NO_WIFI -> stringResource(R.string.mirror_no_wifi, s.detail) ScreencastFault.NO_ANSWER -> stringResource(R.string.mirror_no_answer) ScreencastFault.ASLEEP -> stringResource(R.string.mirror_asleep) ScreencastFault.GLASSES_TIMEOUT -> stringResource(R.string.mirror_timeout) diff --git a/app/src/main/res/values-it/strings_mirror.xml b/app/src/main/res/values-it/strings_mirror.xml index 1f777e90..3d4d8ab1 100644 --- a/app/src/main/res/values-it/strings_mirror.xml +++ b/app/src/main/res/values-it/strings_mirror.xml @@ -14,4 +14,6 @@ Lo schermo degli occhiali è spento. Gli occhiali sono andati in timeout. Gli occhiali hanno rifiutato lo schermo condiviso. + Riavvia gli occhiali una volta per attivare la proiezione. + Il Wi-Fi con gli occhiali non si è attivato: %1$s diff --git a/app/src/main/res/values/strings_mirror.xml b/app/src/main/res/values/strings_mirror.xml index fdb1f0d7..47726847 100644 --- a/app/src/main/res/values/strings_mirror.xml +++ b/app/src/main/res/values/strings_mirror.xml @@ -14,4 +14,6 @@ The glasses\' screen is asleep. The glasses timed out. The glasses refused the screencast. + Restart the glasses once to turn on mirroring. + Wi-Fi to the glasses did not come up: %1$s diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt index a11ba999..efbe3b81 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt @@ -887,6 +887,8 @@ class LiveStream( } // Airship, whose receiver reports for itself; everything else goes in the log. event.serviceId == 28 || event.serviceId == SVC_AIRSHIP -> Unit + // The Display's screencast, which reads its own. + event.serviceId == SVC_HN_SCREENCAST || event.serviceId == SVC_HN_SCREENCAST_MANAGER -> Unit else -> say( "Wi-Fi: service ${event.serviceId} type ${event.messageType}, ${event.payload.size} bytes " + event.payload.take(16).joinToString("") { "%02x".format(it) }, diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt index 0851d08e..8e5e2996 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt @@ -19,6 +19,9 @@ import kotlinx.coroutines.flow.SharedFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asSharedFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch import kotlinx.coroutines.withTimeoutOrNull @@ -30,15 +33,19 @@ const val SCREENCAST_FLAG = "hn_demos.enable_hn_screencast" enum class ScreencastPhase { IDLE, STARTING, LIVE, FAILED } -enum class ScreencastFault { NOT_CONNECTED, NOT_READ, NO_FLAG, FLAG_NOT_SENT, NO_ANSWER, ASLEEP, GLASSES_TIMEOUT, UNKNOWN } +enum class ScreencastFault { NOT_CONNECTED, NOT_READ, NO_FLAG, FLAG_NOT_SENT, NEEDS_RESTART, NO_WIFI, NO_ANSWER, ASLEEP, GLASSES_TIMEOUT, UNKNOWN } data class ScreencastState(val phase: ScreencastPhase = ScreencastPhase.IDLE, val fault: ScreencastFault? = null, val detail: String = "") /** One whole H.264 frame, its parameter sets in band. */ class ScreencastFrame(val timestampMs: Long, val key: Boolean, val data: ByteArray) -/** The Display's own screen, streamed to the phone on svc23463 and driven on svc23464. */ -class Screencast(private val connection: GlassesConnection, private val config: GlassesConfig) { +/** + * The Display's own screen, streamed to the phone on svc23463 and driven on svc23464, over Wi-Fi as the Meta app runs + * it: the manager opened and answered, then the encoder settings and their answer, then the frames opened. The service + * runs only while the MobileConfig switch is on, and the glasses read that switch when they start. + */ +class Screencast(private val connection: GlassesConnection, private val config: GlassesConfig, private val live: LiveStream) { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private val _state = MutableStateFlow(ScreencastState()) val state: StateFlow = _state.asStateFlow() @@ -48,6 +55,7 @@ class Screencast(private val connection: GlassesConnection, private val config: private var job: Job? = null private var answer: CompletableDeferred>? = null + private var stopping = CompletableDeferred() // The frame being assembled: its time, key flag and the chunks still owed. private var timestamp = 0L @@ -69,11 +77,10 @@ class Screencast(private val connection: GlassesConnection, private val config: /** Stop the screencast on the glasses and forget the state. */ @Synchronized fun stop() { - val live = _state.value.phase == ScreencastPhase.STARTING || _state.value.phase == ScreencastPhase.LIVE - job?.cancel() - job = null + // Live, the run sends the stop itself and lets Wi-Fi go; still starting, it is cut short. + if (_state.value.phase != ScreencastPhase.LIVE) job?.cancel() + stopping.complete(Unit) _state.value = ScreencastState() - if (live) connection.submit { it.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONTROL, byteArrayOf(3)) } } private suspend fun run() { @@ -87,22 +94,57 @@ class Screencast(private val connection: GlassesConnection, private val config: if (SCREENCAST_FLAG !in cfg.params) return fail(ScreencastFault.NO_FLAG) if (cfg.values[SCREENCAST_FLAG] != true) { config.send(mapOf(SCREENCAST_FLAG to true))?.let { return fail(ScreencastFault.FLAG_NOT_SENT, it) } + return fail(ScreencastFault.NEEDS_RESTART) } - - val reply = CompletableDeferred>() - answer = reply - connection.submit { it.sendOnService(SVC_HN_SCREENCAST, MT_CONTROL, byteArrayOf(0)) } - connection.submit { it.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONTROL, byteArrayOf(0)) } - connection.submit { it.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONFIGURE, configure()) } - val got = withTimeoutOrNull(ANSWER_MS) { reply.await() } - answer = null - when { - got == null -> fail(ScreencastFault.NO_ANSWER) - got.first -> _state.value = ScreencastState(ScreencastPhase.LIVE) - got.second == 2 -> fail(ScreencastFault.ASLEEP) - got.second == 3 -> fail(ScreencastFault.GLASSES_TIMEOUT) - else -> fail(ScreencastFault.UNKNOWN) + stopping = CompletableDeferred() + val why = live.overWifi { mux -> + coroutineScope { + val listening = launch { live.wifiData.collect { onData(it) } } + // A service that is not running answers 0xC001: the switch is on but the glasses have not restarted since. + val absent = launch { + live.notes.first { it.contains("service $SVC_HN_SCREENCAST_MANAGER answered status 0xc001") } + answer?.complete(Pair(false, NOT_RUNNING)) + } + try { + // The open is not answered; the Meta app goes straight on to the settings, which are. + mux.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONTROL, byteArrayOf(0)) + delay(OPEN_MS) + val configured = ask { mux.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONFIGURE, configure()) } + if (configured?.first == true) { + mux.sendOnService(SVC_HN_SCREENCAST, MT_CONTROL, byteArrayOf(0)) + _state.value = ScreencastState(ScreencastPhase.LIVE) + stopping.await() + mux.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONTROL, byteArrayOf(3)) + delay(STOP_MS) + } else refused(configured) + } finally { + listening.cancel() + absent.cancel() + } + } + null } + if (why != null) fail(ScreencastFault.NO_WIFI, why) + } + + /** Send, and wait for the manager's answer: success, and why not. */ + private suspend fun ask(send: suspend () -> Unit): Pair? { + val reply = CompletableDeferred>().also { answer = it } + send() + return withTimeoutOrNull(ANSWER_MS) { reply.await() }.also { answer = null } + } + + private fun refused(got: Pair?) { + _state.value = ScreencastState( + ScreencastPhase.FAILED, + when { + got == null -> ScreencastFault.NO_ANSWER + got.second == NOT_RUNNING -> ScreencastFault.NEEDS_RESTART + got.second == 2 -> ScreencastFault.ASLEEP + got.second == 3 -> ScreencastFault.GLASSES_TIMEOUT + else -> ScreencastFault.UNKNOWN + }, + ) } private fun onData(m: ServiceData) { @@ -141,5 +183,8 @@ class Screencast(private val connection: GlassesConnection, private val config: const val HEADER_BYTES = 32 const val KEY_FLAG = 1 const val ANSWER_MS = 15_000L + const val STOP_MS = 300L + const val OPEN_MS = 500L + const val NOT_RUNNING = -1 } } From ddee7baa490760e1be7f11cae986f4db8b814a1b Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 15:44:29 +0200 Subject: [PATCH 28/90] Display: the teleprompter knows captions or translation took the glasses' screen, from the Display opening their session on svc72, so its card no longer says it is on the glasses. --- .../com/macedonga/visor/core/connection/Teleprompter.kt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Teleprompter.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Teleprompter.kt index 43852f6d..16e94767 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Teleprompter.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Teleprompter.kt @@ -33,7 +33,7 @@ data class PrompterLook(val sizeSp: Double = 35.0, val speed: Double = 32.0, val * The Display's teleprompter, run from the phone as the Meta app runs it: svc28 StartApp (2000) for app 35 with the * script and its look as intent data, answered by 2001; a start while it shows replaces the text. StopApp (2002) * closes it, answered by 2003. Following the voice is the glasses' own `asr_enabled`, which the Meta app never sends. - * The glasses do not say when the wearer closes it there. + * The glasses do not say when the wearer closes it there; captions opening is seen, on svc72. */ class Teleprompter(private val connection: GlassesConnection) { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) @@ -44,6 +44,11 @@ class Teleprompter(private val connection: GlassesConnection) { init { scope.launch { connection.serviceData.collect { data -> + // Captions or translation opening on the glasses takes the screen from it. + if (data.serviceId == SVC_TRANSLATION && data.messageType == MT_INVITE_TO_JOIN && _state.value.phase == PrompterPhase.SHOWING) { + wait?.cancel() + _state.value = PrompterState() + } if (data.serviceId != SVC_COMPANION) return@collect val code = runCatching { Wire.parse(data.payload).varint(1) }.getOrDefault(0L) when (data.messageType) { @@ -111,6 +116,8 @@ class Teleprompter(private val connection: GlassesConnection) { const val SNAPP_ALREADY_STARTED = 12L const val SNAPP_DENIED_BY_PEAK_POWER = 15L private const val SVC_COMPANION = 28 + private const val SVC_TRANSLATION = 72 + private const val MT_INVITE_TO_JOIN = 2 private const val MT_START_APP = 2000 private const val MT_START_APP_RESPONSE = 2001 private const val MT_STOP_APP = 2002 From 8acfe430430828062bc8b4a64e9b5178e8e9cdbd Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 15:53:28 +0200 Subject: [PATCH 29/90] Debug API: SEND puts one message on a service and answers with what the glasses said back, a status included. --- .../com/macedonga/visor/debug/DebugReceiver.kt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index 7aef2cf7..d25478f9 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -329,6 +329,20 @@ class DebugReceiver : BroadcastReceiver() { { "failed: ${it.message}" }, ) } + // --ei svc --ei type --es hex: one message on a service; the answer is what the glasses said within 5 s. + "SEND" -> answerLater { + val svc = intent.getIntExtra("svc", 0) + val body = intent.getStringExtra("hex").orEmpty().chunked(2).filter { it.length == 2 }.map { it.toInt(16).toByte() }.toByteArray() + val heard = java.util.Collections.synchronizedList(mutableListOf()) + val watch = scope.launch { + launch { connection.serviceData.collect { if (it.serviceId == svc) heard += "type ${it.messageType}: " + it.payload.joinToString("") { b -> "%02x".format(b) } } } + launch { connection.warnings.collect { if (it.contains("service $svc ")) heard += it } } + } + connection.submit { it.sendOnService(svc, intent.getIntExtra("type", 0), body) } + kotlinx.coroutines.delay(5_000) + watch.cancel() + heard.joinToString("\n").ifEmpty { "nothing" } + } // --es key --ez value true|false: one bool written to the glasses. "MC_SET" -> answerLater { config.send(mapOf(intent.getStringExtra("key").orEmpty() to intent.getBooleanExtra("value", false))) ?: "sent" From 155c913291accee24f13b51dcfd4e5fe8f5496b8 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 16:15:55 +0200 Subject: [PATCH 30/90] Display: the glasses' apps fetch through the phone. Visor serves svc9000, the data fetcher: an http(s) link the Display's apps ask for comes back on their channel in chunks, an image scaled to the size and format they ask, then the DataX OK status (0); other schemes are refused. Instagram's pictures now show on the Display. Visor also answers svc41108 with the phone's internet state (online, metered, DNS servers). The event log opens a Requests page: each request the glasses made, its link, outcome, size and time, kept in memory only; a tap copies the link. --- .../macedonga/visor/debug/DebugReceiver.kt | 5 + .../kotlin/com/macedonga/visor/VisorNav.kt | 6 +- .../macedonga/visor/connection/DataFetcher.kt | 201 ++++++++++++++++++ .../visor/connection/InternetSwitch.kt | 53 +++++ .../macedonga/visor/connection/RequestLog.kt | 51 +++++ .../macedonga/visor/di/ConnectionModule.kt | 4 + .../com/macedonga/visor/log/EventLogScreen.kt | 3 +- .../macedonga/visor/log/RequestLogScreen.kt | 110 ++++++++++ .../main/res/values-it/strings_requests.xml | 8 + app/src/main/res/values/strings_requests.xml | 8 + 10 files changed, 447 insertions(+), 2 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt create mode 100644 app/src/main/res/values-it/strings_requests.xml create mode 100644 app/src/main/res/values/strings_requests.xml diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index d25478f9..4f9d0cb8 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -63,6 +63,7 @@ class DebugReceiver : BroadcastReceiver() { @Inject lateinit var languages: com.macedonga.visor.language.LanguageInstaller @Inject lateinit var captionPacks: com.macedonga.visor.display.CaptionPacks @Inject lateinit var screencast: com.macedonga.visor.core.media.Screencast + @Inject lateinit var requests: com.macedonga.visor.connection.RequestLog override fun onReceive(context: Context, intent: Intent) { val action = intent.action?.substringAfterLast('.').orEmpty() @@ -343,6 +344,10 @@ class DebugReceiver : BroadcastReceiver() { watch.cancel() heard.joinToString("\n").ifEmpty { "nothing" } } + // The request log's newest entries, links cut at their query. + "REQUESTS" -> resultData = requests.items.value.takeLast(intent.getIntExtra("n", 30)).joinToString("\n") { + "${it.source} ${it.outcome ?: "running"} ${it.bytes}B ${it.millis}ms [${it.options}] ${it.url.substringBefore('?').take(160)}" + }.ifEmpty { "none" } // --es key --ez value true|false: one bool written to the glasses. "MC_SET" -> answerLater { config.send(mapOf(intent.getStringExtra("key").orEmpty() to intent.getBooleanExtra("value", false))) ?: "sent" diff --git a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt index 188f6e3a..2e7a08e5 100644 --- a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt +++ b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt @@ -160,6 +160,9 @@ data object GesturesKey : NavKey @Serializable data object CaptionsKey : NavKey +@Serializable +data object RequestLogKey : NavKey + @Serializable data object WifiNetworksKey : NavKey @@ -259,7 +262,8 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) entry { NativeLoginPage(onDone = { backStack.removeLastOrNull() }) } - entry { EventLogScreen() } + entry { EventLogScreen(onRequests = { backStack.add(RequestLogKey) }) } + entry { com.macedonga.visor.log.RequestLogScreen() } entry { GalleryScreen() } // The pair travels between the home and Settings on the one stage, so the screens only cross-fade, both ways. entry( diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt b/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt new file mode 100644 index 00000000..d60c4409 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt @@ -0,0 +1,201 @@ +package com.macedonga.visor.connection + +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import java.io.ByteArrayOutputStream +import java.io.InputStream +import java.net.HttpURLConnection +import java.net.URL +import java.util.concurrent.ConcurrentHashMap +import java.util.zip.DeflaterOutputStream +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlin.math.max +import kotlin.math.roundToInt +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit + +/** + * svc9000, the data fetcher: the Display's apps (Instagram, messaging, the prompter) ask the phone for a URL and get + * its bytes back on the same channel, a ResponseData {1 data, 2 compressed, 3 total} per chunk, then an OK status. + * Plain http(s) streams; with an image transformation the whole image is scaled first. Other schemes are refused. + */ +@Singleton +class DataFetcher @Inject constructor(private val log: Provider, private val requests: RequestLog) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val running = ConcurrentHashMap() + private val slots = Semaphore(PARALLEL) + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val channel = message.info.channelId + if (message.info.flags.isClosed) { + running.remove(channel)?.cancel() + if (message.payload.isEmpty()) return + } + if (message.info.messageType != MT_REQUEST) return + val query = runCatching { Query.decode(message.payload) }.getOrNull() ?: return + val reply = DataXConnection.mirror(channel) + // Off the reader: a download must not hold up everything else on the link. + running[channel] = scope.launch { + try { + slots.withPermit { serve(query, reply, connection) } + } finally { + running.remove(channel) + } + } + } + + private suspend fun serve(q: Query, reply: Int, mux: DataXConnection) { + val id = requests.start(q.url, q.source, q.options()) + var sent = 0L + val scheme = q.url.substringBefore(':').lowercase() + if (scheme != "http" && scheme != "https") { + requests.finish(id, "refused, no $scheme here", failed = true) + mux.sendStatus(reply, UNKNOWN_TYPE) + return + } + val chunk = (if (q.chunkSize > 0) q.chunkSize else DEFAULT_CHUNK).coerceAtMost(MAX_CHUNK) + val c = runCatching { open(q.url) }.getOrElse { + log.get().add("Fetch: ${host(q.url)} failed: ${it.message}") + requests.finish(id, "failed: ${it.message}", failed = true) + mux.sendStatus(reply, INTERNAL_ERROR) + return + } + try { + val code = c.responseCode + if (code !in 200..299) { + log.get().add("Fetch: ${host(q.url)} answered $code") + requests.finish(id, "HTTP $code", failed = true) + mux.sendStatus(reply, INTERNAL_ERROR) + return + } + if (q.scale == null) { + sent = stream(c.inputStream, c.contentLengthLong, chunk, reply, mux) + } else { + var body = c.inputStream.use { it.readBytes() } + body = scaled(body, q.scale) ?: body + val compressed = q.compress + if (compressed) body = ByteArrayOutputStream().also { out -> DeflaterOutputStream(out).use { it.write(body) } }.toByteArray() + for (at in body.indices step chunk) { + send(mux, reply, body.copyOfRange(at, minOf(body.size, at + chunk)), body.size.toLong(), compressed) + } + sent = body.size.toLong() + } + mux.sendStatus(reply, OK) + requests.finish(id, "HTTP $code" + if (q.scale != null) ", scaled" else "", failed = false, bytes = sent) + } catch (error: kotlinx.coroutines.CancellationException) { + requests.finish(id, "cancelled", failed = false, bytes = sent) + throw error + } catch (error: Exception) { + log.get().add("Fetch: ${host(q.url)} broke off: ${error.message}") + requests.finish(id, "broke off: ${error.message}", failed = true, bytes = sent) + runCatching { mux.sendStatus(reply, INTERNAL_ERROR) } + } finally { + c.disconnect() + } + } + + /** The body as it comes, a chunk per message; how much went. */ + private suspend fun stream(input: InputStream, total: Long, chunk: Int, reply: Int, mux: DataXConnection): Long { + var sent = 0L + input.use { + val buffer = ByteArray(chunk) + while (true) { + kotlin.coroutines.coroutineContext.ensureActive() + val n = it.read(buffer) + if (n < 0) break + if (n > 0) { + send(mux, reply, buffer.copyOf(n), total, false) + sent += n + } + } + } + return sent + } + + private suspend fun send(mux: DataXConnection, reply: Int, data: ByteArray, total: Long, compressed: Boolean) { + val body = Wire.fieldBytes(1, data) + Wire.fieldVarint(2, if (compressed) 1 else 0) + Wire.fieldVarint(3, total.coerceIn(-1, Int.MAX_VALUE.toLong())) + mux.send(reply, body, messageType = MT_REQUEST) + } + + private fun open(url: String): HttpURLConnection = (URL(url).openConnection() as HttpURLConnection).apply { + connectTimeout = TIMEOUT_MS + readTimeout = TIMEOUT_MS + instanceFollowRedirects = true + } + + /** The image fitted to the asked size and format; null to send it as it came, also when that is no bigger. */ + private fun scaled(image: ByteArray, s: Scale): ByteArray? { + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(image, 0, image.size, bounds) + if (bounds.outWidth <= 0 || s.width <= 0 || s.height <= 0) return null + var sample = 1 + while (bounds.outWidth / (sample * 2) >= s.width && bounds.outHeight / (sample * 2) >= s.height) sample *= 2 + val decoded = BitmapFactory.decodeByteArray(image, 0, image.size, BitmapFactory.Options().apply { inSampleSize = sample }) ?: return null + val (w, h) = if (s.keepAspect) { + val k = minOf(s.width.toFloat() / decoded.width, s.height.toFloat() / decoded.height, 1f) + max(1, (decoded.width * k).roundToInt()) to max(1, (decoded.height * k).roundToInt()) + } else s.width to s.height + val fitted = if (w == decoded.width && h == decoded.height) decoded else Bitmap.createScaledBitmap(decoded, w, h, true) + val format = when (s.format) { + FORMAT_PNG -> Bitmap.CompressFormat.PNG + FORMAT_WEBP -> Bitmap.CompressFormat.WEBP_LOSSY + else -> Bitmap.CompressFormat.JPEG + } + val out = ByteArrayOutputStream().also { fitted.compress(format, if (s.quality in 1..100) s.quality else DEFAULT_QUALITY, it) }.toByteArray() + return out.takeIf { it.size < image.size } + } + + private fun host(url: String) = runCatching { URL(url).host }.getOrDefault("?") + + private class Scale(val width: Int, val height: Int, val format: Int, val keepAspect: Boolean, val quality: Int) + + /** RequestQuery: 1 url, 2 compress, 3 transformations (1 image scale), 4 chunk size, 6 who asked. */ + private class Query(val url: String, val compress: Boolean, val scale: Scale?, val chunkSize: Int, val source: Int) { + fun options(): String = listOfNotNull( + scale?.let { s -> "${s.width}x${s.height} ${listOf("jpeg", "png", "webp").getOrElse(s.format) { "format ${s.format}" }}" }, + "zlib".takeIf { compress }, + "chunk $chunkSize".takeIf { chunkSize > 0 }, + ).joinToString(", ") + + companion object { + fun decode(payload: ByteArray): Query { + val f = Wire.parse(payload) + val scale = f.repeatedBytes(3).firstNotNullOfOrNull { t -> + Wire.parse(t).takeIf { it.has(1) }?.let { Wire.parse(it.bytes(1)) }?.let { + Scale(it.varint(1).toInt(), it.varint(2).toInt(), it.varint(3).toInt(), it.varint(5) != 0L, it.varint(6).toInt()) + } + } + return Query(f.bytes(1).decodeToString(), f.varint(2) != 0L, scale, f.varint(4).toInt(), f.varint(6).toInt()) + } + } + } + + companion object { + const val SVC = 9000 + private const val MT_REQUEST = 0 + private const val OK = 0 + private const val INTERNAL_ERROR = 0xC004 + private const val UNKNOWN_TYPE = 0xD001 + private const val DEFAULT_CHUNK = 1024 + // A DataX message holds at most 16383 bytes, header included. + private const val MAX_CHUNK = 16_000 + private const val FORMAT_PNG = 1 + private const val FORMAT_WEBP = 2 + private const val DEFAULT_QUALITY = 85 + private const val PARALLEL = 4 + private const val TIMEOUT_MS = 20_000 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt new file mode 100644 index 00000000..d5b7f23d --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt @@ -0,0 +1,53 @@ +package com.macedonga.visor.connection + +import android.content.Context +import android.net.ConnectivityManager +import android.net.NetworkCapabilities +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * svc41108, internet_switch: the Display asks how the phone's internet is (type 6) and is told (type 3): connected, + * metered, and the phone's DNS servers. Told it is connected, the Display may carry its apps' traffic to the phone. + */ +@Singleton +class InternetSwitch @Inject constructor( + @ApplicationContext private val context: Context, + private val log: Provider, +) : Responder { + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.messageType != MT_LISTEN) return + val state = phoneInternet() + log.get().add("Internet: the Display asked; the phone is ${if (state.connected) "online" else "offline"}" + if (state.metered) ", metered" else "") + connection.send(DataXConnection.mirror(message.info.channelId), encode(state), messageType = MT_COMPANION_STATE) + } + + private data class PhoneInternet(val connected: Boolean, val metered: Boolean, val dns: List) + + private fun phoneInternet(): PhoneInternet { + val cm = context.getSystemService(ConnectivityManager::class.java) ?: return PhoneInternet(false, false, emptyList()) + val network = cm.activeNetwork + val caps = network?.let { cm.getNetworkCapabilities(it) } + val online = caps?.hasCapability(NetworkCapabilities.NET_CAPABILITY_VALIDATED) == true + val dns = network?.let { cm.getLinkProperties(it) }?.dnsServers.orEmpty().mapNotNull { it.hostAddress } + return PhoneInternet(online, cm.isActiveNetworkMetered, dns) + } + + private fun encode(s: PhoneInternet): ByteArray = + Wire.fieldVarint(1, if (s.connected) 1 else 0) + Wire.fieldVarint(2, if (s.metered) 1 else 0) + + Wire.fieldBytes(3, s.dns.fold(ByteArray(0)) { acc, a -> acc + Wire.fieldBytes(1, a.encodeToByteArray()) }) + + companion object { + const val SVC = 41108 + private const val MT_COMPANION_STATE = 3 + private const val MT_LISTEN = 6 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt b/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt new file mode 100644 index 00000000..6533fb3f --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt @@ -0,0 +1,51 @@ +package com.macedonga.visor.connection + +import java.util.concurrent.atomic.AtomicLong +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update + +/** One request the glasses had the phone make: [outcome] stays null while it runs. */ +data class GlassesRequest( + val id: Long, + val at: Long, + val url: String, + val source: Int, + /** What was asked besides the link: image size and format, compression. */ + val options: String = "", + val outcome: String? = null, + val failed: Boolean = false, + val bytes: Long = 0, + val millis: Long = 0, +) + +/** The requests the glasses made through the phone, newest last, in memory only: their links can carry tokens. */ +@Singleton +class RequestLog @Inject constructor() { + private val ids = AtomicLong() + private val _items = MutableStateFlow>(emptyList()) + val items: StateFlow> = _items.asStateFlow() + + fun start(url: String, source: Int, options: String = ""): Long { + val id = ids.incrementAndGet() + _items.update { (it + GlassesRequest(id, System.currentTimeMillis(), url, source, options)).takeLast(MAX) } + return id + } + + fun finish(id: Long, outcome: String, failed: Boolean, bytes: Long = 0) { + _items.update { list -> + list.map { if (it.id == id) it.copy(outcome = outcome, failed = failed, bytes = bytes, millis = System.currentTimeMillis() - it.at) else it } + } + } + + fun clear() { + _items.value = emptyList() + } + + private companion object { + const val MAX = 500 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 74914de2..3d22500b 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -141,6 +141,8 @@ object ConnectionModule { assistant: javax.inject.Provider, lease: javax.inject.Provider, icons: NotifyIcons, + internet: com.macedonga.visor.connection.InternetSwitch, + fetcher: com.macedonga.visor.connection.DataFetcher, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -151,6 +153,8 @@ object ConnectionModule { SVC_GLASSES_PHOTO to photos, SVC_TAR to timers, GlassesNotification.SVC to icons, + com.macedonga.visor.connection.InternetSwitch.SVC to internet, + com.macedonga.visor.connection.DataFetcher.SVC to fetcher, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), diff --git a/app/src/main/kotlin/com/macedonga/visor/log/EventLogScreen.kt b/app/src/main/kotlin/com/macedonga/visor/log/EventLogScreen.kt index 84983823..196ac763 100644 --- a/app/src/main/kotlin/com/macedonga/visor/log/EventLogScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/log/EventLogScreen.kt @@ -61,7 +61,7 @@ class EventLogViewModel @Inject constructor(val log: EventLog, val media: MediaI /** The log as one card of lines, opened at the end: what matters is what just happened. */ @Composable -fun EventLogScreen(model: EventLogViewModel = hiltViewModel()) { +fun EventLogScreen(onRequests: () -> Unit = {}, model: EventLogViewModel = hiltViewModel()) { val context = LocalContext.current val lines by model.log.lines.collectAsStateWithLifecycle() val list = rememberLazyListState() @@ -79,6 +79,7 @@ fun EventLogScreen(model: EventLogViewModel = hiltViewModel()) { light = 0.3f, list = list, actions = { + RoundIcon(VisorIcons.Globe, stringResource(R.string.requests_title), onRequests) RoundIcon(VisorIcons.Download, stringResource(R.string.event_log_save), { create.launch("visor-log-${LocalDateTime.now().format(FILE_TIME)}.txt") }) RoundIcon(VisorIcons.Share, stringResource(R.string.event_log_share), { share(context, model.log.text()) }) RoundIcon(VisorIcons.Trash, stringResource(R.string.event_log_clear), model.log::clear) diff --git a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt new file mode 100644 index 00000000..c347d8c5 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt @@ -0,0 +1,110 @@ +package com.macedonga.visor.log + +import android.content.ClipData +import android.content.ClipboardManager +import android.content.Context +import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.itemsIndexed +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.RectangleShape +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.macedonga.visor.R +import com.macedonga.visor.connection.GlassesRequest +import com.macedonga.visor.connection.RequestLog +import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.gallery.RoundIcon +import com.macedonga.visor.settings.Footnote +import com.macedonga.visor.settings.Hairline +import com.macedonga.visor.settings.LocalHost +import com.macedonga.visor.settings.MONO +import com.macedonga.visor.settings.SettingsPage +import com.macedonga.visor.ui.VisorIcons +import dagger.hilt.android.lifecycle.HiltViewModel +import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter +import javax.inject.Inject + +@HiltViewModel +class RequestLogViewModel @Inject constructor(val log: RequestLog) : ViewModel() + +/** What the glasses' apps asked the phone to fetch, newest first; a tap copies the whole link. */ +@Composable +fun RequestLogScreen(model: RequestLogViewModel = hiltViewModel()) { + val items by model.log.items.collectAsStateWithLifecycle() + val shown = items.asReversed() + SettingsPage( + stringResource(R.string.requests_title), + light = 0.3f, + actions = { RoundIcon(VisorIcons.Trash, stringResource(R.string.event_log_clear), model.log::clear) }, + ) { + if (shown.isEmpty()) item(key = "empty") { Footnote(stringResource(R.string.requests_empty)) } + itemsIndexed(shown, key = { _, r -> r.id }) { i, r -> + val shape = when { + shown.size == 1 -> RoundedCornerShape(18.dp) + i == 0 -> RoundedCornerShape(topStart = 18.dp, topEnd = 18.dp) + i == shown.lastIndex -> RoundedCornerShape(bottomStart = 18.dp, bottomEnd = 18.dp) + else -> RectangleShape + } + Column(Modifier.padding(horizontal = 16.dp).fillMaxWidth().clip(shape).background(Visor.colors.raised)) { + if (i > 0) Hairline() + RequestRow(r) + } + } + } +} + +@Composable +private fun RequestRow(r: GlassesRequest) { + val colors = Visor.colors + val context = LocalContext.current + val host = LocalHost.current + val copied = stringResource(R.string.requests_copied) + Column( + Modifier.fillMaxWidth().clickable { + context.getSystemService(ClipboardManager::class.java)?.setPrimaryClip(ClipData.newPlainText("url", r.url)) + host.notice.show(copied) + }.padding(horizontal = 16.dp, vertical = 10.dp), + ) { + Text(place(r.url), style = Visor.type.label, color = colors.textPrimary, maxLines = 2, overflow = TextOverflow.Ellipsis) + val outcome = r.outcome ?: stringResource(R.string.requests_running) + val parts = listOfNotNull( + TIME.format(Instant.ofEpochMilli(r.at).atZone(ZoneId.systemDefault())), + outcome, + r.bytes.takeIf { it > 0 }?.let(::size), + r.millis.takeIf { r.outcome != null }?.let { "$it ms" }, + stringResource(R.string.requests_source, r.source), + r.options.ifEmpty { null }, + ) + Text(parts.joinToString(" · "), style = MONO, color = if (r.failed) colors.signal else colors.textTertiary, modifier = Modifier.padding(top = 2.dp)) + } +} + +/** Host and path; the query, often a long signature, is left to the copied link. */ +private fun place(url: String): String { + val noQuery = url.substringBefore('?') + return noQuery.substringAfter("://").ifEmpty { url }.let { if (url.startsWith("http")) it else noQuery } +} + +private fun size(bytes: Long): String = when { + bytes >= 1_000_000 -> "%.1f MB".format(bytes / 1_000_000.0) + bytes >= 1_000 -> "${bytes / 1_000} KB" + else -> "$bytes B" +} + +private val TIME = DateTimeFormatter.ofPattern("HH:mm:ss") diff --git a/app/src/main/res/values-it/strings_requests.xml b/app/src/main/res/values-it/strings_requests.xml new file mode 100644 index 00000000..424510a1 --- /dev/null +++ b/app/src/main/res/values-it/strings_requests.xml @@ -0,0 +1,8 @@ + + + Richieste + Qui compare ciò che gli occhiali chiedono al telefono di scaricare. + Link copiato + in corso + origine %1$d + diff --git a/app/src/main/res/values/strings_requests.xml b/app/src/main/res/values/strings_requests.xml new file mode 100644 index 00000000..49c68d54 --- /dev/null +++ b/app/src/main/res/values/strings_requests.xml @@ -0,0 +1,8 @@ + + + Requests + What the glasses ask the phone to fetch shows here. + Link copied + running + source %1$d + From e36741883fbc3fb46995e213654301573d3ec0b5 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 16:32:03 +0200 Subject: [PATCH 31/90] Display: the glasses' apps on the phone's internet. Visor opens its own svc41108 channel and tells the Display the phone is online, which lifts the Display's block on the phone's internet; the Display then tunnels its apps' IP packets on svc41111 (framed 1c fb, length, ethertype), and Visor ends each TCP connection and UDP flow on sockets of the phone's own, IPv4 and IPv6. Each connection goes in the Requests page, named by its TLS server where it has one. DNS, the Display's connectivity probe and Instagram's MQTT connection get through; Instagram itself still says it has no connection. --- .../macedonga/visor/connection/DataFetcher.kt | 2 +- .../visor/connection/InternetSwitch.kt | 37 +- .../macedonga/visor/connection/IpPackets.kt | 175 ++++++++ .../visor/connection/PhoneInternet.kt | 380 ++++++++++++++++++ .../macedonga/visor/connection/RequestLog.kt | 5 + .../macedonga/visor/di/ConnectionModule.kt | 2 + .../macedonga/visor/log/RequestLogScreen.kt | 2 +- .../main/res/values-it/strings_requests.xml | 1 + app/src/main/res/values/strings_requests.xml | 1 + 9 files changed, 599 insertions(+), 6 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/IpPackets.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt b/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt index d60c4409..b024c059 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt @@ -62,7 +62,7 @@ class DataFetcher @Inject constructor(private val log: Provider, priva var sent = 0L val scheme = q.url.substringBefore(':').lowercase() if (scheme != "http" && scheme != "https") { - requests.finish(id, "refused, no $scheme here", failed = true) + requests.finish(id, "not served: $scheme links", failed = true) mux.sendStatus(reply, UNKNOWN_TYPE) return } diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt index d5b7f23d..bbef615e 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt @@ -2,32 +2,61 @@ package com.macedonga.visor.connection import android.content.Context import android.net.ConnectivityManager +import android.net.Network import android.net.NetworkCapabilities +import android.net.LinkProperties +import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.protocol.Wire import com.macedonga.visor.core.session.DataXConnection import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.core.session.Responder import com.macedonga.visor.log.EventLog import dagger.hilt.android.qualifiers.ApplicationContext +import java.util.concurrent.atomic.AtomicBoolean import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton /** - * svc41108, internet_switch: the Display asks how the phone's internet is (type 6) and is told (type 3): connected, - * metered, and the phone's DNS servers. Told it is connected, the Display may carry its apps' traffic to the phone. + * svc41108, internet_switch: how the phone's internet is (type 3): connected, metered, and its DNS servers. The Display + * asks on a channel of its own (type 6), but takes the state only on the phone's channel, which the phone opens when the + * Display asks and writes again whenever its network changes. The Display keeps its apps off the phone's internet until + * it hears the phone is connected. */ @Singleton class InternetSwitch @Inject constructor( @ApplicationContext private val context: Context, + private val connection: Provider, private val log: Provider, ) : Responder { + private val watching = AtomicBoolean(false) - override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + override suspend fun respond(message: DataXMessage, mux: DataXConnection) { if (message.info.messageType != MT_LISTEN) return val state = phoneInternet() log.get().add("Internet: the Display asked; the phone is ${if (state.connected) "online" else "offline"}" + if (state.metered) ", metered" else "") - connection.send(DataXConnection.mirror(message.info.channelId), encode(state), messageType = MT_COMPANION_STATE) + mux.send(DataXConnection.mirror(message.info.channelId), encode(state), messageType = MT_COMPANION_STATE) + mux.sendOnService(SVC, MT_COMPANION_STATE, encode(state)) + watchNetwork() + } + + /** From the first question on, every change of the phone's network goes to the Display while it is connected. */ + private fun watchNetwork() { + if (!watching.compareAndSet(false, true)) return + val cm = context.getSystemService(ConnectivityManager::class.java) ?: return + cm.registerDefaultNetworkCallback(object : ConnectivityManager.NetworkCallback() { + override fun onCapabilitiesChanged(network: Network, caps: NetworkCapabilities) = push() + override fun onLinkPropertiesChanged(network: Network, props: LinkProperties) = push() + override fun onLost(network: Network) = push() + }) + } + + private fun push() { + val glasses = connection.get() + if (glasses.state.value.link != LinkState.READY) return + val state = encode(phoneInternet()) + glasses.submit { it.sendOnService(SVC, MT_COMPANION_STATE, state) } } private data class PhoneInternet(val connected: Boolean, val metered: Boolean, val dns: List) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/IpPackets.kt b/app/src/main/kotlin/com/macedonga/visor/connection/IpPackets.kt new file mode 100644 index 00000000..a3912fd3 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/IpPackets.kt @@ -0,0 +1,175 @@ +package com.macedonga.visor.connection + +import java.net.InetAddress +import java.nio.ByteBuffer + +/** One IP packet as the Display's tunnel carries it: version 4 or 6, TCP or UDP. */ +class IpPacket( + val version: Int, + val src: ByteArray, + val dst: ByteArray, + val protocol: Int, + val transport: ByteArray, +) { + val srcAddress: InetAddress get() = InetAddress.getByAddress(src) + val dstAddress: InetAddress get() = InetAddress.getByAddress(dst) + + companion object { + const val TCP = 6 + const val UDP = 17 + + fun parse(b: ByteArray): IpPacket? { + if (b.isEmpty()) return null + return when (b[0].toInt() ushr 4) { + 4 -> { + if (b.size < 20) return null + val ihl = (b[0].toInt() and 0x0F) * 4 + val total = u16(b, 2).coerceAtMost(b.size) + if (ihl < 20 || total < ihl) return null + // Fragments are not expected inside a 1400-byte tunnel; they are dropped. + if (u16(b, 6) and 0x3FFF != 0) return null + IpPacket(4, b.copyOfRange(12, 16), b.copyOfRange(16, 20), b[9].toInt() and 0xFF, b.copyOfRange(ihl, total)) + } + 6 -> { + if (b.size < 40) return null + val length = u16(b, 4) + // Extension headers are not followed; such packets are dropped. + IpPacket(6, b.copyOfRange(8, 24), b.copyOfRange(24, 40), b[6].toInt() and 0xFF, b.copyOfRange(40, (40 + length).coerceAtMost(b.size))) + } + else -> null + } + } + + /** An IP packet around [transport], its checksums filled in. */ + fun build(version: Int, src: ByteArray, dst: ByteArray, protocol: Int, transport: ByteArray, id: Int): ByteArray { + val sum = checksum(pseudoHeader(version, src, dst, protocol, transport.size), transport) + val at = if (protocol == TCP) 16 else 6 + transport[at] = (sum ushr 8).toByte() + transport[at + 1] = sum.toByte() + return if (version == 4) { + val h = ByteBuffer.allocate(20) + h.put(0x45).put(0).putShort((20 + transport.size).toShort()).putShort(id.toShort()).putShort(0x4000) + h.put(64).put(protocol.toByte()).putShort(0).put(src).put(dst) + val header = h.array() + val hs = checksum(ByteArray(0), header) + header[10] = (hs ushr 8).toByte() + header[11] = hs.toByte() + header + transport + } else { + val h = ByteBuffer.allocate(40) + h.putInt(0x60000000).putShort(transport.size.toShort()).put(protocol.toByte()).put(64).put(src).put(dst) + h.array() + transport + } + } + + private fun pseudoHeader(version: Int, src: ByteArray, dst: ByteArray, protocol: Int, length: Int): ByteArray = + if (version == 4) { + ByteBuffer.allocate(12).put(src).put(dst).put(0).put(protocol.toByte()).putShort(length.toShort()).array() + } else { + ByteBuffer.allocate(40).put(src).put(dst).putInt(length).put(0).put(0).put(0).put(protocol.toByte()).array() + } + + /** The Internet checksum over [a] then [b], the checksum field in [b] taken as zero beforehand. */ + private fun checksum(a: ByteArray, b: ByteArray): Int { + var sum = 0L + fun add(x: ByteArray) { + var i = 0 + while (i + 1 < x.size) { + sum += ((x[i].toInt() and 0xFF) shl 8) or (x[i + 1].toInt() and 0xFF) + i += 2 + } + if (i < x.size) sum += (x[i].toInt() and 0xFF) shl 8 + } + add(a) + add(b) + while (sum ushr 16 != 0L) sum = (sum and 0xFFFF) + (sum ushr 16) + return (sum.inv() and 0xFFFF).toInt() + } + + fun u16(b: ByteArray, at: Int) = ((b[at].toInt() and 0xFF) shl 8) or (b[at + 1].toInt() and 0xFF) + + fun u32(b: ByteArray, at: Int): Long = + ((b[at].toLong() and 0xFF) shl 24) or ((b[at + 1].toLong() and 0xFF) shl 16) or ((b[at + 2].toLong() and 0xFF) shl 8) or (b[at + 3].toLong() and 0xFF) + } +} + +/** A TCP segment's header fields and its data. */ +class TcpSegment( + val srcPort: Int, + val dstPort: Int, + val seq: Long, + val ack: Long, + val flags: Int, + val window: Int, + val data: ByteArray, +) { + val syn get() = flags and SYN != 0 + val fin get() = flags and FIN != 0 + val rst get() = flags and RST != 0 + val hasAck get() = flags and ACK != 0 + + companion object { + const val FIN = 0x01 + const val SYN = 0x02 + const val RST = 0x04 + const val PSH = 0x08 + const val ACK = 0x10 + + fun parse(t: ByteArray): TcpSegment? { + if (t.size < 20) return null + val offset = ((t[12].toInt() and 0xF0) ushr 4) * 4 + if (offset < 20 || offset > t.size) return null + return TcpSegment( + IpPacket.u16(t, 0), IpPacket.u16(t, 2), IpPacket.u32(t, 4), IpPacket.u32(t, 8), + t[13].toInt() and 0x3F, IpPacket.u16(t, 14), t.copyOfRange(offset, t.size), + ) + } + + /** A segment, its checksum left for [IpPacket.build]; [mss] adds the option, for a SYN. */ + fun build(srcPort: Int, dstPort: Int, seq: Long, ack: Long, flags: Int, window: Int, data: ByteArray, mss: Int = 0): ByteArray { + val header = if (mss > 0) 24 else 20 + val b = ByteBuffer.allocate(header + data.size) + b.putShort(srcPort.toShort()).putShort(dstPort.toShort()).putInt(seq.toInt()).putInt(ack.toInt()) + b.put(((header / 4) shl 4).toByte()).put(flags.toByte()).putShort(window.toShort()).putShort(0).putShort(0) + if (mss > 0) b.put(2).put(4).putShort(mss.toShort()) + b.put(data) + return b.array() + } + } +} + +/** UDP: ports and data. */ +class UdpDatagram(val srcPort: Int, val dstPort: Int, val data: ByteArray) { + companion object { + fun parse(u: ByteArray): UdpDatagram? { + if (u.size < 8) return null + val length = IpPacket.u16(u, 4).coerceIn(8, u.size) + return UdpDatagram(IpPacket.u16(u, 0), IpPacket.u16(u, 2), u.copyOfRange(8, length)) + } + + fun build(srcPort: Int, dstPort: Int, data: ByteArray): ByteArray = + ByteBuffer.allocate(8 + data.size).putShort(srcPort.toShort()).putShort(dstPort.toShort()) + .putShort((8 + data.size).toShort()).putShort(0).put(data).array() + } +} + +/** The server name a TLS ClientHello asks for, if [hello] is one. */ +fun tlsServerName(hello: ByteArray): String? = runCatching { + if (hello.size < 43 || hello[0].toInt() != 0x16 || hello[5].toInt() != 0x01) return null + var i = 43 + i += 1 + (hello[i].toInt() and 0xFF) + i += 2 + IpPacket.u16(hello, i) + i += 1 + (hello[i].toInt() and 0xFF) + val end = i + 2 + IpPacket.u16(hello, i) + i += 2 + while (i + 4 <= end && i + 4 <= hello.size) { + val type = IpPacket.u16(hello, i) + val len = IpPacket.u16(hello, i + 2) + if (type == 0) { + val nameLen = IpPacket.u16(hello, i + 7) + return String(hello, i + 9, nameLen, Charsets.US_ASCII) + } + i += 4 + len + } + null +}.getOrNull() diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt new file mode 100644 index 00000000..5fdddc5a --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt @@ -0,0 +1,380 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import java.io.InputStream +import java.io.OutputStream +import java.net.DatagramPacket +import java.net.DatagramSocket +import java.net.InetSocketAddress +import java.net.Socket +import java.net.SocketTimeoutException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlin.random.Random +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withTimeoutOrNull + +/** + * svc41111, internet_proxy_remote: the Display's apps reach the internet through the phone. The Display's tunnel sends + * each IP packet as one message (type 1), framed `1c fb`, its length, its ethertype and two spare bytes; replies go + * back the same way on the same channel. The phone ends the connections itself: each TCP connection from the Display + * becomes a socket of the phone's, each UDP flow a datagram socket, so the Display's traffic is the phone's own. + */ +@Singleton +class PhoneInternet @Inject constructor( + private val requests: RequestLog, + private val log: Provider, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val sessions = ConcurrentHashMap() + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val info = message.info + val channel = info.channelId + if (info.flags.isClosed && message.payload.isEmpty()) { + sessions.remove(channel)?.close() + return + } + when (info.messageType) { + MT_START -> { + sessions.remove(channel)?.close() + sessions[channel] = Session(connection, DataXConnection.mirror(channel)) + log.get().add("Internet: the Display's apps are using the phone's internet") + } + MT_FRAME -> sessions.getOrPut(channel) { Session(connection, DataXConnection.mirror(channel)) }.inbound(message.payload) + } + } + + /** One tunnel from the Display: its TCP and UDP flows, and the way back. */ + private inner class Session(private val mux: DataXConnection, private val reply: Int) { + private val tcp = ConcurrentHashMap() + private val udp = ConcurrentHashMap() + private val ids = AtomicInteger(Random.nextInt(0xFFFF)) + + fun inbound(frame: ByteArray) { + if (frame.size < ICEE_HEADER) return + val head = ByteBuffer.wrap(frame).order(ByteOrder.LITTLE_ENDIAN) + if (head.getShort(0).toInt() and 0xFFFF != ICEE_MAGIC) return + val length = head.getShort(2).toInt() and 0xFFFF + val packet = IpPacket.parse(frame.copyOfRange(ICEE_HEADER, (ICEE_HEADER + length).coerceAtMost(frame.size))) ?: return + when (packet.protocol) { + IpPacket.TCP -> { + val seg = TcpSegment.parse(packet.transport) ?: return + val key = key(packet, seg.srcPort, seg.dstPort) + val flow = tcp[key] ?: if (seg.syn && !seg.hasAck) TcpFlow(this, packet, seg).also { tcp[key] = it } else { + // A segment for a connection the phone does not know: tell the Display it is gone. + if (!seg.rst) resetFor(packet, seg) + return + } + flow.inbox.trySend(seg) + } + IpPacket.UDP -> { + val d = UdpDatagram.parse(packet.transport) ?: return + val key = key(packet, d.srcPort, d.dstPort) + udp.getOrPut(key) { UdpFlow(this, packet, d.srcPort, d.dstPort, key) }.send(d.data) + } + } + } + + fun forget(key: String) { + tcp.remove(key) + udp.remove(key) + } + + /** An IP packet to the Display, framed for its tunnel. */ + suspend fun deliver(version: Int, src: ByteArray, dst: ByteArray, protocol: Int, transport: ByteArray) { + val ip = IpPacket.build(version, src, dst, protocol, transport, ids.incrementAndGet()) + val frame = ByteBuffer.allocate(ICEE_HEADER + ip.size).order(ByteOrder.LITTLE_ENDIAN) + .putShort(ICEE_MAGIC.toShort()).putShort(ip.size.toShort()) + .order(ByteOrder.BIG_ENDIAN).putShort(if (version == 4) ETHER_IPV4 else ETHER_IPV6).putShort(0) + .put(ip).array() + mux.send(reply, frame, messageType = MT_FRAME) + } + + private fun resetFor(p: IpPacket, seg: TcpSegment) { + val ack = (seg.seq + seg.data.size + (if (seg.syn) 1 else 0) + (if (seg.fin) 1 else 0)) and MASK + val rst = TcpSegment.build(seg.dstPort, seg.srcPort, if (seg.hasAck) seg.ack else 0, ack, TcpSegment.RST or TcpSegment.ACK, 0, ByteArray(0)) + scope.launch { runCatching { deliver(p.version, p.dst, p.src, IpPacket.TCP, rst) } } + } + + fun close() { + tcp.values.forEach { it.close() } + udp.values.forEach { it.close() } + tcp.clear() + udp.clear() + } + } + + /** One TCP connection from the Display, carried on a socket of the phone's. */ + private inner class TcpFlow(private val s: Session, p: IpPacket, syn: TcpSegment) { + val inbox = Channel(Channel.UNLIMITED) + private val version = p.version + private val display = p.src + private val remote = p.dst + private val remoteAddress = p.dstAddress + private val displayPort = syn.srcPort + private val remotePort = syn.dstPort + private val key = key(p, syn.srcPort, syn.dstPort) + private val mss = if (version == 4) MSS_V4 else MSS_V6 + private val lock = Mutex() + private val progress = MutableStateFlow(0L) + private val logId = requests.start("tcp://${p.dstAddress.hostAddress}:$remotePort", SOURCE_TUNNEL) + + private var socket: Socket? = null + private var output: OutputStream? = null + private val iss = Random.nextLong(0, MASK) + private var sndUna = iss + private var sndNxt = iss + private var rcvNxt = (syn.seq + 1) and MASK + private var peerWindow = syn.window + private var pending = ByteArray(0) + private var established = false + private var peerFin = false + private var finSent = false + private var named = false + private var bytes = 0L + private val job: Job = scope.launch { run() } + + private suspend fun run() { + val connected = runCatching { + runInterruptible { Socket().apply { connect(InetSocketAddress(remoteAddress, remotePort), CONNECT_MS); tcpNoDelay = true } } + }.getOrElse { + send(TcpSegment.RST or TcpSegment.ACK, 0, ByteArray(0)) + return finish("unreachable", failed = true) + } + socket = connected + output = connected.getOutputStream() + lock.withLock { synAck() } + while (true) { + val seg = withTimeoutOrNull(IDLE_MS) { inbox.receive() } ?: return finish("idle") + if (seg.rst) return finish("reset by the Display") + if (seg.syn) { + lock.withLock { synAck() } + continue + } + if (seg.hasAck) onAck(seg) + if (seg.data.isNotEmpty() || seg.fin) onData(seg) + if (peerFin && finSent && sndUna == sndNxt) return finish("closed") + } + } + + private suspend fun synAck() { + sndNxt = (iss + 1) and MASK + send(TcpSegment.SYN or TcpSegment.ACK, iss, ByteArray(0), mss) + } + + private suspend fun onAck(seg: TcpSegment) { + lock.withLock { + val acked = diff(seg.ack, sndUna) + if (acked > 0 && diff(seg.ack, sndNxt) <= 0) { + val dataAcked = acked.coerceAtMost(pending.size.toLong()).toInt() + pending = pending.copyOfRange(dataAcked, pending.size) + sndUna = seg.ack + } + peerWindow = seg.window + if (!established && diff(seg.ack, iss) > 0) { + established = true + scope.launch { pump() } + } + } + progress.value++ + } + + private suspend fun onData(seg: TcpSegment) { + if (seg.seq != rcvNxt) { + // Out of order or a repeat: the last good point again, and the Display sends it anew. + lock.withLock { send(TcpSegment.ACK, sndNxt, ByteArray(0)) } + return + } + if (seg.data.isNotEmpty()) { + if (!named) { + named = true + tlsServerName(seg.data)?.let { requests.rename(logId, "tls://$it:$remotePort") } + } + val ok = runCatching { runInterruptible { output?.write(seg.data); output?.flush() } }.isSuccess + if (!ok) { + lock.withLock { send(TcpSegment.RST or TcpSegment.ACK, sndNxt, ByteArray(0)) } + finish("the server went away", failed = true) + return + } + bytes += seg.data.size + } + lock.withLock { + rcvNxt = (rcvNxt + seg.data.size) and MASK + if (seg.fin && !peerFin) { + peerFin = true + rcvNxt = (rcvNxt + 1) and MASK + runCatching { socket?.shutdownOutput() } + } + send(TcpSegment.ACK, sndNxt, ByteArray(0)) + } + } + + /** The server's bytes to the Display, as much as its window lets through, again when it goes unanswered. */ + private suspend fun pump() { + val input: InputStream = socket?.getInputStream() ?: return + val buffer = ByteArray(mss) + try { + while (scope.isActive) { + val n = runInterruptible { input.read(buffer) } + if (n < 0) break + if (n == 0) continue + waitForWindow(n) + lock.withLock { + val data = buffer.copyOf(n) + send(TcpSegment.ACK or TcpSegment.PSH, sndNxt, data) + pending += data + sndNxt = (sndNxt + n) and MASK + } + bytes += n + } + } catch (_: Exception) { + } + waitForWindow(0) + lock.withLock { + if (!finSent) { + finSent = true + send(TcpSegment.FIN or TcpSegment.ACK, sndNxt, ByteArray(0)) + sndNxt = (sndNxt + 1) and MASK + } + } + } + + private suspend fun waitForWindow(n: Int) { + while (true) { + val (inFlight, window, last) = lock.withLock { Triple(pending.size, peerWindow, progress.value) } + if (inFlight + n <= window.coerceAtLeast(mss) && (n > 0 || inFlight == 0)) return + if (withTimeoutOrNull(RETRANSMIT_MS) { progress.first { it != last } } == null) retransmit() + if (!job.isActive) return + } + } + + private suspend fun retransmit() { + lock.withLock { + var at = 0 + var seq = sndUna + while (at < pending.size) { + val end = minOf(pending.size, at + mss) + send(TcpSegment.ACK or TcpSegment.PSH, seq, pending.copyOfRange(at, end)) + seq = (seq + (end - at)) and MASK + at = end + } + } + } + + private suspend fun send(flags: Int, seq: Long, data: ByteArray, mssOption: Int = 0) { + val t = TcpSegment.build(remotePort, displayPort, seq, rcvNxt, flags, WINDOW, data, mssOption) + runCatching { s.deliver(version, remote, display, IpPacket.TCP, t) } + } + + private fun finish(outcome: String, failed: Boolean = false) { + requests.finish(logId, outcome, failed, bytes) + close() + } + + fun close() { + s.forget(key) + runCatching { socket?.close() } + inbox.close() + job.cancel() + } + } + + /** One UDP flow from the Display: a datagram socket, kept while it is used. */ + private inner class UdpFlow(private val s: Session, p: IpPacket, private val displayPort: Int, private val remotePort: Int, private val key: String) { + private val version = p.version + private val display = p.src + private val remote = p.dst + private val socket = DatagramSocket().apply { + connect(InetSocketAddress(p.dstAddress, remotePort)) + soTimeout = POLL_MS + } + private val logId = requests.start("udp://${p.dstAddress.hostAddress}:$remotePort", SOURCE_TUNNEL) + @Volatile private var lastUsed = System.currentTimeMillis() + private var bytes = 0L + private val job = scope.launch { receive() } + + fun send(data: ByteArray) { + lastUsed = System.currentTimeMillis() + bytes += data.size + scope.launch { runCatching { socket.send(DatagramPacket(data, data.size)) } } + } + + private suspend fun receive() { + val buffer = ByteArray(UDP_MAX) + while (scope.isActive && System.currentTimeMillis() - lastUsed < UDP_IDLE_MS) { + val packet = DatagramPacket(buffer, buffer.size) + try { + runInterruptible { socket.receive(packet) } + } catch (_: SocketTimeoutException) { + continue + } catch (_: Exception) { + break + } + lastUsed = System.currentTimeMillis() + bytes += packet.length + val u = UdpDatagram.build(remotePort, displayPort, buffer.copyOf(packet.length)) + runCatching { s.deliver(version, remote, display, IpPacket.UDP, u) } + } + requests.finish(logId, "done", failed = false, bytes = bytes) + close() + } + + fun close() { + s.forget(key) + socket.close() + job.cancel() + } + } + + private fun key(p: IpPacket, srcPort: Int, dstPort: Int) = "${p.protocol}/${p.srcAddress.hostAddress}:$srcPort>${p.dstAddress.hostAddress}:$dstPort" + + /** a - b in sequence space, as a signed 32-bit distance. */ + private fun diff(a: Long, b: Long): Long { + val d = (a - b) and MASK + return if (d >= 0x80000000L) d - 0x100000000L else d + } + + companion object { + const val SVC = 41111 + private const val MT_FRAME = 1 + private const val MT_START = 3 + private const val ICEE_HEADER = 8 + private const val ICEE_MAGIC = 0xFB1C + private const val ETHER_IPV4: Short = 0x0800 + private const val ETHER_IPV6: Short = 0x86DD.toShort() + private const val MASK = 0xFFFFFFFFL + // The Display's tunnel has an MTU of 1400. + private const val MSS_V4 = 1360 + private const val MSS_V6 = 1340 + private const val WINDOW = 65535 + private const val CONNECT_MS = 10_000 + private const val IDLE_MS = 600_000L + private const val RETRANSMIT_MS = 1_500L + private const val POLL_MS = 1_000 + private const val UDP_MAX = 2048 + private const val UDP_IDLE_MS = 60_000L + // The request log's "source" for traffic of the tunnel rather than of the data fetcher. + private const val SOURCE_TUNNEL = -1 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt b/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt index 6533fb3f..36e340ea 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt @@ -41,6 +41,11 @@ class RequestLog @Inject constructor() { } } + /** A better name for what an entry reached, such as the server a TLS connection asked for. */ + fun rename(id: Long, url: String) { + _items.update { list -> list.map { if (it.id == id) it.copy(url = url) else it } } + } + fun clear() { _items.value = emptyList() } diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 3d22500b..c9987915 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -143,6 +143,7 @@ object ConnectionModule { icons: NotifyIcons, internet: com.macedonga.visor.connection.InternetSwitch, fetcher: com.macedonga.visor.connection.DataFetcher, + tunnel: com.macedonga.visor.connection.PhoneInternet, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -155,6 +156,7 @@ object ConnectionModule { GlassesNotification.SVC to icons, com.macedonga.visor.connection.InternetSwitch.SVC to internet, com.macedonga.visor.connection.DataFetcher.SVC to fetcher, + com.macedonga.visor.connection.PhoneInternet.SVC to tunnel, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), diff --git a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt index c347d8c5..c8c047c3 100644 --- a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt @@ -88,7 +88,7 @@ private fun RequestRow(r: GlassesRequest) { outcome, r.bytes.takeIf { it > 0 }?.let(::size), r.millis.takeIf { r.outcome != null }?.let { "$it ms" }, - stringResource(R.string.requests_source, r.source), + if (r.source < 0) stringResource(R.string.requests_tunnel) else stringResource(R.string.requests_source, r.source), r.options.ifEmpty { null }, ) Text(parts.joinToString(" · "), style = MONO, color = if (r.failed) colors.signal else colors.textTertiary, modifier = Modifier.padding(top = 2.dp)) diff --git a/app/src/main/res/values-it/strings_requests.xml b/app/src/main/res/values-it/strings_requests.xml index 424510a1..fde8a635 100644 --- a/app/src/main/res/values-it/strings_requests.xml +++ b/app/src/main/res/values-it/strings_requests.xml @@ -5,4 +5,5 @@ Link copiato in corso origine %1$d + tunnel diff --git a/app/src/main/res/values/strings_requests.xml b/app/src/main/res/values/strings_requests.xml index 49c68d54..ba5ac886 100644 --- a/app/src/main/res/values/strings_requests.xml +++ b/app/src/main/res/values/strings_requests.xml @@ -5,4 +5,5 @@ Link copied running source %1$d + tunnel From 2b9d763944376171cdcceab1c3c9e3bf2309785a Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 16:44:48 +0200 Subject: [PATCH 32/90] Display: Instagram's Reels, messages and likes work through the phone. The data fetcher serves the other links the Display's apps ask for: instagram://, the web file named by target_url with the headers it carries, from bytes_to_skip on; sgmessaging://whatsapp with encrypted=true, the encrypted file on a whatsapp.net host; sgmessaging://phone, a file of the phone's own messages, a video as its first frame. Messenger's and Instagram's message media and plain WhatsApp media need those apps' logins and are turned down with that reason. The Requests page counts a request repeated with the same end on one row, and the event log no longer lists every tunnel packet and fetch. --- .../macedonga/visor/connection/DataFetcher.kt | 142 ++++++++++++++++-- .../macedonga/visor/connection/RequestLog.kt | 16 +- .../com/macedonga/visor/log/EventLog.kt | 5 +- .../macedonga/visor/log/RequestLogScreen.kt | 1 + 4 files changed, 150 insertions(+), 14 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt b/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt index b024c059..75b17ae8 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt @@ -1,12 +1,18 @@ package com.macedonga.visor.connection +import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory +import android.media.MediaMetadataRetriever +import android.net.Uri +import android.util.Base64 +import org.json.JSONObject import com.macedonga.visor.core.protocol.Wire import com.macedonga.visor.core.session.DataXConnection import com.macedonga.visor.core.session.DataXMessage import com.macedonga.visor.core.session.Responder import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext import java.io.ByteArrayOutputStream import java.io.InputStream import java.net.HttpURLConnection @@ -30,10 +36,18 @@ import kotlinx.coroutines.sync.withPermit /** * svc9000, the data fetcher: the Display's apps (Instagram, messaging, the prompter) ask the phone for a URL and get * its bytes back on the same channel, a ResponseData {1 data, 2 compressed, 3 total} per chunk, then an OK status. - * Plain http(s) streams; with an image transformation the whole image is scaled first. Other schemes are refused. + * Plain http(s) streams; with an image transformation the whole image is scaled first. An `instagram://` link wraps a + * web one: `target_url`, with `headers` (base64 JSON) inside it, and `bytes_to_skip`. `sgmessaging://` is message + * media: `whatsapp` with `encrypted=true` is the encrypted file on a whatsapp.net host, which the Display decrypts; + * `phone` is a content URI of the phone's own messages. Messenger's, Instagram's and plain WhatsApp media need those + * apps' own logins and are refused, as are other schemes. */ @Singleton -class DataFetcher @Inject constructor(private val log: Provider, private val requests: RequestLog) : Responder { +class DataFetcher @Inject constructor( + @ApplicationContext private val context: Context, + private val log: Provider, + private val requests: RequestLog, +) : Responder { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private val running = ConcurrentHashMap() private val slots = Semaphore(PARALLEL) @@ -61,13 +75,53 @@ class DataFetcher @Inject constructor(private val log: Provider, priva val id = requests.start(q.url, q.source, q.options()) var sent = 0L val scheme = q.url.substringBefore(':').lowercase() - if (scheme != "http" && scheme != "https") { - requests.finish(id, "not served: $scheme links", failed = true) - mux.sendStatus(reply, UNKNOWN_TYPE) - return + val target = when (scheme) { + "http", "https" -> Target(q.url, emptyMap(), 0) + "instagram" -> instagramTarget(q.url) ?: run { + // Instagram's bare check, every few seconds; the Meta app turns it down the same way. + requests.finish(id, "no target_url, turned down", failed = false) + mux.sendStatus(reply, INTERNAL_ERROR) + return + } + "sgmessaging" -> { + val link = Uri.parse(q.url) + val inner = link.getQueryParameter("uri") + when (link.authority) { + "whatsapp" -> { + val host = inner?.let { runCatching { Uri.parse(it) }.getOrNull() } + if (link.getQueryParameter("encrypted") != "true" || host?.scheme != "https" || host.host?.endsWith(".whatsapp.net") != true) { + requests.finish(id, "not served: needs WhatsApp's own login", failed = true) + mux.sendStatus(reply, INTERNAL_ERROR) + return + } + Target(inner, emptyMap(), 0) + } + "phone" -> { + if (inner == null) { + requests.finish(id, "no uri in the link", failed = true) + mux.sendStatus(reply, INTERNAL_ERROR) + } else { + requests.rename(id, inner) + servePhone(Uri.parse(inner), link.getQueryParameter("mimeType").orEmpty(), id, reply, mux) + } + return + } + else -> { + requests.finish(id, "not served: needs ${link.authority}'s own login", failed = true) + mux.sendStatus(reply, INTERNAL_ERROR) + return + } + } + } + else -> { + requests.finish(id, "not served: $scheme links", failed = true) + mux.sendStatus(reply, UNKNOWN_TYPE) + return + } } + if (target.url != q.url) requests.rename(id, target.url) val chunk = (if (q.chunkSize > 0) q.chunkSize else DEFAULT_CHUNK).coerceAtMost(MAX_CHUNK) - val c = runCatching { open(q.url) }.getOrElse { + val c = runCatching { open(target) }.getOrElse { log.get().add("Fetch: ${host(q.url)} failed: ${it.message}") requests.finish(id, "failed: ${it.message}", failed = true) mux.sendStatus(reply, INTERNAL_ERROR) @@ -82,7 +136,10 @@ class DataFetcher @Inject constructor(private val log: Provider, priva return } if (q.scale == null) { - sent = stream(c.inputStream, c.contentLengthLong, chunk, reply, mux) + // From a byte on: a ranged answer says the whole length after its slash; a full one is skipped through. + val ranged = code == 206 + val total = if (ranged) c.getHeaderField("Content-Range")?.substringAfterLast('/')?.toLongOrNull() ?: -1 else c.contentLengthLong + sent = stream(c.inputStream, total, chunk, reply, mux, skip = if (ranged) 0 else target.skip) } else { var body = c.inputStream.use { it.readBytes() } body = scaled(body, q.scale) ?: body @@ -94,7 +151,7 @@ class DataFetcher @Inject constructor(private val log: Provider, priva sent = body.size.toLong() } mux.sendStatus(reply, OK) - requests.finish(id, "HTTP $code" + if (q.scale != null) ", scaled" else "", failed = false, bytes = sent) + requests.finish(id, "HTTP $code" + (if (q.scale != null) ", scaled" else "") + (if (scheme == "instagram") ", for instagram://" else ""), failed = false, bytes = sent) } catch (error: kotlinx.coroutines.CancellationException) { requests.finish(id, "cancelled", failed = false, bytes = sent) throw error @@ -107,10 +164,45 @@ class DataFetcher @Inject constructor(private val log: Provider, priva } } - /** The body as it comes, a chunk per message; how much went. */ - private suspend fun stream(input: InputStream, total: Long, chunk: Int, reply: Int, mux: DataXConnection): Long { + /** A file of the phone's own messages: a video as its first frame, anything else as it is. */ + private suspend fun servePhone(uri: Uri, mime: String, id: Long, reply: Int, mux: DataXConnection) { + try { + val sent = if (mime.startsWith("video/")) { + val frame = MediaMetadataRetriever().run { + try { + setDataSource(context, uri) + getFrameAtTime(0) + } finally { + release() + } + } ?: error("no frame in the video") + val jpeg = ByteArrayOutputStream().also { frame.compress(Bitmap.CompressFormat.JPEG, DEFAULT_QUALITY, it) }.toByteArray() + for (at in jpeg.indices step DEFAULT_CHUNK) send(mux, reply, jpeg.copyOfRange(at, minOf(jpeg.size, at + DEFAULT_CHUNK)), jpeg.size.toLong(), false) + jpeg.size.toLong() + } else { + val input = context.contentResolver.openInputStream(uri) ?: error("it could not be opened") + stream(input, -1, DEFAULT_CHUNK, reply, mux) + } + mux.sendStatus(reply, OK) + requests.finish(id, if (mime.startsWith("video/")) "first frame" else "read", failed = false, bytes = sent) + } catch (error: kotlinx.coroutines.CancellationException) { + throw error + } catch (error: Exception) { + requests.finish(id, "failed: ${error.message}", failed = true) + runCatching { mux.sendStatus(reply, INTERNAL_ERROR) } + } + } + + /** The body as it comes after its first [skip] bytes, a chunk per message; how much went. */ + private suspend fun stream(input: InputStream, total: Long, chunk: Int, reply: Int, mux: DataXConnection, skip: Long = 0): Long { var sent = 0L input.use { + var left = skip + while (left > 0) { + val n = it.skip(left) + if (n <= 0) break + left -= n + } val buffer = ByteArray(chunk) while (true) { kotlin.coroutines.coroutineContext.ensureActive() @@ -130,7 +222,11 @@ class DataFetcher @Inject constructor(private val log: Provider, priva mux.send(reply, body, messageType = MT_REQUEST) } - private fun open(url: String): HttpURLConnection = (URL(url).openConnection() as HttpURLConnection).apply { + private fun open(url: String): HttpURLConnection = open(Target(url, emptyMap(), 0)) + + private fun open(t: Target): HttpURLConnection = (URL(t.url).openConnection() as HttpURLConnection).apply { + for ((k, v) in t.headers) setRequestProperty(k, v) + if (t.skip > 0) setRequestProperty("Range", "bytes=${t.skip}-") connectTimeout = TIMEOUT_MS readTimeout = TIMEOUT_MS instanceFollowRedirects = true @@ -160,6 +256,28 @@ class DataFetcher @Inject constructor(private val log: Provider, priva private fun host(url: String) = runCatching { URL(url).host }.getOrDefault("?") + /** What to fetch, the headers to send with it, and from which byte. */ + private class Target(val url: String, val headers: Map, val skip: Long) + + private fun instagramTarget(link: String): Target? { + val outer = runCatching { Uri.parse(link) }.getOrNull() ?: return null + val inner = outer.getQueryParameter("target_url")?.takeIf { it.isNotEmpty() } ?: return null + val skip = outer.getQueryParameter("bytes_to_skip")?.toLongOrNull() ?: 0 + val innerUri = Uri.parse(inner) + val headers = innerUri.getQueryParameter("headers")?.let { encoded -> + runCatching { + val json = JSONObject(String(Base64.decode(encoded, Base64.NO_WRAP))) + json.keys().asSequence().associateWith { json.getString(it) } + }.getOrNull() + }.orEmpty() + val url = if (innerUri.getQueryParameter("headers") == null) inner else { + val rebuilt = innerUri.buildUpon().clearQuery() + for (name in innerUri.queryParameterNames) if (name != "headers") innerUri.getQueryParameter(name)?.let { rebuilt.appendQueryParameter(name, it) } + rebuilt.build().toString() + } + return Target(url, headers, skip) + } + private class Scale(val width: Int, val height: Int, val format: Int, val keepAspect: Boolean, val quality: Int) /** RequestQuery: 1 url, 2 compress, 3 transformations (1 image scale), 4 chunk size, 6 who asked. */ diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt b/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt index 36e340ea..c6453eea 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt @@ -20,6 +20,8 @@ data class GlassesRequest( val failed: Boolean = false, val bytes: Long = 0, val millis: Long = 0, + /** How many times in a row the same request ended the same way. */ + val times: Int = 1, ) /** The requests the glasses made through the phone, newest last, in memory only: their links can carry tokens. */ @@ -37,7 +39,19 @@ class RequestLog @Inject constructor() { fun finish(id: Long, outcome: String, failed: Boolean, bytes: Long = 0) { _items.update { list -> - list.map { if (it.id == id) it.copy(outcome = outcome, failed = failed, bytes = bytes, millis = System.currentTimeMillis() - it.at) else it } + val done = list.map { if (it.id == id) it.copy(outcome = outcome, failed = failed, bytes = bytes, millis = System.currentTimeMillis() - it.at) else it } + // The same request ending the same way as the one before it is counted on that one, not listed again. + val at = done.indexOfFirst { it.id == id } + val earlier = done.subList(0, maxOf(at, 0)).lastOrNull { it.url == done[at].url } + if (at > 0 && earlier != null && earlier.outcome == outcome && earlier.failed == failed) { + done.mapNotNull { e -> + when (e.id) { + id -> null + earlier.id -> e.copy(times = e.times + 1, at = done[at].at) + else -> e + } + } + } else done } } diff --git a/app/src/main/kotlin/com/macedonga/visor/log/EventLog.kt b/app/src/main/kotlin/com/macedonga/visor/log/EventLog.kt index 7fd1abb8..a900d52e 100644 --- a/app/src/main/kotlin/com/macedonga/visor/log/EventLog.kt +++ b/app/src/main/kotlin/com/macedonga/visor/log/EventLog.kt @@ -65,7 +65,9 @@ class EventLog @Inject constructor( } scope.launch { connection.serviceData.collect { data -> - // The assistant says what its svc15 traffic means itself. + // The assistant says what its svc15 traffic means itself; the internet tunnel and the data fetcher go to + // the Requests page, a packet or a request at a time being far too many for this log. + if (data.serviceId in QUIET) return@collect if (data.serviceId != 15) add("Message from service ${data.serviceId}, type ${data.messageType}, ${data.payload.size} bytes") // Debug builds also print the bytes, so a service Visor does not answer can be read over adb. if (BuildConfig.DEBUG && data.serviceId !in setOf(15, 20)) { @@ -148,6 +150,7 @@ class EventLog @Inject constructor( private companion object { const val LIMIT = 1000 + val QUIET = setOf(com.macedonga.visor.connection.DataFetcher.SVC, com.macedonga.visor.connection.PhoneInternet.SVC) val TIME: DateTimeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS") } } diff --git a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt index c8c047c3..d995eb78 100644 --- a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt @@ -85,6 +85,7 @@ private fun RequestRow(r: GlassesRequest) { val outcome = r.outcome ?: stringResource(R.string.requests_running) val parts = listOfNotNull( TIME.format(Instant.ofEpochMilli(r.at).atZone(ZoneId.systemDefault())), + r.times.takeIf { it > 1 }?.let { "×$it" }, outcome, r.bytes.takeIf { it > 0 }?.let(::size), r.millis.takeIf { r.outcome != null }?.let { "$it ms" }, From c17d53a488e21909f82a448cb9aceb6aecd4fa4c Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 17:00:36 +0200 Subject: [PATCH 33/90] Link: Visor reconnects in seconds after a restart. The L2CAP channel opens on the usual PSM at once, its number read over GATT only when that fails (the read held every connect up to 6 s), and an open the glasses do not answer is given up after 8 s instead of Android's 20 to 30 s, which after Visor died with a channel open cost a minute. The internet tunnel and the data fetcher can no longer crash the app: a connection whose tunnel closed ends quietly, and a flow the phone cannot reach (IPv6 here) is dropped. --- .../macedonga/visor/connection/DataFetcher.kt | 3 +- .../visor/connection/PhoneInternet.kt | 20 ++++++++--- .../core/transport/BluetoothLeTransport.kt | 36 +++++++++++++++++-- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt b/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt index 75b17ae8..ff59d3d8 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt @@ -24,6 +24,7 @@ import javax.inject.Provider import javax.inject.Singleton import kotlin.math.max import kotlin.math.roundToInt +import kotlinx.coroutines.CoroutineExceptionHandler import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -48,7 +49,7 @@ class DataFetcher @Inject constructor( private val log: Provider, private val requests: RequestLog, ) : Responder { - private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Fetch: failed: $e") }) private val running = ConcurrentHashMap() private val slots = Semaphore(PARALLEL) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt index 5fdddc5a..ae38c9e5 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt @@ -19,6 +19,7 @@ import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton import kotlin.random.Random +import kotlinx.coroutines.CoroutineExceptionHandler import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -45,7 +46,8 @@ class PhoneInternet @Inject constructor( private val requests: RequestLog, private val log: Provider, ) : Responder { - private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + // A connection that fails ends alone; nothing in here may take the app down. + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Internet: a connection failed: $e") }) private val sessions = ConcurrentHashMap() override suspend fun respond(message: DataXMessage, connection: DataXConnection) { @@ -91,7 +93,9 @@ class PhoneInternet @Inject constructor( IpPacket.UDP -> { val d = UdpDatagram.parse(packet.transport) ?: return val key = key(packet, d.srcPort, d.dstPort) - udp.getOrPut(key) { UdpFlow(this, packet, d.srcPort, d.dstPort, key) }.send(d.data) + // A flow the phone cannot reach (IPv6 on a phone without it) is dropped; the Display falls back. + val flow = udp[key] ?: runCatching { UdpFlow(this, packet, d.srcPort, d.dstPort, key) }.getOrNull()?.also { udp[key] = it } ?: return + flow.send(d.data) } } } @@ -152,6 +156,7 @@ class PhoneInternet @Inject constructor( private var peerFin = false private var finSent = false private var named = false + @Volatile private var finished = false private var bytes = 0L private val job: Job = scope.launch { run() } @@ -166,7 +171,9 @@ class PhoneInternet @Inject constructor( output = connected.getOutputStream() lock.withLock { synAck() } while (true) { - val seg = withTimeoutOrNull(IDLE_MS) { inbox.receive() } ?: return finish("idle") + val next = withTimeoutOrNull(IDLE_MS) { inbox.receiveCatching() } ?: return finish("idle") + // Closed: the tunnel or the link went, and close() has already let go of everything. + val seg = next.getOrNull() ?: return if (seg.rst) return finish("reset by the Display") if (seg.syn) { lock.withLock { synAck() } @@ -288,15 +295,18 @@ class PhoneInternet @Inject constructor( } private fun finish(outcome: String, failed: Boolean = false) { - requests.finish(logId, outcome, failed, bytes) + if (!finished) requests.finish(logId, outcome, failed, bytes) + finished = true close() } fun close() { + if (!finished) requests.finish(logId, "the tunnel ended", failed = false, bytes = bytes) + finished = true s.forget(key) runCatching { socket?.close() } - inbox.close() job.cancel() + inbox.close() } } diff --git a/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt b/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt index 81941f99..3c807041 100644 --- a/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt +++ b/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt @@ -53,19 +53,48 @@ class BluetoothLeTransport( } val device = context.getSystemService(BluetoothManager::class.java)?.adapter?.getRemoteDevice(address) ?: throw TransportConnectException(ConnectFailure.BLUETOOTH_OFF, "Bluetooth is off") - val psm = keeper?.psm ?: withTimeoutOrNull(GATT_MS) { readPsm(device) } ?: defaultPsm - keeper?.note("opening L2CAP psm $psm, LE link ${if (keeper.connected.value) "up" else "down"}") + // The usual PSM first: reading it over GATT on a fresh link cost up to 6 s; read only when that one fails. + val known = keeper?.psm + val first = known ?: defaultPsm + keeper?.note("opening L2CAP psm $first, LE link ${if (keeper.connected.value) "up" else "down"}") + try { + open(device, first) + } catch (refused: TransportConnectException) { + if (known != null || refused.failure == ConnectFailure.PERMISSION_REFUSED) throw refused + val read = withTimeoutOrNull(GATT_MS) { readPsm(device) } + if (read == null || read == first) throw refused + keeper?.note("opening L2CAP psm $read, as the glasses name it") + open(device, read) + } + } + + @android.annotation.SuppressLint("MissingPermission") + private suspend fun open(device: android.bluetooth.BluetoothDevice, psm: Int) { withContext(io) { val opening = try { device.createInsecureL2capChannel(psm) } catch (error: SecurityException) { throw TransportConnectException(ConnectFailure.PERMISSION_REFUSED, "Visor may not use Bluetooth yet", error) } + // A healthy open takes a fraction of a second; after Visor died with a channel open, Android's own wait is + // 20 to 30 s for nothing. Closing the socket ends the wait. + var timedOut = false + val watchdog = java.util.Timer(true).apply { + schedule(object : java.util.TimerTask() { + override fun run() { + timedOut = true + runCatching { opening.close() } + } + }, L2CAP_OPEN_MS) + } try { opening.connect() } catch (error: IOException) { runCatching { opening.close() } - throw TransportConnectException(ConnectFailure.NO_ANSWER, "the glasses did not open L2CAP psm $psm: ${error.message}", error) + val why = if (timedOut) "no answer in ${L2CAP_OPEN_MS / 1000} s" else error.message + throw TransportConnectException(ConnectFailure.NO_ANSWER, "the glasses did not open L2CAP psm $psm: $why", error) + } finally { + watchdog.cancel() } socket = opening stream = SduStream(opening.inputStream, opening.outputStream, wireLog) @@ -119,6 +148,7 @@ class BluetoothLeTransport( private companion object { const val GATT_MS = 6_000L + const val L2CAP_OPEN_MS = 8_000L } } From 7de8d4f71cc864abad541464f19e8c2c0dffbda1 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 17:00:36 +0200 Subject: [PATCH 34/90] Display: the home's panels no longer wait 10 s for the phone. Visor answers svc9002's request for the phone's panels with an empty list; the Meta app's one panel is a Briefing drawn from Meta's server. --- .../com/macedonga/visor/connection/Panels.kt | 30 +++++++++++++++++++ .../macedonga/visor/di/ConnectionModule.kt | 2 ++ 2 files changed, 32 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/Panels.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/Panels.kt b/app/src/main/kotlin/com/macedonga/visor/connection/Panels.kt new file mode 100644 index 00000000..b227d066 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/Panels.kt @@ -0,0 +1,30 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import javax.inject.Inject +import javax.inject.Singleton + +/** + * svc9002, panels: the Display asks which panels the phone offers for its home (type 1) and waits for the list (type 2), + * timing out after 10 s without it. The Meta app's one panel is a Briefing drawn from Meta's server; Visor offers none, + * so the Display stops waiting. + */ +@Singleton +class Panels @Inject constructor() : Responder { + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val reply = DataXConnection.mirror(message.info.channelId) + when (message.info.messageType) { + MT_GET_ENABLED -> connection.send(reply, ByteArray(0), messageType = MT_ENABLED) + else -> if (!message.info.flags.isClosed) connection.sendStatus(reply, BAD_TYPE) + } + } + + companion object { + const val SVC = 9002 + private const val MT_GET_ENABLED = 1 + private const val MT_ENABLED = 2 + private const val BAD_TYPE = 0xD004 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index c9987915..12c26437 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -144,6 +144,7 @@ object ConnectionModule { internet: com.macedonga.visor.connection.InternetSwitch, fetcher: com.macedonga.visor.connection.DataFetcher, tunnel: com.macedonga.visor.connection.PhoneInternet, + panels: com.macedonga.visor.connection.Panels, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -157,6 +158,7 @@ object ConnectionModule { com.macedonga.visor.connection.InternetSwitch.SVC to internet, com.macedonga.visor.connection.DataFetcher.SVC to fetcher, com.macedonga.visor.connection.PhoneInternet.SVC to tunnel, + com.macedonga.visor.connection.Panels.SVC to panels, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), From 81307dfab641b1b2265d745a3e6530b2a451957c Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 17:34:34 +0200 Subject: [PATCH 35/90] Display: Bloks screens from Meta and the phone's location. Visor answers svc43, the Display's Bloks requests (the Glanceables' weather, sports and stocks details among them): it runs Meta's CompanionBloksComponentQuery or action query with the user's login, with the app id, params and device context the Display sent, and hands back the bundle gzipped; fox_edge apps get an empty answer. It answers svc41102 with the phone's location fixes while the Display asks for them. Both show on the Requests page. Debug API: GRAPHQL runs one query on Meta's ar GraphQL as the user. --- .../macedonga/visor/debug/DebugReceiver.kt | 20 +++ .../macedonga/visor/connection/BloksRelay.kt | 123 ++++++++++++++++++ .../visor/connection/LocationShare.kt | 113 ++++++++++++++++ .../macedonga/visor/di/ConnectionModule.kt | 4 + .../macedonga/visor/log/RequestLogScreen.kt | 7 +- .../main/res/values-it/strings_requests.xml | 2 + app/src/main/res/values/strings_requests.xml | 2 + 7 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/BloksRelay.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/LocationShare.kt diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index 4f9d0cb8..f1c88985 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -64,6 +64,7 @@ class DebugReceiver : BroadcastReceiver() { @Inject lateinit var captionPacks: com.macedonga.visor.display.CaptionPacks @Inject lateinit var screencast: com.macedonga.visor.core.media.Screencast @Inject lateinit var requests: com.macedonga.visor.connection.RequestLog + @Inject lateinit var login: com.macedonga.visor.login.MetaLogin override fun onReceive(context: Context, intent: Intent) { val action = intent.action?.substringAfterLast('.').orEmpty() @@ -344,6 +345,25 @@ class DebugReceiver : BroadcastReceiver() { watch.cancel() heard.joinToString("\n").ifEmpty { "nothing" } } + // --es doc --es name --es vars : one query on Meta's ar GraphQL, as the + // user; the answer's first 4000 characters. + "GRAPHQL" -> answerLater { + val token = login.source.get() + val body = com.macedonga.visor.core.identity.pyUrlencode( + linkedMapOf( + "fb_api_req_friendly_name" to intent.getStringExtra("name").orEmpty(), + "client_doc_id" to intent.getStringExtra("doc").orEmpty(), + "variables" to (intent.getStringExtra("vars64")?.let { String(android.util.Base64.decode(it, 0)) } ?: intent.getStringExtra("vars") ?: "{}"), + "method" to "post", "locale" to "en_US", "pretty" to "false", "format" to "json", + ), + ) + val reply = com.macedonga.visor.core.identity.UrlConnectionPost.post( + "https://ar-genai.graph.meta.com/graphql", + mapOf("Authorization" to "OAuth ${token.value}", "Content-Type" to "application/x-www-form-urlencoded"), + body.toByteArray(), + ) + "HTTP ${reply.status}: " + String(reply.body).take(4000) + } // The request log's newest entries, links cut at their query. "REQUESTS" -> resultData = requests.items.value.takeLast(intent.getIntExtra("n", 30)).joinToString("\n") { "${it.source} ${it.outcome ?: "running"} ${it.bytes}B ${it.millis}ms [${it.options}] ${it.url.substringBefore('?').take(160)}" diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/BloksRelay.kt b/app/src/main/kotlin/com/macedonga/visor/connection/BloksRelay.kt new file mode 100644 index 00000000..7cace127 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/BloksRelay.kt @@ -0,0 +1,123 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.pyUrlencode +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.login.MetaLogin +import java.io.ByteArrayOutputStream +import java.util.zip.GZIPOutputStream +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import org.json.JSONArray +import org.json.JSONObject + +/** + * svc43, smartglasses_graphql_query: the Display's System UI, Utilities and AI History draw some screens from Meta's + * server through the phone, the Glanceables (weather, sports, stocks) among them. A Bloks component request (type 9) or + * action (type 11) names an app id and its params; the phone runs Meta's persisted query with the user's login and + * hands back the bundle gzipped (type 10 or 12). `com.bloks.fox_edge.*` gets an empty answer, as from the Meta app. + */ +@Singleton +class BloksRelay @Inject constructor( + private val login: Provider, + private val requests: RequestLog, + private val log: Provider, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Bloks: failed: $e") }) + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val type = message.info.messageType + val query = QUERIES[type] ?: return + val request = runCatching { Request.decode(message.payload) }.getOrNull() ?: return + val reply = DataXConnection.mirror(message.info.channelId) + scope.launch { serve(request, query, reply, connection) } + } + + private suspend fun serve(r: Request, q: Query, reply: Int, mux: DataXConnection) { + val id = requests.start("bloks://${r.appId}", SOURCE_BLOKS, r.params["params"]?.let { runCatching { JSONObject(it).optString("data_type") }.getOrNull() }.orEmpty()) + if (r.appId.isEmpty() || r.appId.startsWith("com.bloks.fox_edge.")) { + mux.send(reply, ByteArray(0), messageType = q.answer) + requests.finish(id, "empty, as the Meta app answers", failed = false) + return + } + try { + val bundle = fetch(r, q) + val gz = ByteArrayOutputStream().also { out -> GZIPOutputStream(out).use { it.write(bundle.toByteArray()) } }.toByteArray() + mux.send(reply, Wire.fieldBytes(1, gz), messageType = q.answer) + requests.finish(id, "from Meta", failed = false, bytes = gz.size.toLong()) + } catch (error: Exception) { + requests.finish(id, "failed: ${error.message}", failed = true) + runCatching { mux.sendStatus(reply, INTERNAL_ERROR) } + } + } + + private fun fetch(r: Request, q: Query): String { + val token = login.get().source.get() + val params = JSONObject().put("app_id", r.appId).put("params", JSONObject(r.params as Map<*, *>).toString()) + .put("bloks_versioning_id", r.versioningId) + if (r.prebundled) params.put("is_prebundled", true) + val context = JSONObject().put("pixel_ratio", r.pixelRatio).put("bloks_version", r.versioningId) + .put("theme_params", JSONArray().put(JSONObject().put("design_system_name", r.designSystem).put("value", JSONArray(r.themeValues)))) + val variables = JSONObject().put("params", params).put("bk_context", context) + val body = pyUrlencode( + linkedMapOf( + "fb_api_req_friendly_name" to q.name, "client_doc_id" to q.doc, "variables" to variables.toString(), + "method" to "post", "locale" to "en_US", "pretty" to "false", "format" to "json", + ), + ) + val answer = UrlConnectionPost.post(GRAPH_URL, mapOf("Authorization" to "OAuth ${token.value}", "Content-Type" to "application/x-www-form-urlencoded"), body.toByteArray()) + val json = JSONObject(String(answer.body)) + json.optJSONArray("errors")?.let { error(it.optJSONObject(0)?.optString("message") ?: "Meta refused it") } + // data..: the first string under the first object, whatever Meta names them. + val data = json.getJSONObject("data") + val root = data.keys().asSequence().mapNotNull { data.optJSONObject(it) }.firstOrNull() ?: error("no answer in the reply") + return root.keys().asSequence().mapNotNull { k -> root.opt(k) as? String }.firstOrNull { it.startsWith("{") } ?: error("no bundle in the reply") + } + + private class Query(val name: String, val doc: String, val answer: Int) + + /** BloksAsync*Request: 1 device context {1 versioning id, 2 pixel ratio, 3 theme {1 name, 2 values}}, 3 app id, 4 params, 5 prebundled. */ + private class Request( + val appId: String, + val params: Map, + val versioningId: String, + val pixelRatio: Double, + val designSystem: String, + val themeValues: List, + val prebundled: Boolean, + ) { + companion object { + fun decode(payload: ByteArray): Request { + val f = Wire.parse(payload) + val device = Wire.parse(f.bytes(1)) + val theme = Wire.parse(device.bytes(3)) + val params = f.repeatedBytes(4).associate { e -> Wire.parse(e).let { it.bytes(1).decodeToString() to it.bytes(2).decodeToString() } } + return Request( + f.bytes(3).decodeToString(), params, device.bytes(1).decodeToString(), device.double(2, 1.0), + theme.bytes(1).decodeToString(), theme.repeatedBytes(2).map { it.decodeToString() }, f.varint(5) != 0L, + ) + } + } + } + + companion object { + const val SVC = 43 + private const val GRAPH_URL = "https://ar-genai.graph.meta.com/graphql" + private val QUERIES = mapOf( + 9 to Query("CompanionBloksComponentQuery", "37274295535564761374285150560", 10), + 11 to Query("CompanionBloksAsyncActionQuery", "35737516989272377596036696859", 12), + ) + private const val INTERNAL_ERROR = 0xC004 + private const val SOURCE_BLOKS = -2 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/LocationShare.kt b/app/src/main/kotlin/com/macedonga/visor/connection/LocationShare.kt new file mode 100644 index 00000000..15a08e7e --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/LocationShare.kt @@ -0,0 +1,113 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.annotation.SuppressLint +import android.content.Context +import android.content.pm.PackageManager +import android.location.Location +import android.location.LocationListener +import android.location.LocationManager +import android.os.Looper +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import dagger.hilt.android.qualifiers.ApplicationContext +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch + +/** + * svc41102, locationservice: the Display asks for the phone's location (type 3) and gets fixes (type 4, + * LocationResult {1 status, 2 Location}) until it says stop (type 5). Its register message on connect (26) needs no + * answer. Without the location permission the answer says so. + */ +@Singleton +class LocationShare @Inject constructor( + @ApplicationContext private val context: Context, + private val requests: RequestLog, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, _ -> }) + private val listening = ConcurrentHashMap() + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val channel = message.info.channelId + if (message.info.flags.isClosed) { + listening.remove(channel)?.stop() + return + } + when (message.info.messageType) { + MT_REQUEST -> if (!listening.containsKey(channel)) { + val precise = runCatching { Wire.parse(message.payload).varint(3) }.getOrDefault(0L) == ACCURACY_BEST + listening[channel] = Subscription(connection, DataXConnection.mirror(channel), precise).also { it.start() } + } + MT_STOP -> listening.remove(channel)?.stop() + } + } + + private inner class Subscription(private val mux: DataXConnection, private val reply: Int, private val precise: Boolean) : LocationListener { + private val manager = context.getSystemService(LocationManager::class.java) + private val logId = requests.start("location://${if (precise) "precise" else "approximate"}", SOURCE_LOCATION) + private var fixes = 0L + + @SuppressLint("MissingPermission") + fun start() { + val permission = if (precise) Manifest.permission.ACCESS_FINE_LOCATION else Manifest.permission.ACCESS_COARSE_LOCATION + if (manager == null || context.checkSelfPermission(permission) != PackageManager.PERMISSION_GRANTED) { + send(Wire.fieldVarint(1, STATUS_NO_PERMISSION)) + requests.finish(logId, "no location permission", failed = true) + return + } + val provider = when { + manager.allProviders.contains(LocationManager.FUSED_PROVIDER) -> LocationManager.FUSED_PROVIDER + precise && manager.isProviderEnabled(LocationManager.GPS_PROVIDER) -> LocationManager.GPS_PROVIDER + else -> LocationManager.NETWORK_PROVIDER + } + manager.getLastKnownLocation(provider)?.let { onLocationChanged(it) } + manager.requestLocationUpdates(provider, INTERVAL_MS, 0f, this, Looper.getMainLooper()) + } + + fun stop() { + manager?.removeUpdates(this) + requests.finish(logId, "stopped after $fixes fixes", failed = false) + } + + override fun onLocationChanged(location: Location) { + fixes++ + send(Wire.fieldVarint(1, STATUS_OK) + Wire.fieldBytes(2, encode(location))) + } + + private fun send(body: ByteArray) { + scope.launch { runCatching { mux.send(reply, body, messageType = MT_RESULT) } } + } + } + + /** Location: 1 latitude, 2 longitude, 3 accuracy m, 5 time ms, 8 altitude, 9 its accuracy, 10 bearing, 11 its accuracy, 12 speed, 13 its accuracy. */ + private fun encode(l: Location): ByteArray { + var b = Wire.fieldDouble(1, l.latitude) + Wire.fieldDouble(2, l.longitude) + Wire.fieldFloat(3, l.accuracy) + Wire.fieldVarint(5, l.time) + if (l.hasAltitude()) b += Wire.fieldDouble(8, l.altitude) + if (l.hasVerticalAccuracy()) b += Wire.fieldFloat(9, l.verticalAccuracyMeters) + if (l.hasBearing()) b += Wire.fieldFloat(10, l.bearing) + if (l.hasBearingAccuracy()) b += Wire.fieldFloat(11, l.bearingAccuracyDegrees) + if (l.hasSpeed()) b += Wire.fieldFloat(12, l.speed) + if (l.hasSpeedAccuracy()) b += Wire.fieldFloat(13, l.speedAccuracyMetersPerSecond) + return b + } + + companion object { + const val SVC = 41102 + private const val MT_REQUEST = 3 + private const val MT_RESULT = 4 + private const val MT_STOP = 5 + private const val ACCURACY_BEST = 0L + private const val STATUS_OK = 0L + private const val STATUS_NO_PERMISSION = 1L + private const val INTERVAL_MS = 2_000L + private const val SOURCE_LOCATION = -3 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 12c26437..e0a65fea 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -145,6 +145,8 @@ object ConnectionModule { fetcher: com.macedonga.visor.connection.DataFetcher, tunnel: com.macedonga.visor.connection.PhoneInternet, panels: com.macedonga.visor.connection.Panels, + bloks: com.macedonga.visor.connection.BloksRelay, + location: com.macedonga.visor.connection.LocationShare, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -159,6 +161,8 @@ object ConnectionModule { com.macedonga.visor.connection.DataFetcher.SVC to fetcher, com.macedonga.visor.connection.PhoneInternet.SVC to tunnel, com.macedonga.visor.connection.Panels.SVC to panels, + com.macedonga.visor.connection.BloksRelay.SVC to bloks, + com.macedonga.visor.connection.LocationShare.SVC to location, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), diff --git a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt index d995eb78..2aab2a54 100644 --- a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt @@ -89,7 +89,12 @@ private fun RequestRow(r: GlassesRequest) { outcome, r.bytes.takeIf { it > 0 }?.let(::size), r.millis.takeIf { r.outcome != null }?.let { "$it ms" }, - if (r.source < 0) stringResource(R.string.requests_tunnel) else stringResource(R.string.requests_source, r.source), + when (r.source) { + -1 -> stringResource(R.string.requests_tunnel) + -2 -> stringResource(R.string.requests_meta) + -3 -> stringResource(R.string.requests_location) + else -> stringResource(R.string.requests_source, r.source) + }, r.options.ifEmpty { null }, ) Text(parts.joinToString(" · "), style = MONO, color = if (r.failed) colors.signal else colors.textTertiary, modifier = Modifier.padding(top = 2.dp)) diff --git a/app/src/main/res/values-it/strings_requests.xml b/app/src/main/res/values-it/strings_requests.xml index fde8a635..1817dd8d 100644 --- a/app/src/main/res/values-it/strings_requests.xml +++ b/app/src/main/res/values-it/strings_requests.xml @@ -6,4 +6,6 @@ in corso origine %1$d tunnel + Meta + posizione diff --git a/app/src/main/res/values/strings_requests.xml b/app/src/main/res/values/strings_requests.xml index ba5ac886..710b0bdb 100644 --- a/app/src/main/res/values/strings_requests.xml +++ b/app/src/main/res/values/strings_requests.xml @@ -6,4 +6,6 @@ running source %1$d tunnel + Meta + location From 2f383b012c8af4aca594817e5b3447ed119f386c Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 19:36:20 +0200 Subject: [PATCH 36/90] Login: a refresh that cannot reach Meta says so, instead of saying the saved login no longer works and asking to log in again. --- .../kotlin/com/macedonga/visor/core/identity/token/Frla.kt | 2 +- .../kotlin/com/macedonga/visor/core/identity/token/Token.kt | 3 ++- .../com/macedonga/visor/core/identity/token/WebLogin.kt | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Frla.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Frla.kt index cd4bbcdc..21c288c5 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Frla.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Frla.kt @@ -32,7 +32,7 @@ private fun post(url: String, fields: Map, http: HttpPost): Json } catch (error: Exception) { // The interesting answers arrive as 4xx/5xx with a JSON body and are read, not thrown; // only failing to reach the server at all lands here. - throw TokenError("$url: ${error.javaClass.simpleName}: ${error.message}") + throw TokenError("$url: ${error.javaClass.simpleName}: ${error.message}", unreachable = true) } return try { Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Token.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Token.kt index eea7d6ef..f09373cb 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Token.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Token.kt @@ -5,7 +5,8 @@ import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.contentOrNull /** Where the provisioning token comes from. */ -class TokenError(message: String) : Exception(message) +/** [unreachable]: Meta was not reached at all, so the login itself may well be fine. */ +class TokenError(message: String, val unreachable: Boolean = false) : Exception(message) class Token( val value: String, diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt index d26bd7ec..d5997ed1 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt @@ -63,7 +63,7 @@ fun exchangeCode(code: String, http: HttpPost = UrlConnectionPost): String { body.toByteArray(Charsets.US_ASCII), ) } catch (error: Exception) { - throw TokenError("token-exchange: ${error.javaClass.simpleName}: ${error.message}") + throw TokenError("token-exchange: ${error.javaClass.simpleName}: ${error.message}", unreachable = true) } val payload = try { @@ -103,8 +103,8 @@ class WebLoginSource( try { return exchangeFrla(cached, http = http) } catch (error: TokenError) { - // An FRLA that no longer works is the one case where the browser has to open - // again. Say why, so this does not look like a network fault. + // No network is not a dead login: say so, and keep "log in again" for an FRLA Meta turned down. + if (error.unreachable) throw TokenError("Meta could not be reached (${error.message})", unreachable = true) throw TokenError("the saved Meta login no longer works (${error.message}); log in again") } } From 1027e5dcf4c04f9176a9b5fca529a8aaf19d9255 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 20:30:18 +0200 Subject: [PATCH 37/90] Display: the Briefing panel, and Stocks. Visor offers the home a Briefing panel, the Glanceables: Meta's XFBGlanceablesQuery run with the user's login and the phone's place, its cards gzipped into a Bloks payload. A message longer than one AirShield frame runs on into the next frames, as the glasses send theirs, so the Stocks details (4.7 KB) reach the Display. Visor keeps one Meta token for the run: each exchange opened a new session and Meta ended the older ones, so two Bloks requests at once failed with "the session has been invalidated". A token Meta drops is replaced once. A fetch of unknown length reports 0 bytes. Debug API: INTERNET switches sharing at once. --- .../macedonga/visor/debug/DebugReceiver.kt | 38 +++-- .../macedonga/visor/connection/BloksRelay.kt | 16 +- .../macedonga/visor/connection/DataFetcher.kt | 3 +- .../visor/connection/InternetSwitch.kt | 8 + .../com/macedonga/visor/connection/Panels.kt | 154 +++++++++++++++++- .../com/macedonga/visor/login/MetaLogin.kt | 23 +++ .../visor/core/identity/token/WebLogin.kt | 29 +++- .../visor/core/session/SecureLink.kt | 17 +- 8 files changed, 243 insertions(+), 45 deletions(-) diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index f1c88985..d8760529 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -65,6 +65,7 @@ class DebugReceiver : BroadcastReceiver() { @Inject lateinit var screencast: com.macedonga.visor.core.media.Screencast @Inject lateinit var requests: com.macedonga.visor.connection.RequestLog @Inject lateinit var login: com.macedonga.visor.login.MetaLogin + @Inject lateinit var internet: com.macedonga.visor.connection.InternetSwitch override fun onReceive(context: Context, intent: Intent) { val action = intent.action?.substringAfterLast('.').orEmpty() @@ -348,21 +349,30 @@ class DebugReceiver : BroadcastReceiver() { // --es doc --es name --es vars : one query on Meta's ar GraphQL, as the // user; the answer's first 4000 characters. "GRAPHQL" -> answerLater { - val token = login.source.get() - val body = com.macedonga.visor.core.identity.pyUrlencode( - linkedMapOf( - "fb_api_req_friendly_name" to intent.getStringExtra("name").orEmpty(), - "client_doc_id" to intent.getStringExtra("doc").orEmpty(), - "variables" to (intent.getStringExtra("vars64")?.let { String(android.util.Base64.decode(it, 0)) } ?: intent.getStringExtra("vars") ?: "{}"), - "method" to "post", "locale" to "en_US", "pretty" to "false", "format" to "json", - ), + val text = login.graphql( + intent.getStringExtra("name").orEmpty(), + intent.getStringExtra("doc").orEmpty(), + intent.getStringExtra("vars64")?.let { String(android.util.Base64.decode(it, 0)) } ?: intent.getStringExtra("vars") ?: "{}", ) - val reply = com.macedonga.visor.core.identity.UrlConnectionPost.post( - "https://ar-genai.graph.meta.com/graphql", - mapOf("Authorization" to "OAuth ${token.value}", "Content-Type" to "application/x-www-form-urlencoded"), - body.toByteArray(), - ) - "HTTP ${reply.status}: " + String(reply.body).take(4000) + if (!intent.getBooleanExtra("shape", false)) text.take(4000) else { + // Every path with its type, a string's length and start: the structure without the bulk. + val out = StringBuilder() + fun walk(path: String, v: Any?) { + when (v) { + is org.json.JSONObject -> v.keys().forEach { walk("$path.$it", v.opt(it)) } + is org.json.JSONArray -> { out.append("$path [${v.length()}]\n"); if (v.length() > 0) walk("$path[0]", v.opt(0)) } + is String -> out.append("$path str(${v.length}) ${v.take(60).replace('\n', ' ')}\n") + else -> out.append("$path = $v\n") + } + } + walk("", org.json.JSONObject(text)) + out.toString().take(4000) + } + } + // --ez on false: tell the Display the phone is offline, so it keeps its apps off the phone's internet. + "INTERNET" -> { + internet.sharing = intent.getBooleanExtra("on", true) + resultData = "sharing=${internet.sharing}" } // The request log's newest entries, links cut at their query. "REQUESTS" -> resultData = requests.items.value.takeLast(intent.getIntExtra("n", 30)).joinToString("\n") { diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/BloksRelay.kt b/app/src/main/kotlin/com/macedonga/visor/connection/BloksRelay.kt index 7cace127..0848b59a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/BloksRelay.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/BloksRelay.kt @@ -1,7 +1,5 @@ package com.macedonga.visor.connection -import com.macedonga.visor.core.identity.UrlConnectionPost -import com.macedonga.visor.core.identity.pyUrlencode import com.macedonga.visor.core.protocol.Wire import com.macedonga.visor.core.session.DataXConnection import com.macedonga.visor.core.session.DataXMessage @@ -62,24 +60,17 @@ class BloksRelay @Inject constructor( } private fun fetch(r: Request, q: Query): String { - val token = login.get().source.get() val params = JSONObject().put("app_id", r.appId).put("params", JSONObject(r.params as Map<*, *>).toString()) .put("bloks_versioning_id", r.versioningId) if (r.prebundled) params.put("is_prebundled", true) val context = JSONObject().put("pixel_ratio", r.pixelRatio).put("bloks_version", r.versioningId) .put("theme_params", JSONArray().put(JSONObject().put("design_system_name", r.designSystem).put("value", JSONArray(r.themeValues)))) val variables = JSONObject().put("params", params).put("bk_context", context) - val body = pyUrlencode( - linkedMapOf( - "fb_api_req_friendly_name" to q.name, "client_doc_id" to q.doc, "variables" to variables.toString(), - "method" to "post", "locale" to "en_US", "pretty" to "false", "format" to "json", - ), - ) - val answer = UrlConnectionPost.post(GRAPH_URL, mapOf("Authorization" to "OAuth ${token.value}", "Content-Type" to "application/x-www-form-urlencoded"), body.toByteArray()) - val json = JSONObject(String(answer.body)) + val answer = login.get().graphql(q.name, q.doc, variables.toString()) + val json = JSONObject(answer) json.optJSONArray("errors")?.let { error(it.optJSONObject(0)?.optString("message") ?: "Meta refused it") } // data..: the first string under the first object, whatever Meta names them. - val data = json.getJSONObject("data") + val data = json.optJSONObject("data") ?: error(json.optJSONObject("error")?.optString("message") ?: answer.take(300)) val root = data.keys().asSequence().mapNotNull { data.optJSONObject(it) }.firstOrNull() ?: error("no answer in the reply") return root.keys().asSequence().mapNotNull { k -> root.opt(k) as? String }.firstOrNull { it.startsWith("{") } ?: error("no bundle in the reply") } @@ -112,7 +103,6 @@ class BloksRelay @Inject constructor( companion object { const val SVC = 43 - private const val GRAPH_URL = "https://ar-genai.graph.meta.com/graphql" private val QUERIES = mapOf( 9 to Query("CompanionBloksComponentQuery", "37274295535564761374285150560", 10), 11 to Query("CompanionBloksAsyncActionQuery", "35737516989272377596036696859", 12), diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt b/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt index ff59d3d8..690c2096 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt @@ -219,7 +219,8 @@ class DataFetcher @Inject constructor( } private suspend fun send(mux: DataXConnection, reply: Int, data: ByteArray, total: Long, compressed: Boolean) { - val body = Wire.fieldBytes(1, data) + Wire.fieldVarint(2, if (compressed) 1 else 0) + Wire.fieldVarint(3, total.coerceIn(-1, Int.MAX_VALUE.toLong())) + // An unknown length goes as 0: the field is an int32, and a negative one does not encode here. + val body = Wire.fieldBytes(1, data) + Wire.fieldVarint(2, if (compressed) 1 else 0) + Wire.fieldVarint(3, total.coerceIn(0, Int.MAX_VALUE.toLong())) mux.send(reply, body, messageType = MT_REQUEST) } diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt index bbef615e..a2f181f0 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt @@ -32,6 +32,13 @@ class InternetSwitch @Inject constructor( ) : Responder { private val watching = AtomicBoolean(false) + /** False keeps the Display off the phone's internet: it is told the phone is offline. */ + @Volatile var sharing = true + set(value) { + field = value + push() + } + override suspend fun respond(message: DataXMessage, mux: DataXConnection) { if (message.info.messageType != MT_LISTEN) return val state = phoneInternet() @@ -62,6 +69,7 @@ class InternetSwitch @Inject constructor( private data class PhoneInternet(val connected: Boolean, val metered: Boolean, val dns: List) private fun phoneInternet(): PhoneInternet { + if (!sharing) return PhoneInternet(false, false, emptyList()) val cm = context.getSystemService(ConnectivityManager::class.java) ?: return PhoneInternet(false, false, emptyList()) val network = cm.activeNetwork val caps = network?.let { cm.getNetworkCapabilities(it) } diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/Panels.kt b/app/src/main/kotlin/com/macedonga/visor/connection/Panels.kt index b227d066..63eb86f4 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/Panels.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/Panels.kt @@ -1,30 +1,170 @@ package com.macedonga.visor.connection +import android.Manifest +import android.annotation.SuppressLint +import android.content.Context +import android.content.pm.PackageManager +import android.location.LocationManager +import com.macedonga.visor.core.protocol.Wire import com.macedonga.visor.core.session.DataXConnection import com.macedonga.visor.core.session.DataXMessage import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.login.MetaLogin +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.ByteArrayOutputStream +import java.util.Locale +import java.util.TimeZone +import java.util.zip.GZIPOutputStream import javax.inject.Inject +import javax.inject.Provider import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import org.json.JSONArray +import org.json.JSONObject /** - * svc9002, panels: the Display asks which panels the phone offers for its home (type 1) and waits for the list (type 2), - * timing out after 10 s without it. The Meta app's one panel is a Briefing drawn from Meta's server; Visor offers none, - * so the Display stops waiting. + * svc9002, panels: the Display's home asks which panels the phone offers (type 1, answered 2) and opens one (type 0, + * answered with a UICommand on 0). The one panel is the Briefing, the Glanceables (weather, reminders and the rest): + * Meta's XFBGlanceablesQuery run with the user's login, its preview and each card's Bloks payload gzipped into a + * BloksPayload, the cards' list beside it. A cache prewarm (3) is answered 4. */ @Singleton -class Panels @Inject constructor() : Responder { +class Panels @Inject constructor( + @ApplicationContext private val context: Context, + private val login: Provider, + private val requests: RequestLog, + private val log: Provider, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Panels: failed: $e") }) + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { val reply = DataXConnection.mirror(message.info.channelId) when (message.info.messageType) { - MT_GET_ENABLED -> connection.send(reply, ByteArray(0), messageType = MT_ENABLED) - else -> if (!message.info.flags.isClosed) connection.sendStatus(reply, BAD_TYPE) + MT_GET_ENABLED -> connection.send(reply, Wire.fieldBytes(1, briefingInfo()), messageType = MT_ENABLED) + MT_OPEN -> { + val host = runCatching { Host.decode(Wire.parse(message.payload)) }.getOrNull() ?: return + scope.launch { open(host, reply, connection) } + } + MT_PREWARM -> connection.send(reply, Wire.fieldVarint(1, 0), messageType = MT_PREWARMED) + } + } + + /** PanelInfo: 1 id, 2 name, 3 ttl s, 4 renderer (Bloks), 5 data provider (DataX), 6 cache policy (enabled). */ + private fun briefingInfo(): ByteArray = + Wire.fieldVarint(1, BRIEFING.toLong()) + Wire.fieldBytes(2, "Briefing".encodeToByteArray()) + Wire.fieldVarint(3, TTL_S) + + Wire.fieldVarint(4, RENDERER_BLOKS) + Wire.fieldVarint(5, PROVIDER_DATAX) + Wire.fieldVarint(6, CACHE_ENABLED) + + private suspend fun open(host: Host, reply: Int, mux: DataXConnection) { + val id = requests.start("bloks://glanceables", SOURCE_META) + val started = System.nanoTime() + try { + val g = fetch(host) + val hashes = mutableMapOf() + val payloads = g.optJSONArray("l1_payload_list") ?: JSONArray() + var bloks = ByteArray(0) + for (i in 0 until payloads.length()) { + val p = payloads.getJSONObject(i) + val cardId = p.optString("id") + val hash = p.optString("content_hash") + hashes[cardId] = hash + bloks += Wire.fieldBytes(1, Wire.fieldBytes(1, cardId.encodeToByteArray()) + Wire.fieldBytes(2, p.optString("payload").encodeToByteArray()) + Wire.fieldBytes(3, hash.encodeToByteArray())) + } + bloks += Wire.fieldBytes(2, g.optString("preview").encodeToByteArray()) + val gz = ByteArrayOutputStream().also { out -> GZIPOutputStream(out).use { it.write(bloks) } }.toByteArray() + + var data = Wire.fieldBytes(1, gz) + val cards = g.optJSONArray("all_glanceables") ?: JSONArray() + for (i in 0 until cards.length()) { + val c = cards.getJSONObject(i) + val name = c.optString("name") + data += Wire.fieldBytes( + 2, + Wire.fieldBytes(1, c.optString("id").encodeToByteArray()) + Wire.fieldBytes(2, name.encodeToByteArray()) + + Wire.fieldBytes(3, "Shows ${name.lowercase()} information".encodeToByteArray()) + Wire.fieldVarint(4, if (c.optBoolean("enabled")) 1 else 0) + + Wire.fieldBytes(6, hashes[c.optString("id")].orEmpty().encodeToByteArray()) + Wire.fieldBytes(7, c.optString("icon_uri").encodeToByteArray()), + ) + } + data += Wire.fieldVarint(3, g.optLong("fresh_ttl")) + Wire.fieldVarint(4, g.optLong("ttl")) + val ms = (System.nanoTime() - started) / 1_000_000f + val stats = Wire.fieldVarint(1, OS_ANDROID) + Wire.fieldFloat(3, ms) + val command = Wire.fieldVarint(1, COMMAND_CREATE) + Wire.fieldVarint(2, WIDGET_DATAPAYLOAD) + Wire.fieldBytes(3, stats) + Wire.fieldBytes(10, data) + mux.send(reply, command, messageType = MT_OPEN) + requests.finish(id, "from Meta, ${payloads.length()} cards", failed = false, bytes = gz.size.toLong()) + } catch (error: Exception) { + requests.finish(id, "failed: ${error.message}", failed = true) + val details = Wire.fieldBytes(1, (error.message ?: "failed").encodeToByteArray()) + runCatching { mux.send(reply, Wire.fieldVarint(1, COMMAND_ERROR) + Wire.fieldBytes(11, details), messageType = MT_OPEN) } + } + } + + private fun fetch(host: Host): JSONObject { + val vars = JSONObject() + .put("bloks_version_id", host.versioningId) + .put("device_type", if (host.device == DEVICE_WRIST) "WRIST" else "GREATWHITE") + .put("design_system_version", host.designSystem) + .put("timezone", TimeZone.getDefault().id) + .put("request_id", host.requestId) + .put("assistant_locale", Locale.getDefault().toString()) + .put("display_locale", Locale.getDefault().toString()) + here()?.let { (lat, lon) -> vars.put("latitude", lat).put("longitude", lon) } + if (host.persist) { + vars.put("enabled_glanceables", JSONArray(host.cards.filter { it.enabled }.map { JSONObject().put("id", it.id).put("name", it.name).put("cached_hash", it.hash) })) + } + val answer = login.get().graphql("XFBGlanceablesQuery", GLANCEABLES_DOC, vars.toString()) + val json = JSONObject(answer) + json.optJSONArray("errors")?.let { error(it.optJSONObject(0)?.optString("message") ?: "Meta refused it") } + val data = json.optJSONObject("data") ?: error(json.optJSONObject("error")?.optString("message") ?: answer.take(300)) + return data.getJSONObject("xfb_glanceables") + } + + /** The phone's last known place, for the weather; none without the permission. */ + @SuppressLint("MissingPermission") + private fun here(): Pair? { + if (context.checkSelfPermission(Manifest.permission.ACCESS_COARSE_LOCATION) != PackageManager.PERMISSION_GRANTED) return null + val lm = context.getSystemService(LocationManager::class.java) ?: return null + val best = lm.getProviders(true).mapNotNull { runCatching { lm.getLastKnownLocation(it) }.getOrNull() }.maxByOrNull { it.time } ?: return null + // Two decimals, as the Display itself sends them: a kilometre is plenty for the weather. + return Math.round(best.latitude * 100) / 100.0 to Math.round(best.longitude * 100) / 100.0 + } + + private class Card(val id: String, val name: String, val enabled: Boolean, val hash: String) + + /** OpenPanel 2 HostInfo: 1 device, 4 Bloks version, 5 design system, 7 panel configs, 8 persist them, 9 request id. */ + private class Host(val device: Long, val versioningId: String, val designSystem: String, val cards: List, val persist: Boolean, val requestId: String) { + companion object { + fun decode(open: Wire.Fields): Host { + val h = Wire.parse(open.bytes(2)) + val cards = h.repeatedBytes(7).map { Wire.parse(it) }.filter { it.varint(5) == BRIEFING.toLong() }.map { + Card(it.bytes(1).decodeToString(), it.bytes(2).decodeToString(), it.varint(4) != 0L, it.bytes(6).decodeToString()) + } + return Host(h.varint(1), h.bytes(4).decodeToString(), h.bytes(5).decodeToString(), cards, h.varint(8) != 0L, h.bytes(9).decodeToString()) + } } } companion object { const val SVC = 9002 + private const val MT_OPEN = 0 private const val MT_GET_ENABLED = 1 private const val MT_ENABLED = 2 - private const val BAD_TYPE = 0xD004 + private const val MT_PREWARM = 3 + private const val MT_PREWARMED = 4 + private const val BRIEFING = 30 + private const val TTL_S = 3600L + private const val RENDERER_BLOKS = 2L + private const val PROVIDER_DATAX = 1L + private const val CACHE_ENABLED = 1L + private const val COMMAND_CREATE = 0L + private const val COMMAND_ERROR = 3L + private const val WIDGET_DATAPAYLOAD = 5L + private const val OS_ANDROID = 1L + private const val DEVICE_WRIST = 2L + private const val GLANCEABLES_DOC = "168198258811666417495687582669" + private const val SOURCE_META = -2 } } diff --git a/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt b/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt index c14319b2..25c24c09 100644 --- a/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt +++ b/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt @@ -11,7 +11,9 @@ import com.macedonga.visor.core.identity.UrlConnectionPost import com.macedonga.visor.core.identity.bloks.BloksClient import com.macedonga.visor.core.identity.bloks.BloksHost import com.macedonga.visor.core.identity.bloks.BloksSession +import com.macedonga.visor.core.identity.pyUrlencode import com.macedonga.visor.core.identity.token.AuthorizeRequest +import com.macedonga.visor.core.identity.token.Token import com.macedonga.visor.core.identity.token.WebLoginSource import com.macedonga.visor.log.EventLog import dagger.hilt.android.qualifiers.ApplicationContext @@ -27,6 +29,7 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch +import org.json.JSONObject /** The Meta login, for as long as the process lives. */ @Singleton @@ -69,6 +72,25 @@ class MetaLogin @Inject constructor( log.add("Login: opening Meta's login page") } + /** One query on Meta's ar GraphQL as the user, its answer as text. A token Meta has dropped is replaced, once. */ + fun graphql(name: String, doc: String, variables: String): String { + val body = pyUrlencode( + linkedMapOf( + "fb_api_req_friendly_name" to name, "client_doc_id" to doc, "variables" to variables, + "method" to "post", "locale" to "en_US", "pretty" to "false", "format" to "json", + ), + ).toByteArray() + val token = source.get() + val answer = post(token, body) + // OAuthException 190: the session behind the token has ended. + if (runCatching { JSONObject(answer).optJSONObject("error")?.optInt("code") }.getOrNull() != 190) return answer + source.forget(token) + return post(source.get(), body) + } + + private fun post(token: Token, body: ByteArray): String = + String(UrlConnectionPost.post(GRAPH_URL, mapOf("Authorization" to "OAuth ${token.value}", "Content-Type" to "application/x-www-form-urlencoded"), body).body) + /** What the login page saw: hosts, paths and statuses, never a query string. */ fun note(text: String) = log.add("Login page: $text") @@ -172,6 +194,7 @@ class MetaLogin @Inject constructor( companion object { private const val TRACE_LIMIT = 4L * 1024 * 1024 + private const val GRAPH_URL = "https://ar-genai.graph.meta.com/graphql" /** Every Bloks call of the last native login, shared with the event log. */ fun traceFile(context: Context): File = File(context.filesDir, "bloks-trace.log") diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt index d5997ed1..ff1fe388 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt @@ -98,17 +98,30 @@ class WebLoginSource( /** True when an FRLA is cached. Says nothing about whether Meta still accepts it. */ val hasCachedLogin: Boolean get() = readCache() != null + // One token for the run: each exchange opens a new Meta session, and Meta ends the older ones, with the + // requests still in flight on them. + @Volatile private var token: Token? = null + override fun get(): Token { - val cached = readCache() ?: throw TokenError("not logged in to Meta") - try { - return exchangeFrla(cached, http = http) - } catch (error: TokenError) { - // No network is not a dead login: say so, and keep "log in again" for an FRLA Meta turned down. - if (error.unreachable) throw TokenError("Meta could not be reached (${error.message})", unreachable = true) - throw TokenError("the saved Meta login no longer works (${error.message}); log in again") + token?.let { return it } + synchronized(this) { + token?.let { return it } + val cached = readCache() ?: throw TokenError("not logged in to Meta") + try { + return exchangeFrla(cached, http = http).also { token = it } + } catch (error: TokenError) { + // No network is not a dead login: say so, and keep "log in again" for an FRLA Meta turned down. + if (error.unreachable) throw TokenError("Meta could not be reached (${error.message})", unreachable = true) + throw TokenError("the saved Meta login no longer works (${error.message}); log in again") + } } } + /** Meta no longer takes [dropped]: the next [get] exchanges the login for a new token. */ + fun forget(dropped: Token) { + synchronized(this) { if (token == dropped) token = null } + } + fun begin(): AuthorizeRequest = authorizeUrl() /** Take the redirect the WebView caught, exchange its code, and cache the FRLA. */ @@ -137,6 +150,7 @@ class WebLoginSource( /** Forget the cached login. The WebView's cookies are the app's to clear. */ fun logOut() { + token = null cacheFile?.delete() } @@ -152,6 +166,7 @@ class WebLoginSource( } private fun writeCache(frla: String) { + token = null if (cacheFile == null || sealer == null) return cacheFile.parentFile?.mkdirs() cacheFile.writeBytes(sealer.seal(frla.toByteArray(Charsets.UTF_8))) diff --git a/core/session/src/main/kotlin/com/macedonga/visor/core/session/SecureLink.kt b/core/session/src/main/kotlin/com/macedonga/visor/core/session/SecureLink.kt index d89a017f..9da4a279 100644 --- a/core/session/src/main/kotlin/com/macedonga/visor/core/session/SecureLink.kt +++ b/core/session/src/main/kotlin/com/macedonga/visor/core/session/SecureLink.kt @@ -208,13 +208,21 @@ class SecureLink( txMutex.withLock { transport.send(sdu) } } - /** Seal and send under one lock, and without a cancellation point between them. */ + /** + * Seal and send under one lock, and without a cancellation point between them. A message longer than one frame + * runs on into the next frames, as the glasses send theirs; the lock keeps another message out of the middle. + */ private suspend fun sealAndSend(plaintext: ByteArray) { txMutex.withLock { val crypto = checkNotNull(txCrypto) { "link not open" } withContext(NonCancellable) { - // Framing.pack adds the 0x40 type byte: [0x40][8B tag][1B blockCount][ct]. - transport.send(Framing.pack(FrameType.DATA, 0, crypto.seal(plaintext))) + var at = 0 + do { + val end = minOf(plaintext.size, at + FRAME_PLAINTEXT) + // Framing.pack adds the 0x40 type byte: [0x40][8B tag][1B blockCount][ct]. + transport.send(Framing.pack(FrameType.DATA, 0, crypto.seal(plaintext.copyOfRange(at, end)))) + at = end + } while (at < plaintext.size) } } } @@ -229,5 +237,8 @@ class SecureLink( /** The parameters this side's RequestEncryption and EnableEncryption announce. */ const val OUR_PARAMETERS = 31 + + /** The most one frame holds: 256 blocks, less the pad's one byte at least. */ + const val FRAME_PLAINTEXT = 256 * 16 - 1 } } \ No newline at end of file From 36d45ef4ee50c99c7ad5c69e38fe5a8fd81ae661 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Tue, 20 Oct 2026 20:35:23 +0200 Subject: [PATCH 38/90] Home: a case reading taken while the case was on its charger is dropped once the case stops reporting. The case reads out only through the glasses sitting in it, so its last level stood for an hour after the glasses left a charging case, at 2%. --- .../kotlin/com/macedonga/visor/home/LastSeen.kt | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/home/LastSeen.kt b/app/src/main/kotlin/com/macedonga/visor/home/LastSeen.kt index a02c56ff..2573902b 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/LastSeen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/LastSeen.kt @@ -36,6 +36,8 @@ data class LastPosture(val folded: Boolean, val inCase: Boolean) data class PairSeen( val glasses: Int? = null, val case: Int? = null, + // The case was on its charger at that reading. + val caseCharging: Boolean = false, val at: Long = 0, val folded: Boolean = false, val inCase: Boolean = false, @@ -110,7 +112,16 @@ class LastSeen @Inject constructor( update(target.serial) { was -> val battery = device.battery var now = was - if (battery != null) now = now.copy(glasses = battery.percent.toInt(), case = device.case?.levelPercent() ?: was.case, at = System.currentTimeMillis()) + if (battery != null) { + // The case reads out only through the glasses in it. One that was charging has gone on charging since: + // its last reading is out of date. + val case = device.case?.levelPercent() + now = now.copy( + glasses = battery.percent.toInt(), at = System.currentTimeMillis(), + case = case ?: was.case.takeUnless { was.caseCharging }, + caseCharging = case != null && device.case?.charging == true, + ) + } // The case's "present" only says the case is known; charging folded is what says they sit in it. if (state.link == LinkState.READY) { val inCase = battery?.charging?.let { it && device.hinge != "OPEN" } @@ -183,6 +194,7 @@ class LastSeen @Inject constructor( private fun encode(s: PairSeen): String = JSONObject().apply { s.glasses?.let { put("glasses", it) } s.case?.let { put("case", it) } + if (s.caseCharging) put("case_charging", true) put("at", s.at) put("folded", s.folded) put("in_case", s.inCase) @@ -198,6 +210,7 @@ class LastSeen @Inject constructor( PairSeen( glasses = o.optInt("glasses", -1).takeIf { it >= 0 }, case = o.optInt("case", -1).takeIf { it >= 0 }, + caseCharging = o.optBoolean("case_charging"), at = o.optLong("at", 0), folded = o.optBoolean("folded"), inCase = o.optBoolean("in_case"), From 80715e64a7db5b3833a7128923cddeb1cc82de3a Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Wed, 21 Oct 2026 15:39:56 +0200 Subject: [PATCH 39/90] Label the band's and other pairs' batteries; README lists Display, band and pairs The merge of main brought the battery labels for TalkBack; the band and the other pairs' pills on Home take them too, and an away pair dims only as far as stays readable. The README drops the display branch: the Display, its Neural Band and more than one pair are in the feature table. --- README.md | 5 ++++- .../main/kotlin/com/macedonga/visor/home/HomeFeed.kt | 10 +++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7ca30645..449da223 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,13 @@ Visor works with Ray-Ban Meta, Oakley Meta and the other Meta glasses. It speaks | **Settings** | Every setting the glasses have, the way the Meta app shows them. Advanced holds the rest, assistant language packs and the firmware's feature flags. | | **Outside the app** | A home screen widget, a Quick Settings tile, launcher shortcuts, and a Live Update notification while the Live view runs. | | **Developer mode** | Other apps and automations can take photos and videos, import captures and hear about new ones. See [Developer mode](docs/developer-mode.md). | +| **More than one pair** | Keep several pairs on one phone. Home swipes between them, each with its name, batteries and a Connect button, and ends with a page to set up another. | +| **Meta Ray-Ban Display** | A teleprompter, live captions and translation, the Neural Band's gestures, the Briefing panel and saved Wi-Fi networks. The glasses' own apps reach the internet through the phone, with Instagram's Reels and messages among them. | +| **Neural Band** | Set up from the Display's settings, with its battery on Home and its wrist in Settings. | ## Requirements -- **Glasses:** a pair of Meta glasses, such as Ray-Ban Meta or Oakley Meta. Support for the Meta Ray-Ban Display is limited and lives on the [`display`](https://github.com/macedonga/visor/tree/display) branch. +- **Glasses:** a pair of Meta glasses, such as Ray-Ban Meta, Oakley Meta or the Meta Ray-Ban Display. Support for the Display and its Neural Band is newer and does not yet cover everything the Meta AI app does. - **Phone:** Android 10 or later. - **Meta account**, for setup. Log in with a password or an email code. Passkeys do not work. - **For the assistant:** an API key from Anthropic or from an OpenAI-compatible service. diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt index 149fea99..661772ef 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt @@ -212,8 +212,8 @@ fun StatusPill( } if (band != null) { Spacer(Modifier.size(14.dp)) - Level(VisorIcons.Band, band, critical = band <= 10 && !bandCharging) - if (bandCharging) Icon(VisorIcons.Bolt, null, Modifier.padding(start = 2.dp).size(13.dp), tint = colors.textPrimary) + Level(VisorIcons.Band, stringResource(R.string.topic_band), band, critical = band <= 10 && !bandCharging) + if (bandCharging) Icon(VisorIcons.Bolt, stringResource(R.string.mood_charging), Modifier.padding(start = 2.dp).size(13.dp), tint = colors.textPrimary) } } } @@ -270,11 +270,11 @@ fun OtherPairPill(mood: Mood, seen: PairSeen?, peeking: Boolean, onConnect: (() val glasses = seen?.glasses if (glasses != null && !peeking) { Spacer(Modifier.size(12.dp)) - Row(Modifier.graphicsLayer { alpha = if (fresh) 1f else 0.45f }, verticalAlignment = Alignment.CenterVertically) { - Level(VisorIcons.Glasses, glasses, critical = false) + Row(Modifier.graphicsLayer { alpha = if (fresh) 1f else 0.6f }, verticalAlignment = Alignment.CenterVertically) { + Level(VisorIcons.Glasses, stringResource(R.string.glasses_battery), glasses, critical = false) seen.case?.let { Spacer(Modifier.size(14.dp)) - Level(VisorIcons.Case, it, critical = false) + Level(VisorIcons.Case, stringResource(R.string.glasses_case_label), it, critical = false) } } } From 5ba028e5aa355de75ae375f01b712aec968865af Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Wed, 21 Oct 2026 20:17:33 +0200 Subject: [PATCH 40/90] Display: Messages on the phone's texts. Visor serves svc70 for provider 3, as the Meta app does: the glasses' Messages app lists the conversations (type 1) and opens one (2) from Android's SMS store, SMS and MMS together, and sends a text (8). A conversation whose Google Messages notification is up answers through that notification's reply, so an RCS chat stays RCS. Any other text goes as SMS and is answered once the radio has sent it. Texts that arrive reach the glasses as native notifications built from the SMS app's own (app com.android.sms, the thread found from the sender's number), and the Messages app files them under their thread. A reply sent from a notification is no longer relayed as an incoming message. Replies over 1000 bytes go in the Meta app's type-32 chunks, and folder names are lowercase, because the glasses mark a message as sent only for "sent". Phone access asks for SMS. RCS chats that Google Messages stores as restricted rows stay hidden, because Android shows those rows only to the default SMS app. Debug API: THREADS lists the thread table as Visor reads it, and NOTIFY takes --es thread. --- .../macedonga/visor/debug/DebugReceiver.kt | 10 +- app/src/main/AndroidManifest.xml | 3 + .../visor/connection/PhoneMessages.kt | 384 ++++++++++++++++++ .../macedonga/visor/di/ConnectionModule.kt | 2 + .../macedonga/visor/log/RequestLogScreen.kt | 1 + .../visor/notify/GlassesNotification.kt | 6 +- .../macedonga/visor/notify/MirrorListener.kt | 48 ++- .../com/macedonga/visor/notify/TextReplies.kt | 40 ++ .../macedonga/visor/settings/PhoneAccess.kt | 3 + app/src/main/res/values-it/strings.xml | 5 + app/src/main/res/values-ru/strings.xml | 5 + app/src/main/res/values/strings.xml | 5 + app/src/main/res/values/strings_requests.xml | 1 + 13 files changed, 502 insertions(+), 11 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/notify/TextReplies.kt diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index b6d39428..08977758 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -67,12 +67,15 @@ class DebugReceiver : BroadcastReceiver() { @Inject lateinit var login: com.macedonga.visor.login.MetaLogin @Inject lateinit var internet: com.macedonga.visor.connection.InternetSwitch @Inject lateinit var developer: com.macedonga.visor.outside.DeveloperMode + @Inject lateinit var texts: com.macedonga.visor.connection.PhoneMessages override fun onReceive(context: Context, intent: Intent) { val action = intent.action?.substringAfterLast('.').orEmpty() Log.i("Visor", "API: $action") when (action) { "STATUS" -> resultData = status() + // The phone's thread list as Visor reads it for the glasses' Messages: --ei count. + "THREADS" -> resultData = texts.describeThreads(intent.getIntExtra("count", 15)) "DEMO_PAIR" -> resultData = demoPair(intent.getBooleanExtra("on", true)) "CONNECT" -> resultData = runCatching { GlassesService.start(context); "connecting" }.getOrElse { "could not start the service: $it" } "DISCONNECT" -> { @@ -122,8 +125,10 @@ class DebugReceiver : BroadcastReceiver() { "refused: unknown namespace or not connected" } } - // A notification built by hand: --ei type --ei provider --es title --es text --es sender --es pkg. + // A notification built by hand: --ei type --ei provider --es title --es text --es sender --es pkg --es thread + // (a text's thread in the SMS store, whose number becomes the sender's id). "NOTIFY" -> { + val thread = intent.getStringExtra("thread").orEmpty() val n = com.macedonga.visor.notify.GlassesNotification( key = "visor|" + System.currentTimeMillis(), pkg = intent.getStringExtra("pkg") ?: "org.telegram.messenger", @@ -134,7 +139,8 @@ class DebugReceiver : BroadcastReceiver() { type = intent.getIntExtra("type", 1), provider = intent.getIntExtra("provider", 3), sender = intent.getStringExtra("sender").orEmpty(), - senderId = intent.getStringExtra("sid").orEmpty(), + senderId = intent.getStringExtra("sid") ?: texts.addressesOf(thread).firstOrNull().orEmpty(), + threadId = thread, ) connection.submit { it.sendOnService(com.macedonga.visor.notify.GlassesNotification.SVC, 1, n.envelope()) } resultData = "sent type ${n.type} provider ${n.provider}" diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 2208f08e..747e8af0 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -30,6 +30,9 @@ + + + diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt new file mode 100644 index 00000000..cb1996bd --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt @@ -0,0 +1,384 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.app.Activity +import android.app.PendingIntent +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.content.pm.PackageManager +import android.net.Uri +import android.os.Build +import android.provider.ContactsContract.PhoneLookup +import android.provider.Telephony +import android.telephony.PhoneNumberUtils +import android.telephony.SmsManager +import android.telephony.TelephonyManager +import androidx.core.content.ContextCompat +import com.macedonga.visor.R +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.notify.TextReplies +import dagger.hilt.android.qualifiers.ApplicationContext +import java.nio.ByteBuffer +import java.util.Locale +import java.util.UUID +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull + +/** + * svc70, smartglasses_messaging, for the phone's own texts (provider 3, native), as the Meta app serves them from + * Android's SMS store: the glasses load the threads (type 1) and one thread's messages (2), SMS and MMS together, and + * send a text (8). A conversation whose notification is up answers through the Messages app's reply, as RCS when it + * is one; any other goes as SMS, answered once the radio has sent it. Each request comes on a channel of its own and + * is answered with its own type; a reply over 1000 bytes goes as type-32 chunks when the request allows them. Texts + * that arrive reach the glasses as native notifications, from the SMS app's own notifications (see MirrorListener). + * + * The RCS chats Google Messages files as MMS are marked restricted, and Android shows those rows only to the default + * SMS app: a chat with nothing else in it is missing from the list here. + */ +@Singleton +class PhoneMessages @Inject constructor( + @ApplicationContext private val context: Context, + private val requests: RequestLog, + private val log: Provider, + private val replies: TextReplies, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Messages: failed: $e") }) + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.payload.isEmpty()) return + val request = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return + val type = message.info.messageType + val reply = DataXConnection.mirror(message.info.channelId) + if (type !in SERVED) return log.get().add("Messages: type $type not served") + // Only the phone's texts; the other providers are the Meta app's. + val provider = request.varint(1) + if (provider != NATIVE) return log.get().add("Messages: provider $provider not served") + scope.launch { + when (type) { + MT_LOAD_THREADS -> threads(request, reply, connection) + MT_LOAD_MESSAGES -> messages(request, reply, connection) + MT_OUTGOING -> send(request, reply, connection) + } + } + } + + /** LoadThreadsRequest {2 request id, 3 count, 5 chunked}: the newest conversations, SMS, MMS and RCS alike. */ + private suspend fun threads(request: Wire.Fields, reply: Int, mux: DataXConnection) { + val id = requests.start("sms://threads", SOURCE_SMS) + val requestId = request.bytes(2) + if (!allowed(Manifest.permission.READ_SMS)) { + answer(mux, reply, MT_LOAD_THREADS, Wire.fieldBytes(1, requestId) + refused(), request.varint(5) != 0L) + return requests.finish(id, "SMS not allowed", failed = true) + } + val count = request.varint(3).toInt().takeIf { it > 0 } ?: DEFAULT_THREADS + val threads = conversations(count, null) + val names = HashMap() + var body = Wire.fieldBytes(1, requestId) + for (t in threads) { + val name = t.addresses.joinToString(", ") { a -> names.getOrPut(a) { nameOf(a) } } + val latest = t.snippet.ifBlank { messagesOf(t.id, 1).firstOrNull()?.text.orEmpty() } + // SgThread: 1 id, 2 name, 3 type (1 group), 4 latest message, 5 its time, 6 Participant {1 id}. + body += Wire.fieldBytes( + 2, + Wire.fieldBytes(1, t.id.toString().encodeToByteArray()) + Wire.fieldBytes(2, name.encodeToByteArray()) + + (if (t.addresses.size > 1) Wire.fieldVarint(3, THREAD_GROUP) else ByteArray(0)) + + Wire.fieldBytes(4, latest.encodeToByteArray()) + Wire.fieldVarint(5, t.date) + + t.addresses.fold(ByteArray(0)) { acc, a -> acc + Wire.fieldBytes(6, Wire.fieldBytes(1, a.encodeToByteArray())) }, + ) + } + body += Wire.fieldVarint(3, threads.maxOfOrNull { it.date } ?: 0) + answer(mux, reply, MT_LOAD_THREADS, body, request.varint(5) != 0L) + requests.finish(id, "${threads.size} threads", failed = false, bytes = body.size.toLong()) + } + + /** LoadMessagesRequest {2 request id, 3 count, 4 thread id, 8 chunked, 9 phone numbers}: oldest first. */ + private suspend fun messages(request: Wire.Fields, reply: Int, mux: DataXConnection) { + val requestId = request.bytes(2) + val numbers = request.repeatedBytes(9).map { it.decodeToString() }.filter { it.isNotBlank() } + val chunked = request.varint(8) != 0L + val id = requests.start("sms://thread/${request.bytes(4).decodeToString()}", SOURCE_SMS) + if (!allowed(Manifest.permission.READ_SMS)) { + answer(mux, reply, MT_LOAD_MESSAGES, Wire.fieldBytes(1, requestId) + refused(), chunked) + return requests.finish(id, "SMS not allowed", failed = true) + } + val thread = request.bytes(4).decodeToString().toLongOrNull() + ?: numbers.takeIf { it.isNotEmpty() }?.let { runCatching { Telephony.Threads.getOrCreateThreadId(context, it.toSet()) }.getOrNull() } + val count = request.varint(3).toInt().takeIf { it > 0 }?.coerceAtMost(MAX_MESSAGES) ?: DEFAULT_MESSAGES + val rows = if (thread != null) messagesOf(thread, count) else emptyList() + var body = Wire.fieldBytes(1, requestId) + for (m in rows.asReversed()) { + // SgMessage: 1 id, 2 text, 3 time, 4 the address (the other side's, in or out), 7 attachment, 19 box, 20 service. + body += Wire.fieldBytes( + 2, + Wire.fieldBytes(1, m.id.encodeToByteArray()) + Wire.fieldBytes(2, m.text.encodeToByteArray()) + + Wire.fieldVarint(3, m.date) + Wire.fieldBytes(4, m.address.encodeToByteArray()) + + (if (m.attachment) Wire.fieldVarint(7, 1) else ByteArray(0)) + + Wire.fieldBytes(19, folder(m.box).encodeToByteArray()) + Wire.fieldVarint(20, m.service), + ) + } + body += Wire.fieldVarint(5, RESPONSE_FILE_BUS) + Wire.fieldBytes(7, (thread ?: 0).toString().encodeToByteArray()) + answer(mux, reply, MT_LOAD_MESSAGES, body, chunked) + requests.finish(id, "${rows.size} messages", failed = false, bytes = body.size.toLong()) + } + + /** OutgoingMessageRequest {2 request id, 3 thread id, 4 text, 7 participants}: OutgoingMessageResponse {1, 2 status, 3 error}. */ + private suspend fun send(request: Wire.Fields, reply: Int, mux: DataXConnection) { + val requestId = request.bytes(2) + val text = request.bytes(4).decodeToString() + val thread = request.bytes(3).decodeToString() + val to = request.repeatedBytes(7).map { it.decodeToString() }.filter { it.isNotBlank() }.ifEmpty { addressesOf(thread) } + // A conversation with its notification up answers through the Messages app, as RCS when it is one. + val viaApp = text.isNotBlank() && replies.has(thread) + val id = requests.start("${if (viaApp) "messages" else "sms"}://send/${to.joinToString(",").ifEmpty { thread }}", SOURCE_SMS) + val error = when { + viaApp -> replies.reply(context, thread, text) + !allowed(Manifest.permission.SEND_SMS) -> "SMS not allowed in Visor" + to.isEmpty() -> "No recipient" + to.size > 1 -> "Group texts are not supported" + text.isBlank() -> "Empty message" + else -> deliver(to.single(), text) + } + val body = Wire.fieldBytes(1, requestId) + Wire.fieldVarint(2, if (error == null) OUT_SUCCESS else OUT_FAILURE) + + (error?.let { Wire.fieldBytes(3, it.encodeToByteArray()) } ?: ByteArray(0)) + // The glasses may have closed the channel by now; the status is then theirs to miss. + runCatching { mux.send(reply, body, messageType = MT_OUTGOING) } + requests.finish(id, error ?: "sent", failed = error != null, bytes = text.length.toLong()) + } + + /** Sends one text and waits for the radio's word on every part: null when all went, else why not. */ + private suspend fun deliver(to: String, text: String): String? { + val sms = if (Build.VERSION.SDK_INT >= 31) context.getSystemService(SmsManager::class.java) else @Suppress("DEPRECATION") SmsManager.getDefault() + val parts = sms.divideMessage(text) + val action = "${context.packageName}.SMS_SENT.${UUID.randomUUID()}" + val results = Channel(parts.size) + val receiver = object : BroadcastReceiver() { + override fun onReceive(c: Context, intent: Intent) { + results.trySend(resultCode) + } + } + ContextCompat.registerReceiver(context, receiver, IntentFilter(action), ContextCompat.RECEIVER_NOT_EXPORTED) + try { + val sent = ArrayList( + parts.indices.map { + PendingIntent.getBroadcast(context, it, Intent(action).setPackage(context.packageName), PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_ONE_SHOT) + }, + ) + // Android files a text a non-default app sends in the SMS store itself. + if (parts.size == 1) sms.sendTextMessage(to, null, text, sent[0], null) else sms.sendMultipartTextMessage(to, null, parts, sent, null) + val codes = withTimeoutOrNull(SEND_TIMEOUT_MS) { List(parts.size) { results.receive() } } ?: return "No answer from the phone's radio" + return when (codes.firstOrNull { it != Activity.RESULT_OK }) { + null -> null + SmsManager.RESULT_ERROR_RADIO_OFF -> "The phone's radio is off" + SmsManager.RESULT_ERROR_NO_SERVICE -> "No service" + else -> "Not sent (error ${codes.first { it != Activity.RESULT_OK }})" + } + } catch (e: Exception) { + return e.message ?: e.toString() + } finally { + context.unregisterReceiver(receiver) + } + } + + private suspend fun answer(mux: DataXConnection, reply: Int, type: Int, body: ByteArray, chunked: Boolean) { + if (!chunked || body.size <= CHUNK) return mux.send(reply, body, messageType = type) + // The Meta app's chunk: the reply's type, 36 zero bytes, the index, 1000, the count, then up to 1000 bytes. + val total = (body.size + CHUNK - 1) / CHUNK + for (i in 0 until total) { + val head = ByteBuffer.allocate(CHUNK_HEAD).putInt(type).put(ByteArray(36)).putInt(i).putInt(CHUNK).putInt(total).array() + mux.send(reply, head + body.copyOfRange(i * CHUNK, minOf((i + 1) * CHUNK, body.size)), messageType = MT_CHUNK) + } + } + + /** The numbers of a thread's other side. */ + fun addressesOf(thread: String): List { + val id = thread.toLongOrNull() ?: return emptyList() + if (!allowed(Manifest.permission.READ_SMS)) return emptyList() + return conversations(1, id).firstOrNull()?.addresses.orEmpty() + } + + private class Conversation(val id: Long, val date: Long, val addresses: List, val snippet: String) + + /** Conversations newest first, from Android's thread list, which holds SMS, MMS and the RCS chats Messages files as MMS. */ + private fun canonical(): Map { + val canonical = HashMap() + context.contentResolver.query(CANONICAL_ADDRESSES, arrayOf("_id", "address"), null, null, null)?.use { c -> + while (c.moveToNext()) canonical[c.getLong(0)] = c.getString(1).orEmpty() + } + return canonical + } + + /** For the debug receiver: the newest rows of the thread list as Visor reads them, ids and counts only. */ + fun describeThreads(count: Int): String { + val canonical = canonical() + val columns = arrayOf(Telephony.Threads._ID, Telephony.Threads.DATE, Telephony.Threads.RECIPIENT_IDS, Telephony.Threads.SNIPPET, Telephony.Threads.MESSAGE_COUNT) + fun rows(uri: Uri, where: String? = null) = context.contentResolver.query(uri, arrayOf("_id"), where, null, null)?.use { it.count } ?: -1 + val lines = mutableListOf( + "canonical addresses: ${canonical.size}, sms ${rows(Telephony.Sms.CONTENT_URI)}, mms ${rows(Telephony.Mms.CONTENT_URI)}", + "thread 489: sms ${rows(Telephony.Sms.CONTENT_URI, "thread_id=489")}, mms ${rows(Telephony.Mms.CONTENT_URI, "thread_id=489")}; " + + "thread 509: sms ${rows(Telephony.Sms.CONTENT_URI, "thread_id=509")}; listed ${rows(CONVERSATIONS, "_id IN (489,509)")}", + ) + context.contentResolver.query(CONVERSATIONS, columns, null, null, "${Telephony.Threads.DATE} DESC")?.use { c -> + lines += "rows: ${c.count}" + while (c.moveToNext() && lines.size < count + 2) { + val ids = c.getString(2).orEmpty() + val known = ids.split(' ').count { canonical[it.toLongOrNull()]?.isNotBlank() == true } + lines += "${c.getLong(0)} date=${c.getLong(1)} messages=${c.getInt(4)} recipients=[$ids] resolved=$known" + } + } ?: lines.add("no cursor") + return lines.joinToString("\n") + } + + private fun conversations(count: Int, only: Long?): List { + val canonical = canonical() + val out = mutableListOf() + val where = only?.let { "${Telephony.Threads._ID} = $it" } + val columns = arrayOf(Telephony.Threads._ID, Telephony.Threads.DATE, Telephony.Threads.RECIPIENT_IDS, Telephony.Threads.SNIPPET, Telephony.Threads.MESSAGE_COUNT) + context.contentResolver.query(CONVERSATIONS, columns, where, null, "${Telephony.Threads.DATE} DESC")?.use { c -> + while (c.moveToNext() && out.size < count) { + if (c.getInt(4) == 0) continue + val addresses = c.getString(2).orEmpty().split(' ').mapNotNull { canonical[it.toLongOrNull()]?.takeIf(String::isNotBlank) } + if (addresses.isEmpty()) continue + out += Conversation(c.getLong(0), c.getLong(1), addresses, c.getString(3).orEmpty()) + } + } + return out + } + + private class Message(val id: String, val text: String, val date: Long, val address: String, val box: Int, val service: Long, val attachment: Boolean) + + /** A thread's newest messages, SMS and MMS together, newest first. */ + private fun messagesOf(thread: Long, count: Int): List { + val out = mutableListOf() + read("${Telephony.Sms.THREAD_ID} = ?", arrayOf(thread.toString())) { + out += Message("content://sms/${it.id}", it.body, it.date, it.address, it.box, SERVICE_SMS, false) + out.size < count + } + val mms = mutableListOf() + val columns = arrayOf(Telephony.Mms._ID, Telephony.Mms.DATE, Telephony.Mms.MESSAGE_BOX) + context.contentResolver.query(Telephony.Mms.CONTENT_URI, columns, "${Telephony.Mms.THREAD_ID} = ?", arrayOf(thread.toString()), "${Telephony.Mms.DATE} DESC")?.use { c -> + while (c.moveToNext() && mms.size < count) { + val id = c.getLong(0) + val box = c.getInt(2) + val (text, kind) = mmsText(id) + // An MMS keeps its date in seconds. + mms += Message("content://mms/$id", text.ifBlank { kind.orEmpty() }, c.getLong(1) * 1000, mmsAddress(id, box), box, SERVICE_MMS, kind != null) + } + } + return (out + mms).sortedByDescending { it.date }.take(count) + } + + /** An MMS's text parts, and a word for its picture or video when it has one. */ + private fun mmsText(id: Long): Pair { + val text = StringBuilder() + var kind: String? = null + context.contentResolver.query(MMS_PARTS, arrayOf(Telephony.Mms.Part.CONTENT_TYPE, Telephony.Mms.Part.TEXT), "${Telephony.Mms.Part.MSG_ID} = ?", arrayOf(id.toString()), null)?.use { c -> + while (c.moveToNext()) { + val type = c.getString(0).orEmpty() + when { + type == "text/plain" -> c.getString(1)?.let { if (text.isNotEmpty()) text.append('\n'); text.append(it) } + type.startsWith("image/") -> kind = kind ?: context.getString(R.string.texts_photo) + type.startsWith("video/") -> kind = kind ?: context.getString(R.string.texts_video) + type != "application/smil" -> kind = kind ?: context.getString(R.string.texts_attachment) + } + } + } + return text.toString() to kind + } + + /** The other side of an MMS: who sent one that came in, who one sent went to. */ + private fun mmsAddress(id: Long, box: Int): String { + val wanted = if (box == Telephony.Mms.MESSAGE_BOX_INBOX) PDU_FROM else PDU_TO + context.contentResolver.query(Uri.parse("content://mms/$id/addr"), arrayOf(Telephony.Mms.Addr.ADDRESS, Telephony.Mms.Addr.TYPE), null, null, null)?.use { c -> + while (c.moveToNext()) if (c.getInt(1) == wanted) c.getString(0)?.takeIf { it.isNotBlank() && it != INSERT_ADDRESS }?.let { return it } + } + return "" + } + + private class Sms(val id: Long, val thread: Long, val address: String, val body: String, val date: Long, val box: Int) + + /** SMS rows newest first, handed to [more] until it says stop. */ + private fun read(selection: String?, args: Array?, more: (Sms) -> Boolean) { + val columns = arrayOf(Telephony.Sms._ID, Telephony.Sms.THREAD_ID, Telephony.Sms.ADDRESS, Telephony.Sms.BODY, Telephony.Sms.DATE, Telephony.Sms.TYPE) + context.contentResolver.query(Telephony.Sms.CONTENT_URI, columns, selection, args, "${Telephony.Sms.DATE} DESC")?.use { c -> + while (c.moveToNext()) { + val address = c.getString(2).orEmpty() + // A draft with no number yet is not a conversation. + if (address.isBlank()) continue + if (!more(Sms(c.getLong(0), c.getLong(1), address, c.getString(3).orEmpty(), c.getLong(4), c.getInt(5)))) break + } + } + } + + /** The contact's name for a number, else the number as the phone writes it. */ + private fun nameOf(number: String): String { + if (allowed(Manifest.permission.READ_CONTACTS)) { + val uri = Uri.withAppendedPath(PhoneLookup.CONTENT_FILTER_URI, Uri.encode(number)) + runCatching { + context.contentResolver.query(uri, arrayOf(PhoneLookup.DISPLAY_NAME), null, null, null)?.use { if (it.moveToFirst()) it.getString(0) else null } + }.getOrNull()?.takeIf { it.isNotBlank() }?.let { return it } + } + val country = context.getSystemService(TelephonyManager::class.java)?.simCountryIso?.uppercase(Locale.ROOT)?.ifEmpty { null } ?: Locale.getDefault().country + return PhoneNumberUtils.formatNumber(number, country) ?: number + } + + private fun allowed(permission: String) = context.checkSelfPermission(permission) == PackageManager.PERMISSION_GRANTED + + /** SgError {1 title, 2 message} in a reply's field 6. */ + private fun refused(): ByteArray = + Wire.fieldBytes(6, Wire.fieldBytes(1, "SMS not allowed".encodeToByteArray()) + Wire.fieldBytes(2, "Allow SMS for Visor on the phone.".encodeToByteArray())) + + companion object { + const val SVC = 70 + private const val MT_LOAD_THREADS = 1 + private const val MT_LOAD_MESSAGES = 2 + private const val MT_OUTGOING = 8 + private const val MT_CHUNK = 32 + private val SERVED = setOf(MT_LOAD_THREADS, MT_LOAD_MESSAGES, MT_OUTGOING) + private const val NATIVE = 3L + private const val DEFAULT_THREADS = 10 + private const val DEFAULT_MESSAGES = 25 + private const val MAX_MESSAGES = 100 + private const val SERVICE_SMS = 1L + private const val SERVICE_MMS = 2L + private const val THREAD_GROUP = 1L + private const val PDU_FROM = 137 + private const val PDU_TO = 151 + private const val INSERT_ADDRESS = "insert-address-token" + private val CONVERSATIONS: Uri = Telephony.Threads.CONTENT_URI.buildUpon().appendQueryParameter("simple", "true").build() + private val CANONICAL_ADDRESSES: Uri = Uri.parse("content://mms-sms/canonical-addresses") + private val MMS_PARTS: Uri = Uri.parse("content://mms/part") + private const val RESPONSE_FILE_BUS = 2L + private const val OUT_SUCCESS = 2L + private const val OUT_FAILURE = 3L + private const val SEND_TIMEOUT_MS = 30_000L + private const val CHUNK = 1000 + private const val CHUNK_HEAD = 52 + private const val SOURCE_SMS = -4 + + // Telephony's box numbers as the Meta app names them. Lowercase: the glasses mark a message as sent only for + // "sent", and any other one of ours shows as still sending. + private fun folder(box: Int) = when (box) { + Telephony.Sms.MESSAGE_TYPE_INBOX -> "inbox" + Telephony.Sms.MESSAGE_TYPE_SENT -> "sent" + Telephony.Sms.MESSAGE_TYPE_DRAFT -> "draft" + Telephony.Sms.MESSAGE_TYPE_OUTBOX -> "outbox" + Telephony.Sms.MESSAGE_TYPE_FAILED -> "failed" + Telephony.Sms.MESSAGE_TYPE_QUEUED -> "queued" + else -> "inbox" + } + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 19d88d29..133c7cf7 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -148,6 +148,7 @@ object ConnectionModule { panels: com.macedonga.visor.connection.Panels, bloks: com.macedonga.visor.connection.BloksRelay, location: com.macedonga.visor.connection.LocationShare, + messages: com.macedonga.visor.connection.PhoneMessages, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -164,6 +165,7 @@ object ConnectionModule { com.macedonga.visor.connection.Panels.SVC to panels, com.macedonga.visor.connection.BloksRelay.SVC to bloks, com.macedonga.visor.connection.LocationShare.SVC to location, + com.macedonga.visor.connection.PhoneMessages.SVC to messages, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), diff --git a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt index 2aab2a54..95d8c744 100644 --- a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt @@ -93,6 +93,7 @@ private fun RequestRow(r: GlassesRequest) { -1 -> stringResource(R.string.requests_tunnel) -2 -> stringResource(R.string.requests_meta) -3 -> stringResource(R.string.requests_location) + -4 -> stringResource(R.string.requests_sms) else -> stringResource(R.string.requests_source, r.source) }, r.options.ifEmpty { null }, diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/GlassesNotification.kt b/app/src/main/kotlin/com/macedonga/visor/notify/GlassesNotification.kt index 6efe614f..c94340a3 100644 --- a/app/src/main/kotlin/com/macedonga/visor/notify/GlassesNotification.kt +++ b/app/src/main/kotlin/com/macedonga/visor/notify/GlassesNotification.kt @@ -70,14 +70,16 @@ data class GlassesNotification( }.toByteArray() out.write(Wire.fieldBytes(12, foa)) } - // Native and SMS: NativeMessageNotificationExtras {1 phone number, 7 threadId, 8 threadName, 12 senderName}. + // Native and SMS: NativeMessageNotificationExtras {1 phone number, 7 threadId, 8 threadName, 11 senderID, + // 12 senderName, 16 shared contact}. The Display's Messages app files it by threadId and senderID. 3, 7 -> out.write( Wire.fieldBytes( 13, (if (senderId.isNotEmpty()) Wire.fieldBytes(1, senderId.toByteArray()) else ByteArray(0)) + Wire.fieldBytes(7, threadId.ifEmpty { key }.toByteArray()) + Wire.fieldBytes(8, (if (isGroup) groupName.ifEmpty { title } else sender.ifEmpty { title }).toByteArray()) + - Wire.fieldBytes(12, sender.ifEmpty { title }.toByteArray()), + (if (senderId.isNotEmpty()) Wire.fieldBytes(11, senderId.toByteArray()) else ByteArray(0)) + + Wire.fieldBytes(12, sender.ifEmpty { title }.toByteArray()) + Wire.fieldVarint(16, 1), ), ) } diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt b/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt index a58815ce..670f4ff3 100644 --- a/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt +++ b/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt @@ -1,13 +1,18 @@ package com.macedonga.visor.notify +import android.Manifest import android.app.Notification import android.app.NotificationManager +import android.app.Person import android.content.ComponentName +import android.content.pm.PackageManager import android.graphics.Bitmap import android.graphics.Canvas import android.media.session.MediaController import android.media.session.MediaSessionManager import android.media.session.PlaybackState +import android.net.Uri +import android.provider.ContactsContract.CommonDataKinds.Phone import android.provider.Telephony import android.service.notification.NotificationListenerService import android.service.notification.StatusBarNotification @@ -59,6 +64,9 @@ class MirrorListener : NotificationListenerService(), Messaging { @Inject lateinit var apps: NotifyApps + @Inject + lateinit var textReplies: TextReplies + // Each conversation sent to the glasses, with the app's reply action, newest last. private val replyable = LinkedHashMap>() @@ -107,7 +115,9 @@ class MirrorListener : NotificationListenerService(), Messaging { val text = (extras.getCharSequence(Notification.EXTRA_BIG_TEXT) ?: extras.getCharSequence(Notification.EXTRA_TEXT))?.toString().orEmpty() val appName = runCatching { packageManager.getApplicationLabel(packageManager.getApplicationInfo(sbn.packageName, 0)).toString() }.getOrDefault(sbn.packageName) val r = Ranking() - PROVIDERS[sbn.packageName]?.let { provider -> + // The phone's texts, SMS or RCS, go as native messages, which the glasses' Messages app files by thread. + val sms = sbn.packageName == Telephony.Sms.getDefaultSmsPackage(this) + (if (sms) PROVIDER_NATIVE else PROVIDERS[sbn.packageName])?.let { provider -> message(sbn, provider, appName)?.let { m -> deliver(m) replyAction(n)?.let { action -> @@ -169,6 +179,7 @@ class MirrorListener : NotificationListenerService(), Messaging { override fun onNotificationRemoved(sbn: StatusBarNotification) { // One held and not yet sent goes no further; one already on the glasses is removed. synchronized(held) { held.remove(sbn.key) } + textReplies.remove(sbn.key) val (provider, uuid) = mirrored.remove(sbn.key) ?: return val request = Wire.fieldBytes(1, sbn.key.toByteArray()) + Wire.fieldVarint(2, provider.toLong()) + Wire.fieldBytes(6, uuid.toByteArray()) connection.submit { it.sendOnService(GlassesNotification.SVC, GlassesNotification.MT_REMOVE, request) } @@ -213,13 +224,24 @@ class MirrorListener : NotificationListenerService(), Messaging { ?: (extras.getCharSequence(Notification.EXTRA_BIG_TEXT) ?: extras.getCharSequence(Notification.EXTRA_TEXT))?.toString().orEmpty() val time = last?.getLong("time")?.takeIf { it > 0 } ?: sbn.postTime if (text.isBlank()) return null + // A reply sent from the notification is added to it as the user's own message: not news for the glasses. + @Suppress("DEPRECATION") + val user = extras.getParcelable(Notification.EXTRA_MESSAGING_PERSON) as? Person + val fromMe = last != null && last.getCharSequence("sender") == null && + (person == null || user != null && person.key == user.key && person.name == user.name) + if (fromMe) return null if (sentAt[sbn.key] == time) return null sentAt[sbn.key] = time while (sentAt.size > 300) sentAt.remove(sentAt.keys.first()) val group = extras.getBoolean(Notification.EXTRA_IS_GROUP_CONVERSATION) + val native = provider == PROVIDER_NATIVE + // A text names its sender by number and its thread as the SMS store does, as svc70 lists them. + val number = if (native) person?.uri?.takeIf { it.startsWith("tel:") }?.let { Uri.decode(it.substringAfter("tel:")) } ?: numberOf(sender) else null + val thread = number?.takeIf { !group }?.let { runCatching { Telephony.Threads.getOrCreateThreadId(this, it).toString() }.getOrNull() } + if (thread != null) replyAction(sbn.notification)?.let { textReplies.put(thread, sbn.key, it) } return GlassesNotification( key = sbn.key, - pkg = sbn.packageName, + pkg = if (native) NATIVE_APP else sbn.packageName, appName = appName, title = sender, text = text, @@ -227,13 +249,23 @@ class MirrorListener : NotificationListenerService(), Messaging { type = TYPE_INCOMING_MESSAGE, provider = provider, sender = sender, - senderId = person?.key ?: person?.uri?.substringAfter(':').orEmpty(), - threadId = sbn.notification.shortcutId ?: sbn.key, + senderId = number ?: person?.key ?: person?.uri?.substringAfter(':').orEmpty(), + threadId = thread ?: sbn.notification.shortcutId ?: sbn.key, isGroup = group, groupName = extras.getCharSequence(Notification.EXTRA_CONVERSATION_TITLE)?.toString().orEmpty(), ) } + /** A contact's number by their name, or the name itself when it is a number. */ + private fun numberOf(name: String): String? { + if (name.isNotEmpty() && name.all { it.isDigit() || it in "+ -()" }) return name.filter { it.isDigit() || it == '+' } + if (checkSelfPermission(Manifest.permission.READ_CONTACTS) != PackageManager.PERMISSION_GRANTED) return null + return runCatching { + contentResolver.query(Phone.CONTENT_URI, arrayOf(Phone.NUMBER), "${Phone.DISPLAY_NAME} = ?", arrayOf(name), null) + ?.use { if (it.moveToFirst()) it.getString(0) else null } + }.getOrNull() + } + // The action that takes typed text: the app's own "Reply". private fun replyAction(n: Notification): Notification.Action? = n.actions?.firstOrNull { it.semanticAction == Notification.Action.SEMANTIC_ACTION_REPLY && !it.remoteInputs.isNullOrEmpty() } @@ -253,8 +285,7 @@ class MirrorListener : NotificationListenerService(), Messaging { private fun ready() = connection.state.value.link == LinkState.READY // The Meta app's filters: nothing ongoing, no group summaries, nothing empty or below default - // importance, and not the SMS app or the dialer, whose messages and calls reach the glasses by - // Bluetooth already. + // importance, and not the dialer, whose calls reach the glasses by Bluetooth already. private fun wanted(sbn: StatusBarNotification, ranking: RankingMap): Boolean { val n = sbn.notification if (sbn.packageName == packageName || sbn.packageName in SYSTEM) return false @@ -263,7 +294,6 @@ class MirrorListener : NotificationListenerService(), Messaging { if (extras.getCharSequence(Notification.EXTRA_TITLE).isNullOrBlank() && extras.getCharSequence(Notification.EXTRA_TEXT).isNullOrBlank()) return false val r = Ranking() if (ranking.getRanking(sbn.key, r) && r.importance < NotificationManager.IMPORTANCE_DEFAULT) return false - if (sbn.packageName == Telephony.Sms.getDefaultSmsPackage(this)) return false if (sbn.packageName == getSystemService(TelecomManager::class.java)?.defaultDialerPackage) return false return true } @@ -291,8 +321,12 @@ class MirrorListener : NotificationListenerService(), Messaging { companion object { private const val PROVIDER_MIRRORED = 8 + private const val PROVIDER_NATIVE = 3 private const val TYPE_INCOMING_MESSAGE = 1 + // The app id the glasses take for the phone's own texts. + private const val NATIVE_APP = "com.android.sms" + // The chat apps the glasses announce themselves, by their NotificationProvider. private val PROVIDERS = mapOf( "com.whatsapp" to 2, "com.whatsapp.w4b" to 2, diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/TextReplies.kt b/app/src/main/kotlin/com/macedonga/visor/notify/TextReplies.kt new file mode 100644 index 00000000..59b5e66e --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/notify/TextReplies.kt @@ -0,0 +1,40 @@ +package com.macedonga.visor.notify + +import android.app.Notification +import android.app.RemoteInput +import android.content.Context +import android.content.Intent +import android.os.Bundle +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Singleton + +/** + * The Messages app's reply actions, by the SMS store's thread each conversation is, while its notification is up. A + * text sent through one goes the way the app sends it, RCS included, which no API offers other apps. + */ +@Singleton +class TextReplies @Inject constructor() { + private val byThread = ConcurrentHashMap>() + + fun put(thread: String, key: String, action: Notification.Action) { + byThread[thread] = key to action + } + + /** The notification [key] is gone, and its reply with it. */ + fun remove(key: String) { + byThread.values.removeIf { it.first == key } + } + + fun has(thread: String) = byThread.containsKey(thread) + + /** Sends [text] in [thread] through the app's reply; null when it went, else why not. */ + fun reply(context: Context, thread: String, text: String): String? { + val action = byThread[thread]?.second ?: return "No reply in that conversation" + val inputs = action.remoteInputs ?: return "The app gives no reply field" + val results = Bundle().apply { inputs.forEach { putCharSequence(it.resultKey, text) } } + val fill = Intent() + RemoteInput.addResultsToIntent(inputs, fill, results) + return runCatching { action.actionIntent.send(context, 0, fill) }.exceptionOrNull()?.let { it.message ?: it.toString() } + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt b/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt index 46970bb3..5f4a3fbd 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt @@ -46,6 +46,7 @@ enum class Need(val title: Int, val why: Int) { NOTIFY(R.string.access_notify, R.string.access_notify_why), MIC(R.string.access_mic, R.string.access_mic_why), CONTACTS(R.string.access_contacts, R.string.access_contacts_why), + SMS(R.string.access_sms, R.string.access_sms_why), LOCATION(R.string.access_location, R.string.access_location_why), /** Location while Visor is in the background: Android asks it on its own, once [LOCATION] is given. */ @@ -62,6 +63,7 @@ enum class Need(val title: Int, val why: Int) { NOTIFY -> context.allows(Manifest.permission.POST_NOTIFICATIONS) MIC -> context.allows(Manifest.permission.RECORD_AUDIO) CONTACTS -> context.allows(Manifest.permission.READ_CONTACTS) + SMS -> context.allows(Manifest.permission.READ_SMS) && context.allows(Manifest.permission.SEND_SMS) LOCATION -> context.allows(Manifest.permission.ACCESS_FINE_LOCATION) LOCATION_ALWAYS -> context.allows(Manifest.permission.ACCESS_BACKGROUND_LOCATION) WIFI -> wifiDirectAllowed(context) @@ -75,6 +77,7 @@ enum class Need(val title: Int, val why: Int) { NOTIFY -> arrayOf(Manifest.permission.POST_NOTIFICATIONS) MIC -> arrayOf(Manifest.permission.RECORD_AUDIO) CONTACTS -> arrayOf(Manifest.permission.READ_CONTACTS) + SMS -> arrayOf(Manifest.permission.READ_SMS, Manifest.permission.SEND_SMS) LOCATION -> arrayOf(Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_COARSE_LOCATION) // Android grants it only after the foreground one, so that one comes first when it is missing. LOCATION_ALWAYS -> if (context.allows(Manifest.permission.ACCESS_FINE_LOCATION)) arrayOf(Manifest.permission.ACCESS_BACKGROUND_LOCATION) diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 506f40fa..2037ec75 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -565,6 +565,11 @@ Per capire le domande successive e Live AI con il riconoscimento del telefono. Il microfono del telefono ascolta solo mentre tieni premuto il microfono in Meta AI. Contatti Così gli occhiali possono dire da chi arriva un messaggio. + Foto + Video + Allegato + SMS + Così gli occhiali possono mostrare i tuoi SMS e inviarne uno. Posizione Per annotare dove gli occhiali hanno fatto ogni foto. Posizione, sempre diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 7648481a..8074a361 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -556,6 +556,11 @@ Чтобы распознавать уточняющие вопросы и Live AI средствами телефона. Собственный микрофон телефона слушает, только пока вы удерживаете микрофон в Meta AI. Контакты Чтобы очки могли сказать, от кого сообщение. + Фото + Видео + Вложение + SMS + Чтобы очки могли показывать ваши SMS и отправлять их. Местоположение Чтобы отмечать, где очки сделали каждое фото. Местоположение всегда diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 0c31e88e..cb405f2a 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -566,6 +566,11 @@ To understand follow-ups and Live AI with the phone\'s recogniser. The phone\'s own microphone listens only while you hold the mic in Meta AI. Contacts So the glasses can say who a message is from. + Photo + Video + Attachment + Text messages + So the glasses can show your texts and send one. Location To note where the glasses took each photo. Location, all the time diff --git a/app/src/main/res/values/strings_requests.xml b/app/src/main/res/values/strings_requests.xml index 710b0bdb..bee0afb9 100644 --- a/app/src/main/res/values/strings_requests.xml +++ b/app/src/main/res/values/strings_requests.xml @@ -8,4 +8,5 @@ tunnel Meta location + SMS From 3f1dfe19364c18f4bcb034557b822ff1727ddb7d Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Wed, 21 Oct 2026 20:17:48 +0200 Subject: [PATCH 41/90] Display: photos sent from the glasses' Messages app go out as MMS. Visor opens svc26, smartglasses_sharing, on every connection with an empty enumerate request, as the Meta app does. Without that channel the glasses' sharing service refused every share ("No remote channel at identifier: MWA"). Visor accepts the photo's offer (7, answered with 8 on Visor's own channel, so with the originator bit set), takes the photo from airship "Sharing" under the offer's uuid, and answers the share intent (5) with its result (6). A share to the phone's texts goes as an MMS through Android's MMS service, scaled to the carrier's size limit. Other destinations are turned down. The airship receiver now asks a list of takers in order, and the assistant still gets each shared photo it wants. Tried on the Display up to Android's MMS service, which took the message. The line used sends no MMS, so delivery is not tested. --- .../visor/connection/GlassesService.kt | 4 + .../com/macedonga/visor/connection/Mms.kt | 154 +++++++++++++++ .../visor/connection/PhoneSharing.kt | 175 ++++++++++++++++++ .../macedonga/visor/di/ConnectionModule.kt | 2 + app/src/main/res/xml/shared_files.xml | 2 + .../visor/core/assistant/Assistant.kt | 7 +- .../com/macedonga/visor/core/media/Airship.kt | 18 +- 7 files changed, 352 insertions(+), 10 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/Mms.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/PhoneSharing.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt index 068cca0f..54eeac44 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt @@ -56,6 +56,9 @@ class GlassesService : Service() { @Inject lateinit var contacts: PhoneContacts + @Inject + lateinit var sharing: PhoneSharing + @Inject lateinit var kind: LinkKind @@ -130,6 +133,7 @@ class GlassesService : Service() { started = true connection.start(scope, targetFrom(intent)) contacts.start(scope) + sharing.start(scope) moveToLe() holdLe() com.macedonga.visor.gallery.notePlaces(this, scope, connection, media.places) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/Mms.kt b/app/src/main/kotlin/com/macedonga/visor/connection/Mms.kt new file mode 100644 index 00000000..57700ad7 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/Mms.kt @@ -0,0 +1,154 @@ +package com.macedonga.visor.connection + +import android.app.Activity +import android.app.PendingIntent +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.graphics.Bitmap +import android.graphics.ImageDecoder +import android.os.Build +import android.telephony.SmsManager +import androidx.core.content.ContextCompat +import androidx.core.content.FileProvider +import java.io.ByteArrayOutputStream +import java.io.File +import java.util.UUID +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.withTimeoutOrNull + +/** + * One picture, with an optional caption, as an MMS through Android's MMS service: an M-Send.req (OMA MMS 1.2) with a + * multipart/related body of SMIL, the JPEG and the text. Android files what a non-default app sends in the SMS store. + */ +object Mms { + /** Sends [image] to [to]; null when the carrier took it, else why not. */ + suspend fun send(context: Context, to: String, image: ByteArray, caption: String): String? { + val jpeg = fit(image, maxBytes(context)) ?: return "The picture could not be read" + val pdu = sendReq(to, jpeg, caption) + val dir = File(context.cacheDir, "mms").apply { mkdirs() } + val file = File(dir, "${UUID.randomUUID()}.pdu").apply { writeBytes(pdu) } + val uri = FileProvider.getUriForFile(context, "${context.packageName}.files", file) + // The MMS service reads the message from the uri in its own process. + for (pkg in listOf("com.android.phone", "com.android.mms.service")) { + runCatching { context.grantUriPermission(pkg, uri, Intent.FLAG_GRANT_READ_URI_PERMISSION) } + } + val action = "${context.packageName}.MMS_SENT.${UUID.randomUUID()}" + val result = CompletableDeferred() + val receiver = object : BroadcastReceiver() { + override fun onReceive(c: Context, intent: Intent) { + result.complete(resultCode) + } + } + ContextCompat.registerReceiver(context, receiver, IntentFilter(action), ContextCompat.RECEIVER_NOT_EXPORTED) + return try { + val sent = PendingIntent.getBroadcast(context, 0, Intent(action).setPackage(context.packageName), PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_ONE_SHOT) + val sms = if (Build.VERSION.SDK_INT >= 31) context.getSystemService(SmsManager::class.java) else @Suppress("DEPRECATION") SmsManager.getDefault() + sms.sendMultimediaMessage(context, uri, null, null, sent) + when (val code = withTimeoutOrNull(SEND_TIMEOUT_MS) { result.await() }) { + null -> "No answer from the phone's MMS service" + Activity.RESULT_OK -> null + else -> "Not sent (MMS error $code)" + } + } catch (e: Exception) { + e.message ?: e.toString() + } finally { + context.unregisterReceiver(receiver) + file.delete() + } + } + + // SmsManager's MMS config needs no permission; the carrier config would need the phone state. + private fun maxBytes(context: Context): Int { + val sms = if (Build.VERSION.SDK_INT >= 31) context.getSystemService(SmsManager::class.java) else @Suppress("DEPRECATION") SmsManager.getDefault() + val max = runCatching { sms.carrierConfigValues.getInt(SmsManager.MMS_CONFIG_MAX_MESSAGE_SIZE, 0) }.getOrDefault(0) + return (if (max > 0) max else DEFAULT_MAX_BYTES) - PDU_ROOM + } + + /** The picture as a JPEG under [maxBytes]: scaled to at most 1600 px, then lower quality until it fits. */ + private fun fit(image: ByteArray, maxBytes: Int): ByteArray? { + val bitmap = runCatching { + ImageDecoder.decodeBitmap(ImageDecoder.createSource(java.nio.ByteBuffer.wrap(image))) { decoder, info, _ -> + val side = maxOf(info.size.width, info.size.height) + if (side > MAX_SIDE) decoder.setTargetSampleSize((side + MAX_SIDE - 1) / MAX_SIDE) + decoder.allocator = ImageDecoder.ALLOCATOR_SOFTWARE + } + }.getOrNull() ?: return null + var scaled = bitmap + while (true) { + for (quality in intArrayOf(85, 70, 55, 40)) { + val out = ByteArrayOutputStream() + scaled.compress(Bitmap.CompressFormat.JPEG, quality, out) + if (out.size() <= maxBytes) return out.toByteArray() + } + if (scaled.width < 320) return null + scaled = Bitmap.createScaledBitmap(scaled, scaled.width * 3 / 4, scaled.height * 3 / 4, true) + } + } + + private fun sendReq(to: String, jpeg: ByteArray, caption: String): ByteArray { + val out = ByteArrayOutputStream() + out.write(byteArrayOf(0x8C.toByte(), 0x80.toByte())) // X-Mms-Message-Type: m-send-req + out.write(0x98) // X-Mms-Transaction-ID + out.write(text(UUID.randomUUID().toString().replace("-", ""))) + out.write(byteArrayOf(0x8D.toByte(), 0x92.toByte())) // X-Mms-MMS-Version: 1.2 + out.write(byteArrayOf(0x89.toByte(), 0x01, 0x81.toByte())) // From: insert-address-token + out.write(0x97) // To + out.write(text("${to.filter { it.isDigit() || it == '+' }}/TYPE=PLMN")) + // Content-Type, last of the headers: multipart/related, start , type application/smil. + out.write(0x84) + out.write(valueLength(byteArrayOf(0xB3.toByte()) + byteArrayOf(0x8A.toByte()) + text("") + byteArrayOf(0x89.toByte()) + text("application/smil"))) + + val parts = mutableListOf( + part(text("application/smil"), "smil.xml", "", smil(caption.isNotBlank()).encodeToByteArray()), + // image/jpeg; name=image.jpg + part(valueLength(byteArrayOf(0x9E.toByte(), 0x85.toByte()) + text("image.jpg")), "image.jpg", "", jpeg), + ) + if (caption.isNotBlank()) { + // text/plain; charset=utf-8 + parts += part(valueLength(byteArrayOf(0x83.toByte(), 0x81.toByte(), 0xEA.toByte())), "text.txt", "", caption.encodeToByteArray()) + } + out.write(uintvar(parts.size.toLong())) + parts.forEach { out.write(it) } + return out.toByteArray() + } + + /** One body part: its encoded content type, then Content-Location and Content-ID, then the data. */ + private fun part(contentType: ByteArray, location: String, id: String, data: ByteArray): ByteArray { + val headers = contentType + byteArrayOf(0x8E.toByte()) + text(location) + byteArrayOf(0xC0.toByte(), 0x22) + id.encodeToByteArray() + byteArrayOf(0) + return uintvar(headers.size.toLong()) + uintvar(data.size.toLong()) + headers + data + } + + private fun smil(withText: Boolean): String = + "" + + "" + + "" + + "" + + (if (withText) "" else "") + + "" + + /** Text-string: a leading byte at or above 0x80 is quoted, and the string ends with a zero. */ + private fun text(s: String): ByteArray { + val b = s.encodeToByteArray() + return (if (b.isNotEmpty() && (b[0].toInt() and 0xFF) >= 0x80) byteArrayOf(0x7F) else ByteArray(0)) + b + byteArrayOf(0) + } + + private fun valueLength(value: ByteArray): ByteArray = + (if (value.size <= 30) byteArrayOf(value.size.toByte()) else byteArrayOf(31) + uintvar(value.size.toLong())) + value + + private fun uintvar(v: Long): ByteArray { + var n = v + val groups = ArrayList() + do { + groups.add((n and 0x7F).toInt()) + n = n shr 7 + } while (n > 0) + return ByteArray(groups.size) { i -> (groups[groups.size - 1 - i] or if (i < groups.size - 1) 0x80 else 0).toByte() } + } + + private const val DEFAULT_MAX_BYTES = 300 * 1024 + private const val PDU_ROOM = 8 * 1024 + private const val MAX_SIDE = 1600 + private const val SEND_TIMEOUT_MS = 120_000L +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneSharing.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneSharing.kt new file mode 100644 index 00000000..e7cba4a7 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneSharing.kt @@ -0,0 +1,175 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.media.AirshipReceiver +import com.macedonga.visor.core.media.ExtraFiles +import com.macedonga.visor.core.media.NAMESPACE_SHARING +import com.macedonga.visor.core.media.Offer +import com.macedonga.visor.core.media.Received +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.File +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull + +/** + * svc26, smartglasses_sharing: the glasses share a capture through the phone. Visor opens the channel on connect with + * an empty enumerate request (3), as the Meta app does; the glasses' sharing service refuses to start without it. A + * photo comes as an offer (7, or 11 for the last capture) that Visor accepts (8), then over airship "Sharing" under the + * offer's uuid, then a media intent (5) naming where it goes. A share to the phone's own texts (provider 3) goes out + * as an MMS and is answered (6) with the result; other destinations are the Meta app's and are turned down. The + * assistant's photo comes the same way and still reaches the assistant. + */ +@Singleton +class PhoneSharing @Inject constructor( + @ApplicationContext private val context: Context, + private val connection: Provider, + private val airship: AirshipReceiver, + private val messages: PhoneMessages, + private val kind: LinkKind, + private val requests: RequestLog, + private val log: Provider, +) : Responder, ExtraFiles { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Sharing: failed: $e") }) + private val files = ConcurrentHashMap>() + + fun start(scope: CoroutineScope) { + airship.extras.add(0, this) + scope.launch { + connection.get().state.map { it.link == LinkState.READY }.distinctUntilChanged().collect { ready -> + if (ready) connection.get().submit { it.sendOnService(SVC, MT_ENUMERATE_REQUEST, ByteArray(0)) } + } + } + } + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.payload.isEmpty() && message.info.flags.isClosed) return + val type = message.info.messageType + // The channel is Visor's own: the glasses answer on it without the originator bit, and Visor writes it with. + val reply = message.info.channelId xor 0x8000 + val f = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return + log.get().add("Sharing: type $type, ${message.payload.size} bytes, fields ${f.fieldNumbers()}") + when (type) { + MT_MEDIA_OFFER, MT_MEDIA_OFFER_LAST -> { + val uuid = f.bytes(1) + files.getOrPut(uuidText(uuid)) { CompletableDeferred() } + connection.send(reply, Wire.fieldBytes(1, uuid) + Wire.fieldVarint(2, ACCEPT), messageType = MT_MEDIA_OFFER_RESPONSE) + } + // The phone keeps the link it has: the glasses are told which one that is. + MT_LINK_SWITCH -> connection.send(reply, Wire.fieldVarint(1, if (kind.overLe.value) LINK_BLE else LINK_BTC), messageType = MT_LINK_UPDATE) + MT_MEDIA_INTENT -> scope.launch { intent(f, reply, connection) } + } + } + + override fun wants(offer: Offer): Boolean = offer.namespace == NAMESPACE_SHARING && files.containsKey(offer.uuid) + + // The receiver deletes its file afterwards, so the photo is copied; the assistant may want it too. + override fun take(received: Received) { + val kept = File(File(context.cacheDir, "share").apply { mkdirs() }, received.offer.uuid) + received.file.copyTo(kept, overwrite = true) + files[received.offer.uuid]?.complete(kept) + airship.extras.filter { it !== this && it.wants(received.offer) }.forEach { runCatching { it.take(received) } } + } + + /** MediaIntent {1 uuid, 2 media uuid, 4 ShareIntent, 8 AssistantOffloadIntent}: answered with MediaIntentResponse. */ + private suspend fun intent(f: Wire.Fields, reply: Int, mux: DataXConnection) { + val uuid = f.bytes(1) + // A share of a photo just offered names no media uuid: the intent's own uuid is the offer's. + val media = uuidText(if (f.has(2)) f.bytes(2) else uuid) + try { + when { + f.has(INTENT_SHARE) -> { + val (result, error) = try { + share(Wire.parse(f.bytes(INTENT_SHARE)), media) + } catch (e: Exception) { + SHARE_ERROR_APP to (e.message ?: e.toString()) + } + val response = (error?.let { Wire.fieldBytes(1, it.encodeToByteArray()) } ?: ByteArray(0)) + Wire.fieldVarint(2, result) + + (if (result == SHARE_SUCCESS) Wire.fieldVarint(3, System.currentTimeMillis()) else ByteArray(0)) + mux.send(reply, Wire.fieldBytes(1, uuid) + Wire.fieldBytes(RESPONSE_SHARE, response), messageType = MT_MEDIA_INTENT_RESPONSE) + } + // The assistant has the photo from the airship transfer. + f.has(INTENT_ASSISTANT_OFFLOAD) -> + mux.send(reply, Wire.fieldBytes(1, uuid) + Wire.fieldBytes(RESPONSE_ASSISTANT_OFFLOAD, Wire.fieldVarint(1, OFFLOAD_SUCCESS)), messageType = MT_MEDIA_INTENT_RESPONSE) + else -> log.get().add("Sharing: intent ${f.fieldNumbers()} not served") + } + } finally { + files.remove(media)?.let { d -> if (d.isCompleted) runCatching { d.getCompleted().delete() } } + } + } + + /** ShareIntent {2 ShareDestination {1 provider, 3 thread, 6 numbers}, 5 text, 8 share id}: the result and why not. */ + private suspend fun share(share: Wire.Fields, media: String): Pair { + val destination = Wire.parse(share.bytes(2)) + val id = requests.start("mms://share/${share.bytes(8).decodeToString()}", SOURCE_SMS) + val provider = destination.varint(1) + val outcome: Pair = if (provider != PROVIDER_NATIVE) { + SHARE_ERROR_APP to "Only the phone's texts can be sent through Visor" + } else { + val numbers = destination.repeatedBytes(6).map { Wire.parse(it).bytes(1).decodeToString() }.filter { it.isNotBlank() } + val to = numbers.ifEmpty { messages.addressesOf(destination.bytes(3).decodeToString()) } + val file = files[media]?.let { withTimeoutOrNull(FILE_WAIT_MS) { it.await() } } + when { + context.checkSelfPermission(Manifest.permission.SEND_SMS) != PackageManager.PERMISSION_GRANTED -> SHARE_ERROR_NATIVE_DISABLED to "SMS not allowed in Visor" + to.isEmpty() -> SHARE_ERROR_APP to "No recipient" + to.size > 1 -> SHARE_ERROR_APP to "Group messages are not supported" + file == null -> SHARE_ERROR_MEDIA_PROCESSING to "The picture did not arrive" + else -> Mms.send(context, to.single(), file.readBytes(), share.bytes(5).decodeToString())?.let { SHARE_ERROR_APP to it } ?: (SHARE_SUCCESS to null) + } + } + requests.finish(id, outcome.second ?: "sent as MMS", failed = outcome.second != null) + return outcome + } + + private fun uuidText(raw: ByteArray): String { + val h = raw.joinToString("") { "%02x".format(it) } + if (raw.size != 16) return h + return "${h.substring(0, 8)}-${h.substring(8, 12)}-${h.substring(12, 16)}-${h.substring(16, 20)}-${h.substring(20)}" + } + + companion object { + const val SVC = 26 + private const val MT_ENUMERATE_REQUEST = 3 + private const val MT_MEDIA_INTENT = 5 + private const val MT_MEDIA_INTENT_RESPONSE = 6 + private const val MT_MEDIA_OFFER = 7 + private const val MT_MEDIA_OFFER_RESPONSE = 8 + private const val MT_MEDIA_OFFER_LAST = 11 + private const val MT_LINK_SWITCH = 12 + private const val MT_LINK_UPDATE = 13 + private const val ACCEPT = 0L + private const val LINK_BLE = 1L + private const val LINK_BTC = 2L + private const val INTENT_SHARE = 4 + private const val INTENT_ASSISTANT_OFFLOAD = 8 + private const val RESPONSE_SHARE = 2 + private const val RESPONSE_ASSISTANT_OFFLOAD = 5 + private const val OFFLOAD_SUCCESS = 1L + private const val PROVIDER_NATIVE = 3L + private const val SHARE_SUCCESS = 1L + private const val SHARE_ERROR_MEDIA_PROCESSING = 2L + private const val SHARE_ERROR_APP = 5L + private const val SHARE_ERROR_NATIVE_DISABLED = 8L + private const val FILE_WAIT_MS = 90_000L + private const val SOURCE_SMS = -4 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 133c7cf7..4c6f9be3 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -149,6 +149,7 @@ object ConnectionModule { bloks: com.macedonga.visor.connection.BloksRelay, location: com.macedonga.visor.connection.LocationShare, messages: com.macedonga.visor.connection.PhoneMessages, + sharing: com.macedonga.visor.connection.PhoneSharing, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -166,6 +167,7 @@ object ConnectionModule { com.macedonga.visor.connection.BloksRelay.SVC to bloks, com.macedonga.visor.connection.LocationShare.SVC to location, com.macedonga.visor.connection.PhoneMessages.SVC to messages, + com.macedonga.visor.connection.PhoneSharing.SVC to sharing, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), diff --git a/app/src/main/res/xml/shared_files.xml b/app/src/main/res/xml/shared_files.xml index 95a25138..ac6206ca 100644 --- a/app/src/main/res/xml/shared_files.xml +++ b/app/src/main/res/xml/shared_files.xml @@ -3,4 +3,6 @@ + + diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt index b14122ad..c8c7fd13 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt @@ -7,6 +7,7 @@ import com.macedonga.visor.core.connection.GlassesLink import com.macedonga.visor.core.media.AirshipReceiver import com.macedonga.visor.core.media.ExtraFiles import com.macedonga.visor.core.media.NAMESPACE_SHARING +import com.macedonga.visor.core.media.Offer import com.macedonga.visor.core.media.Received import com.macedonga.visor.core.protocol.Thrift import com.macedonga.visor.core.protocol.ThriftWriter @@ -306,8 +307,10 @@ class Assistant( init { // The assistant's photo comes over airship, in a namespace the media import refuses. - airship.extra = object : ExtraFiles { - override fun wants(namespace: String, size: Long): Boolean { + airship.extras += object : ExtraFiles { + override fun wants(offer: Offer): Boolean { + val namespace = offer.namespace + val size = offer.size val waiting = pending != null && size < 20_000_000 || namespace == NAMESPACE_SHARING && store.config.value.enabled && size < EARLY_MAX_BYTES if (waiting) say("Taking ${size / 1000} KB in namespace \"$namespace\" for the photo") diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Airship.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Airship.kt index d4b2b805..f615284c 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Airship.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Airship.kt @@ -5,6 +5,7 @@ import com.macedonga.visor.core.session.DataXMessage import com.macedonga.visor.core.session.Responder import java.io.File import java.io.FileOutputStream +import java.util.concurrent.CopyOnWriteArrayList import java.util.zip.Adler32 import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.Flow @@ -61,7 +62,7 @@ data class Transfer(val uuid: String, val received: Long, val size: Long, val by /** Files for someone other than the media import, such as the assistant's photo. */ interface ExtraFiles { /** Whether to take an offer outside the media and bug report namespaces. */ - fun wants(namespace: String, size: Long): Boolean + fun wants(offer: Offer): Boolean /** A file this side wanted, whole. The receiver deletes it afterwards. */ fun take(received: Received) @@ -77,10 +78,10 @@ class AirshipReceiver(private val spool: File) : Responder { @Volatile var describe: (String) -> String = { "" } - @Volatile - var extra: ExtraFiles? = null + /** Takers of other files, asked in order; the first that wants an offer gets it. */ + val extras = CopyOnWriteArrayList() - private class Live(val offer: Offer, val file: File) { + private class Live(val offer: Offer, val file: File, val taker: ExtraFiles?) { var out = FileOutputStream(file) val adler = Adler32() var received = 0L @@ -262,13 +263,14 @@ class AirshipReceiver(private val spool: File) : Responder { val f = fields(payload) val offer = Offer(f.bytes(1), f.long(2), f.bytes(3).decodeToString()) live.remove(channel)?.discard() + val taker = if (offer.namespace in ACCEPTED) null else extras.firstOrNull { runCatching { it.wants(offer) }.getOrDefault(false) } // Refusing still needs the drop, or the glasses' queue stalls behind this offer. - if (offer.namespace !in ACCEPTED && extra?.wants(offer.namespace, offer.size) != true) { + if (offer.namespace !in ACCEPTED && taker == null) { if (offer.namespace !in QUIET) _notes.trySend("Refused ${sizeWords(offer.size)} in namespace \"${offer.namespace}\"") return MT_DROP to fileId(offer.raw) } spool.mkdirs() - live[channel] = Live(offer, File(spool, offer.uuid)) + live[channel] = Live(offer, File(spool, offer.uuid), taker) _notes.trySend("Receiving ${sizeWords(offer.size)}${describe(offer.uuid)}") publish() // Offset 0 with the adler32 of nothing: what FileReceiver::acceptOffer sends. @@ -298,10 +300,10 @@ class AirshipReceiver(private val spool: File) : Responder { val verified = f.long(1) == t.received && f.long(2) == t.adler.value val whole = Received(t.offer, t.file, verified) if (t.offer.namespace in ACCEPTED) { - extra?.let { runCatching { it.peek(whole) } } + extras.forEach { runCatching { it.peek(whole) } } _received.trySend(whole) } else { - extra?.let { runCatching { it.take(whole) } } + t.taker?.let { runCatching { it.take(whole) } } t.file.delete() } publish() From 445dd983c133c49aaaeffb33c53cc25df66b25f8 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Thu, 22 Oct 2026 09:27:47 +0200 Subject: [PATCH 42/90] Display: answer the delivery check, the ping and motion activity. Visor confirms the Display's svc4 delivery check with a bare DataX OK (status 0), as the Display's connection monitor expects. Unanswered, its counter stayed at 1; answered, it counts up on one channel. svc76 pings get PingResult with whether the phone is online. svc101 gets the Meta app's answer with driving detection off (SETTINGS_NOT_ENABLED), which the Display reads as "not driving". Products 270 and 271 count as Displays in the features catalogue too. CLAUDE.md lists the Display's protocol traps, and the session's note on 0x1000 now says it is svc36's own result. --- CLAUDE.md | 19 ++++++++++++++ .../visor/connection/InternetSwitch.kt | 10 +++++++ .../visor/connection/MotionActivity.kt | 26 +++++++++++++++++++ .../macedonga/visor/di/ConnectionModule.kt | 2 ++ .../com/macedonga/visor/features/Catalog.kt | 6 ++--- .../visor/core/session/GlassesSession.kt | 16 +++++++++++- 6 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/MotionActivity.kt diff --git a/CLAUDE.md b/CLAUDE.md index 61e647c4..e5021212 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,6 +37,25 @@ the frame style and language packs. Works on real glasses as of 2026-10-05. `s` low-half. Any mistake shows up as `0x1043 INVALID_APP_IDENTITY`. - Airship: accepting a file (or refusing it) must end with a drop on type 7, or the glasses' send queue stalls behind it. +- DataX OK on the wire is status 0. 0x1000 is svc36's own result; sent as OK, it made the Display's + image loader retry every image. +- Replies on a channel the glasses opened go on `DataXConnection.mirror`. Replies on a channel Visor + opened keep the originator bit. + +Display: + +- svc28 device state comes wrapped in HNDeviceState `{1: stateCommon}`. Unwrapped as a plain + DeviceState, the battery reads 0%. +- svc41108: the Display takes the phone's internet state only on a channel the phone opens. Until + then its apps stay off the phone's internet. +- svc26 must be opened with an empty type 3 on every connection, or every share fails ("No remote + channel at identifier: MWA"). +- svc70 folder names are lowercase. With "SENT" a sent text reloads as still sending. +- A text notification needs app id `com.android.sms`, provider 3 and the SMS thread id, or the + Messages app drops it. +- Captions and translation turn on only through a svc29 pack install and FEATURE_CHANGE. The same + pack on airship `language_resource` turns nothing on. +- Each Meta `/login` exchange ends the older sessions. Keep one token per run. ## Build diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt index a2f181f0..a3ee79e5 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt @@ -48,6 +48,13 @@ class InternetSwitch @Inject constructor( watchNetwork() } + /** svc76, wearables_ping: a ping (type 1) is answered with PingResult {1 hasInternet} (type 2). */ + val ping = Responder { message, mux -> + if (message.info.messageType != MT_PING || message.info.flags.isClosed) return@Responder + val online = phoneInternet().connected + mux.send(DataXConnection.mirror(message.info.channelId), Wire.fieldVarint(1, if (online) 1 else 0), messageType = MT_PING_RESULT) + } + /** From the first question on, every change of the phone's network goes to the Display while it is connected. */ private fun watchNetwork() { if (!watching.compareAndSet(false, true)) return @@ -86,5 +93,8 @@ class InternetSwitch @Inject constructor( const val SVC = 41108 private const val MT_COMPANION_STATE = 3 private const val MT_LISTEN = 6 + const val PING_SVC = 76 + private const val MT_PING = 1 + private const val MT_PING_RESULT = 2 } } diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/MotionActivity.kt b/app/src/main/kotlin/com/macedonga/visor/connection/MotionActivity.kt new file mode 100644 index 00000000..36966726 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/MotionActivity.kt @@ -0,0 +1,26 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder + +/** + * svc101, motion_activity_service: the Display's driving detection subscribes (type 0) and asks (type 2) whether the + * phone sees its owner driving. Visor has no activity recognition, so it answers every message as the Meta app does + * with driving detection off: MotionActivityServiceError {1 SETTINGS_NOT_ENABLED} (type 3), read as "not driving". + */ +object MotionActivity : Responder { + const val SVC = 101 + private const val MT_SERVICE_ERROR = 3 + private const val SETTINGS_NOT_ENABLED = 2L + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed) return + connection.send( + DataXConnection.mirror(message.info.channelId), + Wire.fieldVarint(1, SETTINGS_NOT_ENABLED), + messageType = MT_SERVICE_ERROR, + ) + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 4c6f9be3..2e5f2e83 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -161,6 +161,8 @@ object ConnectionModule { SVC_TAR to timers, GlassesNotification.SVC to icons, com.macedonga.visor.connection.InternetSwitch.SVC to internet, + com.macedonga.visor.connection.InternetSwitch.PING_SVC to internet.ping, + com.macedonga.visor.connection.MotionActivity.SVC to com.macedonga.visor.connection.MotionActivity, com.macedonga.visor.connection.DataFetcher.SVC to fetcher, com.macedonga.visor.connection.PhoneInternet.SVC to tunnel, com.macedonga.visor.connection.Panels.SVC to panels, diff --git a/app/src/main/kotlin/com/macedonga/visor/features/Catalog.kt b/app/src/main/kotlin/com/macedonga/visor/features/Catalog.kt index af51d6fe..19aa79ad 100644 --- a/app/src/main/kotlin/com/macedonga/visor/features/Catalog.kt +++ b/app/src/main/kotlin/com/macedonga/visor/features/Catalog.kt @@ -38,8 +38,8 @@ sealed interface Gate { } // Meta Ray-Ban Display, whose image has no hearingservice. -private const val DISPLAY = 259 -private val NO_HEARING_SERVICE = setOf(DISPLAY) +private val DISPLAY = setOf(259, 270, 271) +private val NO_HEARING_SERVICE = DISPLAY // The hearing service starts focus only above the minimum build MobileConfig gives this model (libhearing's // getDeviceTypeMinSoCVersion); -1 or none, as on glasses Meta never configured, is INVALID_DEVICE. @@ -103,7 +103,7 @@ enum class Feature( ), Topic.HEARING, // The Display ships neither aurachat nor the group app. - absentOn = setOf(DISPLAY), + absentOn = DISPLAY, ), // The glasses' TAR gate (isSgAlarmMCEnabled); Visor's timer service keeps the alarms Meta's app refuses. ALARMS( diff --git a/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt b/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt index 083c7c0c..407a7cad 100644 --- a/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt +++ b/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt @@ -267,6 +267,15 @@ class GlassesSession( mux.sendStatus(DataXConnection.mirror(info.channelId), SERVICE_NOT_FOUND) } + // svc4, connection_monitor: the glasses' delivery check (type 0, a counter) is confirmed with a bare OK status. + if (serviceId == CONNECTION_MONITOR && + (info.channelId and 0x8000) != 0 && + !info.flags.hasError && + !info.flags.isClosed + ) { + mux.sendStatus(DataXConnection.mirror(info.channelId), OK) + } + val responder = responders[serviceId] if (responder != null && !info.flags.hasError) { try { @@ -392,12 +401,17 @@ class GlassesSession( private val BACKOFF_MILLIS = longArrayOf(1_000, 2_000, 4_000, 8_000) - /** 0x1000 is the DataX success status every service answers with. */ + /** 0x1000 is the success status svc36 answers with; DataX's own is [OK]. */ private const val SUCCESS = 0x1000 /** DataX service_not_found: what a peer answers for a service it does not host. */ private const val SERVICE_NOT_FOUND = 0xC001 private val REFUSED_PEER_SERVICES = setOf(30, 79) + + /** DataX OK, the status a delivery check is confirmed with. */ + private const val OK = 0 + + private const val CONNECTION_MONITOR = 4 } } From 546b53e825ab9bc1f1bfa44c2aa963cca6a28a99 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Thu, 22 Oct 2026 10:28:17 +0200 Subject: [PATCH 43/90] Display: recent calls in the Phone app. The Display's Phone app asks for recent calls on svc70 (LoadCallLogRequest, type 11) once per provider and waits for each answer, so it loaded forever. Visor answers with LoadCallLogResponse (type 12): the phone's own calls from Android's call log for provider 3, and an empty list for the other providers, as the Meta app does when it has no handler. Visor also answers svc11006, phone_book_sync, which the Display's telephony service asks on connect and after each call: the reply says the phone is Android and carries the call log as the JSON the Display parses. Phone access asks for call history. Confirmed on the Display. --- app/src/main/AndroidManifest.xml | 1 + .../macedonga/visor/connection/PhoneCalls.kt | 152 ++++++++++++++++++ .../visor/connection/PhoneMessages.kt | 5 + .../macedonga/visor/di/ConnectionModule.kt | 2 + .../macedonga/visor/log/RequestLogScreen.kt | 1 + .../macedonga/visor/settings/PhoneAccess.kt | 3 + app/src/main/res/values-it/strings.xml | 2 + .../main/res/values-it/strings_requests.xml | 1 + app/src/main/res/values-ru/strings.xml | 2 + app/src/main/res/values/strings.xml | 2 + app/src/main/res/values/strings_requests.xml | 1 + 11 files changed, 172 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/PhoneCalls.kt diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 747e8af0..7939a0b4 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -33,6 +33,7 @@ + diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneCalls.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneCalls.kt new file mode 100644 index 00000000..2798ff44 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneCalls.kt @@ -0,0 +1,152 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.provider.CallLog +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import org.json.JSONArray +import org.json.JSONObject + +/** + * The phone's recent calls, asked for twice by the Display: + * + * - svc11006, phone_book_sync, by its telephony service, on connect, after a call and now and then: type 1 {1 request + * id}, answered with PhoneBookSyncResponse {1 request id, 2 call log, 3 error, 4 phone OS} (type 2). Told the phone is + * Android, it keeps the call log from the answer; an iPhone it would read over PBAP instead. The call log is JSON, a + * list of {"cType": "CDEntryAdd", "id", "entry": {id, cachedName, phoneNumber, type, dateEpochMilliseconds, + * durationSeconds}} with the type by name (INCOMING, OUTGOING, MISSED, ...). + * - svc70 LoadCallLogRequest {1 provider, 2 request id, 3 count, 4 request type} (type 11), by the Phone app once per + * provider, answered with LoadCallLogResponse {1 request id, 2 CallLog} (type 12). The phone's own calls (provider 3) + * come from the call log; the other providers get an empty list, as the Meta app gives when it has no handler. + */ +@Singleton +class PhoneCalls @Inject constructor( + @ApplicationContext private val context: Context, + private val requests: RequestLog, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, _ -> }) + + private data class Call(val id: Long, val name: String?, val number: String?, val type: Int, val date: Long, val duration: Long, val video: Boolean) + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.messageType != MT_SYNC_REQUEST || message.info.flags.isClosed) return + val requestId = runCatching { Wire.parse(message.payload).bytes(1) }.getOrDefault(ByteArray(0)) + val reply = DataXConnection.mirror(message.info.channelId) + scope.launch { + val id = requests.start("calls://sync", SOURCE_CALLS) + val calls = if (allowed()) recent(MAX_CALLS) else null + val json = JSONArray() + calls?.forEach { c -> + val entry = JSONObject() + .put("id", c.id) + .put("cachedName", c.name ?: JSONObject.NULL) + .put("phoneNumber", c.number ?: JSONObject.NULL) + .put("type", typeName(c.type)) + .put("dateEpochMilliseconds", c.date) + .put("durationSeconds", c.duration) + json.put(JSONObject().put("cType", "CDEntryAdd").put("id", c.date.toString()).put("entry", entry)) + } + val body = Wire.fieldBytes(1, requestId) + + Wire.fieldBytes(2, (if (calls != null) json.toString() else "").encodeToByteArray()) + + Wire.fieldVarint(3, if (calls != null) SUCCESS else NO_PERMISSION) + + Wire.fieldVarint(4, OS_ANDROID) + connection.send(reply, body, messageType = MT_SYNC_RESPONSE) + if (calls != null) requests.finish(id, "${calls.size} calls", failed = false, bytes = body.size.toLong()) + else requests.finish(id, "call history not allowed", failed = true) + } + } + + /** svc70 type 11, from [PhoneMessages]: [reply] is the request's mirror channel. */ + fun loadCallLog(request: Wire.Fields, reply: Int, mux: DataXConnection) { + scope.launch { + val provider = request.varint(1) + val id = requests.start("calls://recent/$provider", SOURCE_CALLS) + val count = request.varint(3).toInt().takeIf { it > 0 }?.coerceAtMost(MAX_CALLS) ?: DEFAULT_RECENT + val calls = if (provider == NATIVE && allowed()) recent(count) else emptyList() + var body = Wire.fieldBytes(1, request.bytes(2)) + for (c in calls) { + // CallLog: 1 name, 3 time, 5 type (1 incoming, 2 outgoing, 3 missed, 0 other), 7 number, 11 video. + if (c.type == CallLog.Calls.VOICEMAIL_TYPE) continue + body += Wire.fieldBytes( + 2, + (if (!c.name.isNullOrEmpty()) Wire.fieldBytes(1, c.name.encodeToByteArray()) else ByteArray(0)) + + Wire.fieldVarint(3, c.date) + Wire.fieldVarint(5, wireType(c.type)) + + (if (!c.number.isNullOrEmpty()) Wire.fieldBytes(7, c.number.encodeToByteArray()) else ByteArray(0)) + + (if (c.video) Wire.fieldVarint(11, 1) else ByteArray(0)), + ) + } + mux.send(reply, body, messageType = MT_LOAD_CALL_LOG_RESPONSE) + val outcome = when { + provider != NATIVE -> "not the phone's, none" + !allowed() -> "call history not allowed" + else -> "${calls.size} calls" + } + requests.finish(id, outcome, failed = provider == NATIVE && !allowed(), bytes = body.size.toLong()) + } + } + + private fun allowed() = context.checkSelfPermission(Manifest.permission.READ_CALL_LOG) == PackageManager.PERMISSION_GRANTED + + private fun recent(count: Int): List { + val calls = ArrayList() + val columns = arrayOf( + CallLog.Calls._ID, CallLog.Calls.CACHED_NAME, CallLog.Calls.NUMBER, CallLog.Calls.TYPE, CallLog.Calls.DATE, + CallLog.Calls.DURATION, CallLog.Calls.FEATURES, + ) + val uri = CallLog.Calls.CONTENT_URI.buildUpon().appendQueryParameter(CallLog.Calls.LIMIT_PARAM_KEY, count.toString()).build() + context.contentResolver.query(uri, columns, null, null, "${CallLog.Calls.DATE} DESC")?.use { c -> + while (c.moveToNext() && calls.size < count) { + calls += Call( + c.getLong(0), c.getString(1), c.getString(2), c.getInt(3), c.getLong(4), c.getLong(5), + c.getInt(6) and CallLog.Calls.FEATURES_VIDEO != 0, + ) + } + } + return calls + } + + private fun typeName(type: Int): String = when (type) { + CallLog.Calls.INCOMING_TYPE -> "INCOMING" + CallLog.Calls.OUTGOING_TYPE -> "OUTGOING" + CallLog.Calls.MISSED_TYPE -> "MISSED" + CallLog.Calls.VOICEMAIL_TYPE -> "VOICEMAIL" + CallLog.Calls.REJECTED_TYPE -> "REJECTED" + CallLog.Calls.BLOCKED_TYPE -> "BLOCKED" + CallLog.Calls.ANSWERED_EXTERNALLY_TYPE -> "ANSWERED_EXTERNALLY" + else -> "UNKNOWN" + } + + // A rejected call shows as missed, as the Meta app sends it. + private fun wireType(type: Int): Long = when (type) { + CallLog.Calls.INCOMING_TYPE -> 1 + CallLog.Calls.OUTGOING_TYPE -> 2 + CallLog.Calls.MISSED_TYPE, CallLog.Calls.REJECTED_TYPE -> 3 + else -> 0 + } + + companion object { + const val SVC = 11006 + private const val MT_SYNC_REQUEST = 1 + private const val MT_SYNC_RESPONSE = 2 + private const val MT_LOAD_CALL_LOG_RESPONSE = 12 + private const val SUCCESS = 0L + private const val NO_PERMISSION = 1L + private const val OS_ANDROID = 1L + private const val NATIVE = 3L + private const val MAX_CALLS = 100 + private const val DEFAULT_RECENT = 15 + private const val SOURCE_CALLS = -5 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt index cb1996bd..f8fdf3fe 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt @@ -45,6 +45,7 @@ import kotlinx.coroutines.withTimeoutOrNull * is one; any other goes as SMS, answered once the radio has sent it. Each request comes on a channel of its own and * is answered with its own type; a reply over 1000 bytes goes as type-32 chunks when the request allows them. Texts * that arrive reach the glasses as native notifications, from the SMS app's own notifications (see MirrorListener). + * The Phone app's recent calls (type 11) come on this service too; [PhoneCalls] answers them. * * The RCS chats Google Messages files as MMS are marked restricted, and Android shows those rows only to the default * SMS app: a chat with nothing else in it is missing from the list here. @@ -55,6 +56,7 @@ class PhoneMessages @Inject constructor( private val requests: RequestLog, private val log: Provider, private val replies: TextReplies, + private val calls: PhoneCalls, ) : Responder { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Messages: failed: $e") }) @@ -63,6 +65,8 @@ class PhoneMessages @Inject constructor( val request = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return val type = message.info.messageType val reply = DataXConnection.mirror(message.info.channelId) + // The Phone app's recent calls, asked once per provider, each waiting for its answer. + if (type == MT_LOAD_CALL_LOG) return calls.loadCallLog(request, reply, connection) if (type !in SERVED) return log.get().add("Messages: type $type not served") // Only the phone's texts; the other providers are the Meta app's. val provider = request.varint(1) @@ -346,6 +350,7 @@ class PhoneMessages @Inject constructor( private const val MT_LOAD_THREADS = 1 private const val MT_LOAD_MESSAGES = 2 private const val MT_OUTGOING = 8 + private const val MT_LOAD_CALL_LOG = 11 private const val MT_CHUNK = 32 private val SERVED = setOf(MT_LOAD_THREADS, MT_LOAD_MESSAGES, MT_OUTGOING) private const val NATIVE = 3L diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 2e5f2e83..4de09a84 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -150,6 +150,7 @@ object ConnectionModule { location: com.macedonga.visor.connection.LocationShare, messages: com.macedonga.visor.connection.PhoneMessages, sharing: com.macedonga.visor.connection.PhoneSharing, + calls: com.macedonga.visor.connection.PhoneCalls, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -170,6 +171,7 @@ object ConnectionModule { com.macedonga.visor.connection.LocationShare.SVC to location, com.macedonga.visor.connection.PhoneMessages.SVC to messages, com.macedonga.visor.connection.PhoneSharing.SVC to sharing, + com.macedonga.visor.connection.PhoneCalls.SVC to calls, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), diff --git a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt index 95d8c744..949663c0 100644 --- a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt @@ -94,6 +94,7 @@ private fun RequestRow(r: GlassesRequest) { -2 -> stringResource(R.string.requests_meta) -3 -> stringResource(R.string.requests_location) -4 -> stringResource(R.string.requests_sms) + -5 -> stringResource(R.string.requests_calls) else -> stringResource(R.string.requests_source, r.source) }, r.options.ifEmpty { null }, diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt b/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt index 5f4a3fbd..c414a8fd 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt @@ -47,6 +47,7 @@ enum class Need(val title: Int, val why: Int) { MIC(R.string.access_mic, R.string.access_mic_why), CONTACTS(R.string.access_contacts, R.string.access_contacts_why), SMS(R.string.access_sms, R.string.access_sms_why), + CALLS(R.string.access_calls, R.string.access_calls_why), LOCATION(R.string.access_location, R.string.access_location_why), /** Location while Visor is in the background: Android asks it on its own, once [LOCATION] is given. */ @@ -64,6 +65,7 @@ enum class Need(val title: Int, val why: Int) { MIC -> context.allows(Manifest.permission.RECORD_AUDIO) CONTACTS -> context.allows(Manifest.permission.READ_CONTACTS) SMS -> context.allows(Manifest.permission.READ_SMS) && context.allows(Manifest.permission.SEND_SMS) + CALLS -> context.allows(Manifest.permission.READ_CALL_LOG) LOCATION -> context.allows(Manifest.permission.ACCESS_FINE_LOCATION) LOCATION_ALWAYS -> context.allows(Manifest.permission.ACCESS_BACKGROUND_LOCATION) WIFI -> wifiDirectAllowed(context) @@ -78,6 +80,7 @@ enum class Need(val title: Int, val why: Int) { MIC -> arrayOf(Manifest.permission.RECORD_AUDIO) CONTACTS -> arrayOf(Manifest.permission.READ_CONTACTS) SMS -> arrayOf(Manifest.permission.READ_SMS, Manifest.permission.SEND_SMS) + CALLS -> arrayOf(Manifest.permission.READ_CALL_LOG) LOCATION -> arrayOf(Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_COARSE_LOCATION) // Android grants it only after the foreground one, so that one comes first when it is missing. LOCATION_ALWAYS -> if (context.allows(Manifest.permission.ACCESS_FINE_LOCATION)) arrayOf(Manifest.permission.ACCESS_BACKGROUND_LOCATION) diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 2037ec75..e8f72af7 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -570,6 +570,8 @@ Allegato SMS Così gli occhiali possono mostrare i tuoi SMS e inviarne uno. + Registro chiamate + Così gli occhiali possono mostrare le tue chiamate recenti. Posizione Per annotare dove gli occhiali hanno fatto ogni foto. Posizione, sempre diff --git a/app/src/main/res/values-it/strings_requests.xml b/app/src/main/res/values-it/strings_requests.xml index 1817dd8d..7507a677 100644 --- a/app/src/main/res/values-it/strings_requests.xml +++ b/app/src/main/res/values-it/strings_requests.xml @@ -8,4 +8,5 @@ tunnel Meta posizione + chiamate diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 8074a361..d546bc33 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -561,6 +561,8 @@ Вложение SMS Чтобы очки могли показывать ваши SMS и отправлять их. + Журнал звонков + Чтобы очки могли показывать ваши недавние звонки. Местоположение Чтобы отмечать, где очки сделали каждое фото. Местоположение всегда diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index cb405f2a..4db22405 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -571,6 +571,8 @@ Attachment Text messages So the glasses can show your texts and send one. + Call history + So the glasses can show your recent calls. Location To note where the glasses took each photo. Location, all the time diff --git a/app/src/main/res/values/strings_requests.xml b/app/src/main/res/values/strings_requests.xml index bee0afb9..f88feeed 100644 --- a/app/src/main/res/values/strings_requests.xml +++ b/app/src/main/res/values/strings_requests.xml @@ -9,4 +9,5 @@ Meta location SMS + calls From 7950040d746db1d3a0105145060e3f32bf0fdd44 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Thu, 22 Oct 2026 11:12:22 +0200 Subject: [PATCH 44/90] Display: album art on the music screen. With an Android phone the Display's music screen ignores the cover art that comes over AVRCP and takes it only from svc9012, partner_ecosystem, which Visor did not serve. Visor now answers the Display's subscription (type 5) as the Meta app does: the playing app as MediaPlayerInfo (type 100), and the cover from the media session as a JPEG of at most 300 x 300 in 1024-byte pieces (101, the last one 102), again whenever the track or the player changes. Confirmed on the Display with YouTube Music. --- CLAUDE.md | 2 + .../macedonga/visor/connection/PhoneMedia.kt | 180 ++++++++++++++++++ .../macedonga/visor/di/ConnectionModule.kt | 2 + 3 files changed, 184 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/PhoneMedia.kt diff --git a/CLAUDE.md b/CLAUDE.md index e5021212..0aed14a9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,6 +56,8 @@ Display: - Captions and translation turn on only through a svc29 pack install and FEATURE_CHANGE. The same pack on airship `language_resource` turns nothing on. - Each Meta `/login` exchange ends the older sessions. Keep one token per run. +- With an Android phone the music screen ignores AVRCP cover art. The cover comes only over svc9012. +- The Phone app's recent calls wait on svc70 type 11, once per provider. svc11006 does not fill them. ## Build diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMedia.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMedia.kt new file mode 100644 index 00000000..84ac126c --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMedia.kt @@ -0,0 +1,180 @@ +package com.macedonga.visor.connection + +import android.content.ComponentName +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.media.MediaMetadata +import android.media.session.MediaController +import android.media.session.MediaSessionManager +import android.media.session.PlaybackState +import android.net.Uri +import android.os.Handler +import android.os.Looper +import androidx.core.app.NotificationManagerCompat +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.notify.MirrorListener +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.ByteArrayOutputStream +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +/** + * svc9012, partner_ecosystem, for what the phone plays. The Display subscribes (type 5) on connect and takes, on that + * channel, MediaPlayerInfo {1 player, 2 content type, 3 liked, 4 package} (type 100) and the cover art as a JPEG in + * 1024-byte pieces (101, the last one 102; an empty 102 clears it), again whenever the playing session changes. With an + * Android phone the Display's music screen takes the cover only from here; title and artist come over AVRCP. + */ +@Singleton +class PhoneMedia @Inject constructor(@ApplicationContext private val context: Context) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main + CoroutineExceptionHandler { _, _ -> }) + private val sending = Mutex() + private val main = Handler(Looper.getMainLooper()) + + @Volatile private var subscriber: Pair? = null + private var sessions: MediaSessionManager? = null + @Volatile private var controller: MediaController? = null + private var sentInfo: ByteArray? = null + private var sentArt: Bitmap? = null + + private val onSessions = MediaSessionManager.OnActiveSessionsChangedListener { follow(it.orEmpty()) } + private val onChange = object : MediaController.Callback() { + override fun onMetadataChanged(metadata: MediaMetadata?) = push() + override fun onPlaybackStateChanged(state: PlaybackState?) = sessions?.let { follow(active(it)) } ?: Unit + override fun onSessionDestroyed() = sessions?.let { follow(active(it)) } ?: Unit + } + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val reply = DataXConnection.mirror(message.info.channelId) + if (message.info.flags.isClosed) { + if (subscriber?.second == reply) subscriber = null + return + } + when (message.info.messageType) { + MT_SUBSCRIBE -> scope.launch { + subscriber = connection to reply + sentInfo = null + sentArt = null + watch() + push() + } + MT_INFO_REQUEST -> { + val (player, name) = player(controller) + connection.send(reply, Wire.fieldVarint(1, player) + Wire.fieldBytes(2, name.encodeToByteArray()), messageType = MT_INFO_RESPONSE) + } + } + } + + /** From the first subscription on, follows the playing session; without notification access there is none. */ + private fun watch() { + if (sessions != null) return + val manager = context.getSystemService(MediaSessionManager::class.java) ?: return + runCatching { + manager.addOnActiveSessionsChangedListener(onSessions, listener(), main) + sessions = manager + follow(active(manager)) + } + } + + private fun listener() = ComponentName(context, MirrorListener::class.java) + + private fun active(manager: MediaSessionManager): List = + runCatching { manager.getActiveSessions(listener()) }.getOrDefault(emptyList()) + + // The playing session, or else the first one Android lists. + private fun follow(controllers: List) { + val next = controllers.firstOrNull { it.playbackState?.state == PlaybackState.STATE_PLAYING } ?: controllers.firstOrNull() + if (next?.sessionToken != controller?.sessionToken) { + controller?.unregisterCallback(onChange) + controller = next + next?.registerCallback(onChange, main) + } + push() + } + + // Sends what changed since the last push: the player, the cover, or both. + private fun push() { + val session = controller + scope.launch(Dispatchers.Default) { + sending.withLock { + val (mux, reply) = subscriber ?: return@withLock + val (player, name) = player(session) + val info = Wire.fieldVarint(1, player) + Wire.fieldVarint(2, CONTENT_UNKNOWN) + Wire.fieldVarint(3, LIKED_UNKNOWN) + + Wire.fieldBytes(4, name.encodeToByteArray()) + val art = session?.metadata?.let { art(it) } + val newInfo = !info.contentEquals(sentInfo) + val newArt = !(art === sentArt || (art != null && sentArt?.let { art.sameAs(it) } == true)) + runCatching { + if (newInfo) mux.send(reply, info, messageType = MT_PLAYER_INFO) + if (newArt) sendArt(mux, reply, art) + sentInfo = info + sentArt = art + }.onFailure { if (subscriber?.first === mux) subscriber = null } + } + } + } + + private suspend fun sendArt(mux: DataXConnection, reply: Int, art: Bitmap?) { + if (art == null) return mux.send(reply, ByteArray(0), messageType = MT_ART_LAST) + val jpeg = ByteArrayOutputStream().also { fit(art).compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, it) }.toByteArray() + for (at in jpeg.indices step CHUNK) { + val end = minOf(at + CHUNK, jpeg.size) + mux.send(reply, Wire.fieldBytes(1, jpeg.copyOfRange(at, end)), messageType = if (end == jpeg.size) MT_ART_LAST else MT_ART_PIECE) + } + } + + // At most 300 x 300, as the Meta app sends it. + private fun fit(art: Bitmap): Bitmap { + if (art.width <= MAX_SIDE && art.height <= MAX_SIDE) return art + val scale = MAX_SIDE.toFloat() / maxOf(art.width, art.height) + return Bitmap.createScaledBitmap(art, (art.width * scale).toInt().coerceAtLeast(1), (art.height * scale).toInt().coerceAtLeast(1), true) + } + + private fun art(metadata: MediaMetadata): Bitmap? { + for (key in listOf(MediaMetadata.METADATA_KEY_ALBUM_ART, MediaMetadata.METADATA_KEY_ART, MediaMetadata.METADATA_KEY_DISPLAY_ICON)) { + metadata.getBitmap(key)?.let { return it } + } + for (key in listOf(MediaMetadata.METADATA_KEY_ALBUM_ART_URI, MediaMetadata.METADATA_KEY_ART_URI, MediaMetadata.METADATA_KEY_DISPLAY_ICON_URI)) { + val uri = metadata.getString(key)?.takeIf { it.isNotBlank() } ?: continue + runCatching { context.contentResolver.openInputStream(Uri.parse(uri))?.use { BitmapFactory.decodeStream(it) } }.getOrNull()?.let { return it } + } + return null + } + + // MediaPlayer: the partner apps the Display knows by name, any other player as 0 with its package. + private fun player(session: MediaController?): Pair { + if (context.packageName !in NotificationManagerCompat.getEnabledListenerPackages(context)) return NO_ACCESS to "" + val pkg = session?.packageName ?: return 0L to "" + return (PLAYERS[pkg] ?: 0L) to pkg + } + + companion object { + const val SVC = 9012 + private const val MT_INFO_REQUEST = 3 + private const val MT_INFO_RESPONSE = 4 + private const val MT_SUBSCRIBE = 5 + private const val MT_PLAYER_INFO = 100 + private const val MT_ART_PIECE = 101 + private const val MT_ART_LAST = 102 + private const val CONTENT_UNKNOWN = 0L + private const val LIKED_UNKNOWN = 0L + private const val NO_ACCESS = 7L + private const val CHUNK = 1024 + private const val MAX_SIDE = 300 + private const val JPEG_QUALITY = 70 + private val PLAYERS = mapOf( + "com.spotify.music" to 1L, "com.apple.android.music" to 2L, "com.amazon.mp3" to 3L, + "com.audible.application" to 4L, "com.clearchannel.iheartradio.controller" to 5L, "com.calm.android" to 6L, + ) + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 4de09a84..90ee8c4e 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -151,6 +151,7 @@ object ConnectionModule { messages: com.macedonga.visor.connection.PhoneMessages, sharing: com.macedonga.visor.connection.PhoneSharing, calls: com.macedonga.visor.connection.PhoneCalls, + media: com.macedonga.visor.connection.PhoneMedia, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -172,6 +173,7 @@ object ConnectionModule { com.macedonga.visor.connection.PhoneMessages.SVC to messages, com.macedonga.visor.connection.PhoneSharing.SVC to sharing, com.macedonga.visor.connection.PhoneCalls.SVC to calls, + com.macedonga.visor.connection.PhoneMedia.SVC to media, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), From 668ac28ee3fa25fab5327764d35670b1ec7c07c4 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 09:34:48 +0200 Subject: [PATCH 45/90] Display: Meta AI answers as cards. When the glasses are a Display, an answer in the glasses' own voice also carries a simple card: ev60's fulfillmentResult gets onDeviceUxMessage (field 27) with one base64, Thrift-compact AssistantTemplate (type 0, a SimpleTemplate with the question as title and the answer as subtitle, spokenResponse, and hideAfterMs, which the Display needs), and the VARM sets expectedUxTemplate (field 24). Other glasses get the same bytes as before. Not tried on the Display yet. --- .../macedonga/visor/di/ConnectionModule.kt | 1 + .../visor/core/assistant/Assistant.kt | 10 ++++-- .../macedonga/visor/core/assistant/Oxide.kt | 35 ++++++++++++++++--- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 90ee8c4e..9ca75ec0 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -250,6 +250,7 @@ object ConnectionModule { hearing, { (glassesConfig.state.value.values[com.macedonga.visor.language.PROCESSING_TIMEOUT] as? Number)?.toLong() }, phone, + { com.macedonga.visor.settings.hasDisplay(connection.state.value.info?.build) }, ) @Provides diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt index c8c7fd13..ec7844f5 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt @@ -107,6 +107,8 @@ class Assistant( private val processingTimeoutS: () -> Long? = { null }, /** The phone's player, for pausing, resuming and skipping by voice. */ private val phone: Phone? = null, + /** The connected glasses have a display: each answer then comes with a card for it. */ + private val display: () -> Boolean = { false }, ) : Responder { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) @@ -958,9 +960,13 @@ class Assistant( preamble(l, interaction, transcript, text) _phase.value = AssistantPhase.SPEAKING val voiceId = glassesVoice() - val answer = netEvent(EV_VOICE_ACTION_RESPONSE, varm(interaction, transcript, text, onGlasses = true, voiceId = voiceId)) + val card = if (display()) answerCard(transcript.trim().take(80).ifEmpty { "Meta AI" }, text) else null + val answer = netEvent(EV_VOICE_ACTION_RESPONSE, varm(interaction, transcript, text, onGlasses = true, voiceId = voiceId, card = card != null)) // An answer that asks something has the glasses listen for the reply, without the wake word. - val complete = netEvent(EV_MWA_VOICE_INTERACTION_COMPLETE, interactionComplete(interaction, text, expectsAnswer = ASKS.containsMatchIn(text))) + val complete = netEvent( + EV_MWA_VOICE_INTERACTION_COMPLETE, + interactionComplete(interaction, text, expectsAnswer = ASKS.containsMatchIn(text), card = card), + ) // Without simplified forwarding the VARM is acted on as it lands, so the text must be there before it. for (event in if (l.simplified) listOf(answer, complete) else listOf(complete, answer)) { l.mux.send(m0, event, messageType = MT_NET_EVENT) diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt index 0b91e88f..cb7fa1d0 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt @@ -129,7 +129,14 @@ private fun inflate(data: ByteArray): ByteArray { * request and interaction ids were replaced in the file with neutral ones of the same shape. * Splicing keeps every other byte; re-encoding it would change integer widths. */ -internal fun varm(interactionId: String, transcript: String, text: String, onGlasses: Boolean = false, voiceId: String? = null): ByteArray { +internal fun varm( + interactionId: String, + transcript: String, + text: String, + onGlasses: Boolean = false, + voiceId: String? = null, + card: Boolean = false, +): ByteArray { val raw = checkNotNull(Assistant::class.java.getResourceAsStream("varm_template.bin")) { "varm_template.bin is missing" } .use { it.readBytes() } val old = Thrift.loads(raw) @@ -144,14 +151,32 @@ internal fun varm(interactionId: String, transcript: String, text: String, onGla val spliced = Thrift.replaceStrings(raw, mapping) if (!onGlasses) return spliced // No audio follows: streamingBackTts (6) and isUsingPartialResponseStreaming (14) go false; ev60 then carries the text. - return listOf(6, 14).fold(spliced) { b, field -> Thrift.setBool(b, field, false) } + val quiet = listOf(6, 14).fold(spliced) { b, field -> Thrift.setBool(b, field, false) } + // expectedUxTemplate (24) tells the glasses a card is coming in ev60. + return if (card) Thrift.setBool(quiet, 24, true) else quiet +} + +/** + * A simple card for a Display, as ev60's onDeviceUxMessage carries it: base64 of a compact AssistantTemplate + * (type 0, a SimpleTemplate inside). The Display throws on a template without hideAfterMs. + */ +internal fun answerCard(title: String, text: String): String { + val simple = ThriftWriter().str(1, title).str(2, text).done() + val hideAfterMs = (8_000L + 60L * text.length).coerceAtMost(30_000L) + val template = ThriftWriter().i32(1, 0).str(2, simple).str(9, text).i64(10, hideAfterMs).done() + return java.util.Base64.getEncoder().encodeToString(template) } /** * ev60 MwaVoiceInteractionComplete with a fulfillmentResult the glasses speak themselves: ttsMessage, fulfillment * type 1 (server), requiresAnswer when they should listen for a reply, and nothing streaming. Status 1000 is success. + * A [card] goes in onDeviceUxMessage (field 27) for the Display to draw. */ -internal fun interactionComplete(interactionId: String, text: String, expectsAnswer: Boolean = false): ByteArray { - val result = ThriftWriter().str(1, text).i32(2, 1).bool(3, expectsAnswer).bool(4, false).bool(7, false).bool(8, false).done() - return ThriftWriter().str(1, interactionId).i32(2, 1000).str(3, "").struct(4, result).done() +internal fun interactionComplete(interactionId: String, text: String, expectsAnswer: Boolean = false, card: String? = null): ByteArray { + var result = ThriftWriter().str(1, text).i32(2, 1).bool(3, expectsAnswer).bool(4, false).bool(7, false).bool(8, false) + if (card != null) { + val ux = ThriftWriter().listOfBinaries(1, listOf(card.toByteArray(Charsets.UTF_8))).str(2, interactionId).done() + result = result.struct(27, ux) + } + return ThriftWriter().str(1, interactionId).i32(2, 1000).str(3, "").struct(4, result.done()).done() } From b85c2bdcb878bd91d3915a9ca763da0fbadd84f2 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 09:35:53 +0200 Subject: [PATCH 46/90] Neural Band: the buzz waits for the band's answer. Vibrate sends HapticsReq VibrateReq (RpcRequest field 17) and waits up to 5 s for the HapticsResp (RpcResponse field 18), matched by its sequence number; a refusal or no answer shows under the row. Stop is built as StopReq. The band is still taken from the Display for the request. Not tried on the band yet. --- .../com/macedonga/visor/band/BandLink.kt | 40 ++++++++++++++++++- .../com/macedonga/visor/settings/BandPart.kt | 15 ++++++- .../main/res/values-it/strings_haptics.xml | 4 ++ app/src/main/res/values/strings_haptics.xml | 4 ++ .../com/macedonga/visor/core/protocol/Band.kt | 21 ++++++++-- 5 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 app/src/main/res/values-it/strings_haptics.xml create mode 100644 app/src/main/res/values/strings_haptics.xml diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt index f3f9de3a..099ce92d 100644 --- a/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt @@ -19,14 +19,17 @@ import com.macedonga.visor.core.session.SessionCommand import com.macedonga.visor.core.transport.BluetoothLeTransport import com.macedonga.visor.log.EventLog import dagger.hilt.android.qualifiers.ApplicationContext +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger import javax.inject.Inject import javax.inject.Singleton +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.delay +import kotlinx.coroutines.withTimeoutOrNull import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow @@ -71,6 +74,8 @@ class BandLink @Inject constructor( private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) private val gate = Mutex() private val seq = AtomicInteger(1) + // Answers still awaited, by request seq; a haptics request waits for its own. + private val replies = ConcurrentHashMap>() private val leased = MutableStateFlow(false) private val held = MutableStateFlow(true) private val lesson = MutableStateFlow(false) @@ -130,8 +135,37 @@ class BandLink @Inject constructor( log.add("Band: the lesson gives the band back") } - /** A short buzz on the band, taken from the Display for it. */ - fun buzz() = borrow { seq -> listOf(Band.vibrate(seq.getAndIncrement(), BUZZ_DURATION, BUZZ_HZ, BUZZ_AMPLITUDE)) } + /** A buzz on the band; true when the band answers OK. */ + suspend fun vibrate(duration: Long = BUZZ_DURATION, frequency: Float = BUZZ_HZ, amplitude: Int = BUZZ_AMPLITUDE): Boolean = + haptic { seq -> Band.vibrate(seq, duration, frequency, amplitude) } + + /** Stops a buzz; true when the band answers OK. */ + suspend fun stop(): Boolean = haptic { seq -> Band.stopHaptics(seq) } + + /** One haptics request, taking the band from the Display for it; true when the band answers OK. */ + private suspend fun haptic(request: (Int) -> ByteArray): Boolean { + val borrowed = _state.value.link != LinkState.READY + if (borrowed) borrowing.value = true + try { + if (borrowed && withTimeoutOrNull(BORROW_MS) { _state.first { it.link == LinkState.READY } } == null) { + log.add("Band: not reached for a buzz in ${BORROW_MS / 1000} s") + return false + } + val n = seq.getAndIncrement() + val answer = CompletableDeferred() + replies[n] = answer + val code = if (send { listOf(request(n)) }) withTimeoutOrNull(REPLY_MS) { answer.await() } else null + replies.remove(n) + if (code != null && code != Band.OK) log.add("Band: the buzz was refused, code $code") + if (code == null) log.add("Band: no answer to the buzz") + return code == Band.OK + } finally { + if (borrowed) scope.launch { + delay(ERRAND_MS) + borrowing.value = false + } + } + } /** Take the band for one errand, then give it back; at once when the phone already has it. */ private fun borrow(requests: (AtomicInteger) -> List) { @@ -244,6 +278,7 @@ class BandLink @Inject constructor( val battery = runCatching { when (type) { Band.RESPONSE -> Band.response(event.payload).also { r -> + replies.remove(r.seq)?.complete(r.haptics ?: r.code) if (r.leftHanded != null) _state.update { it.copy(leftHanded = r.leftHanded) } if (r.protocol != null) { _state.update { it.copy(protocol = r.protocol) } @@ -273,6 +308,7 @@ class BandLink @Inject constructor( const val INFO_WAIT_MS = 1_500L const val BORROW_MS = 20_000L const val ERRAND_MS = 1_500L + const val REPLY_MS = 5_000L // Nothing in the Meta app sends a buzz, so these are a guess at milliseconds, hertz and a 0-255 strength. const val BUZZ_DURATION = 200L const val BUZZ_HZ = 170f diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt index 340ff753..9d83e69b 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt @@ -2,11 +2,16 @@ package com.macedonga.visor.settings import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.setValue import androidx.compose.ui.res.stringResource import androidx.lifecycle.compose.collectAsStateWithLifecycle import com.macedonga.visor.R import com.macedonga.visor.band.BandSetup import com.macedonga.visor.core.session.LinkState +import kotlinx.coroutines.launch /** The Neural Band that goes with this Display: set it up, or its battery and the wrist it is on. */ @Composable @@ -22,6 +27,8 @@ internal fun BandPart(model: SettingsViewModel) { val setup by model.bandSetup.status.collectAsStateWithLifecycle() val running = setup is BandSetup.Status.Running val problem = (setup as? BandSetup.Status.Failed)?.let { failWords(it) } + val scope = rememberCoroutineScope() + var refused by remember { mutableStateOf(false) } @Composable fun SetUpRow(title: Int) = ValueRow( stringResource(title), @@ -52,7 +59,13 @@ internal fun BandPart(model: SettingsViewModel) { onPick = { model.band.setLeftHanded(it) }, ) }, - { ValueRow(stringResource(R.string.band_buzz), null, onClick = { model.band.buzz() }) }, + { + ValueRow( + stringResource(R.string.band_buzz), null, + onClick = { scope.launch { refused = !model.band.vibrate() } }, + problem = if (refused) stringResource(R.string.band_buzz_refused) else null, + ) + }, { SetUpRow(R.string.band_set_up_again) }, ) } diff --git a/app/src/main/res/values-it/strings_haptics.xml b/app/src/main/res/values-it/strings_haptics.xml new file mode 100644 index 00000000..a5fa09db --- /dev/null +++ b/app/src/main/res/values-it/strings_haptics.xml @@ -0,0 +1,4 @@ + + + Il Neural Band non ha vibrato + diff --git a/app/src/main/res/values/strings_haptics.xml b/app/src/main/res/values/strings_haptics.xml new file mode 100644 index 00000000..76e81ec5 --- /dev/null +++ b/app/src/main/res/values/strings_haptics.xml @@ -0,0 +1,4 @@ + + + The band did not buzz + diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt index 2731848f..3aab2056 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt @@ -9,13 +9,23 @@ object Band { const val RESPONSE = 21 const val STREAM_UPDATE = 22 + /** The answer's code when the band did what it was asked. */ + const val OK = 0 + /** The PSM the band's GATT names; the glasses' is 128. */ const val PSM = 129 class Battery(val percent: Int, val charging: Boolean) - /** RpcResponse {1: seq, 2: code, 3: BatteryInfoResp, 4: DeviceInfoResp, 6: ConfigResp, ...}, as far as Visor reads it. */ - class Response(val seq: Int, val code: Int, val battery: Battery?, val leftHanded: Boolean?, val protocol: Int? = null) + /** RpcResponse {1: seq, 2: code, 3: BatteryInfoResp, 4: DeviceInfoResp, 6: ConfigResp, 18: HapticsResp, ...}, as far as Visor reads it. */ + class Response( + val seq: Int, + val code: Int, + val battery: Battery?, + val leftHanded: Boolean?, + val protocol: Int? = null, + val haptics: Int? = null, + ) /** * A gesture, `EmgImu$GestureEvent`, sent as stream type 13 or a granular type from 60 to 200: {1 seq, 3 finger, 4 action, @@ -51,6 +61,9 @@ object Band { fun vibrate(seq: Int, duration: Long, frequency: Float, amplitude: Int): ByteArray = request(seq, 17, Wire.fieldBytes(1, Wire.fieldVarint(1, duration) + Wire.fieldFloat(2, frequency) + Wire.fieldVarint(3, amplitude.toLong()))) + /** RpcRequest {1: seq, 17: HapticsReq {3: StopReq {}}}. */ + fun stopHaptics(seq: Int): ByteArray = request(seq, 17, Wire.fieldBytes(3, ByteArray(0))) + fun isGesture(type: Int): Boolean = type == 13 || type in 60..200 fun gesture(type: Int, payload: ByteArray): Gesture { @@ -67,7 +80,9 @@ object Band { val battery = if (f.has(3)) battery(Wire.parse(f.bytes(3)).bytes(1)) else null val left = if (f.has(6)) Wire.parse(f.bytes(6)).let { c -> if (c.has(10)) c.varint(10) != 0L else null } else null val protocol = if (f.has(4)) Wire.parse(f.bytes(4)).let { d -> if (d.has(15)) d.varint(15).toInt() else null } else null - return Response(f.varint(1).toInt(), f.varint(2).toInt(), battery, left, protocol) + // HapticsResp {1: code}, in field 18 of the response. + val haptics = if (f.has(18)) Wire.parse(f.bytes(18)).let { h -> if (h.has(1)) h.varint(1).toInt() else null } else null + return Response(f.varint(1).toInt(), f.varint(2).toInt(), battery, left, protocol, haptics) } /** RpcStreamUpdate field 2, BatteryUpdate {1: BatteryData}. */ From 9f63afd283024177bf8ef35f9cc40f522aad51c7 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 09:39:39 +0200 Subject: [PATCH 47/90] Display: share the captions transcript. The captions screen gets a Share action that sends the turns of the current session and the one before as plain text, one line per turn with the time it arrived, a translation under what was heard. The turns stay in memory only; a new session keeps the last one's turns for sharing. --- .../macedonga/visor/display/CaptionsScreen.kt | 25 +++++++++++++++++++ .../main/res/values-it/strings_transcript.xml | 4 +++ .../main/res/values/strings_transcript.xml | 4 +++ .../visor/core/connection/Captions.kt | 19 ++++++++------ 4 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 app/src/main/res/values-it/strings_transcript.xml create mode 100644 app/src/main/res/values/strings_transcript.xml diff --git a/app/src/main/kotlin/com/macedonga/visor/display/CaptionsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/display/CaptionsScreen.kt index c2397398..1dbabaa2 100644 --- a/app/src/main/kotlin/com/macedonga/visor/display/CaptionsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/display/CaptionsScreen.kt @@ -1,6 +1,7 @@ package com.macedonga.visor.display import android.content.Context +import android.content.Intent import androidx.compose.foundation.background import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box @@ -27,6 +28,7 @@ import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Brush import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.stringResource import androidx.compose.ui.unit.dp import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel @@ -35,11 +37,13 @@ import androidx.lifecycle.viewModelScope import androidx.lifecycle.compose.collectAsStateWithLifecycle import com.macedonga.visor.R import com.macedonga.visor.core.connection.Captions +import com.macedonga.visor.core.connection.CaptionLine import com.macedonga.visor.core.connection.CaptionsMode import com.macedonga.visor.core.connection.CaptionsPhase import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.design.Visor import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.gallery.RoundIcon import com.macedonga.visor.settings.ChoiceSheet import com.macedonga.visor.settings.PillsRow import com.macedonga.visor.settings.Rows @@ -48,9 +52,13 @@ import com.macedonga.visor.settings.SettingsPage import com.macedonga.visor.settings.Spinner import com.macedonga.visor.settings.ValueRow import com.macedonga.visor.settings.row +import com.macedonga.visor.ui.VisorIcons import com.macedonga.visor.ui.rememberHaptics import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.qualifiers.ApplicationContext +import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter import java.util.Locale import javax.inject.Inject import javax.inject.Singleton @@ -123,7 +131,9 @@ fun CaptionsScreen(model: CaptionsModel = hiltViewModel()) { LaunchedEffect(feature, ready) { if (ready) model.packs.look(feature) } var picking by remember { mutableStateOf(null) } val haptics = rememberHaptics() + val context = LocalContext.current val list = rememberLazyListState() + val transcript = state.earlier + state.lines val last = state.lines.lastOrNull() // Follow the newest turn, unless the transcript was scrolled back to read. LaunchedEffect(state.lines.size, last?.text) { @@ -134,6 +144,9 @@ fun CaptionsScreen(model: CaptionsModel = hiltViewModel()) { SettingsPage( stringResource(R.string.captions_title), list = list, + actions = { + if (transcript.isNotEmpty()) RoundIcon(VisorIcons.Share, stringResource(R.string.transcript_share), { share(context, transcript) }) + }, sheet = { CaptionsBar( ready = ready, @@ -189,6 +202,18 @@ fun CaptionsScreen(model: CaptionsModel = hiltViewModel()) { private enum class Side { SELF, OTHER } +private val CLOCK = DateTimeFormatter.ofPattern("HH:mm") + +/** Plain text, one turn per line as `HH:mm said`; a translation goes on the line under it. */ +private fun share(context: Context, lines: List) { + val text = lines.joinToString("\n") { line -> + val time = CLOCK.format(Instant.ofEpochMilli(line.time).atZone(ZoneId.systemDefault())) + if (line.source == null) "$time ${line.text}" else "$time ${line.source}\n${line.text}" + } + val send = Intent(Intent.ACTION_SEND).setType("text/plain").putExtra(Intent.EXTRA_TEXT, text) + context.startActivity(Intent.createChooser(send, null)) +} + /** One turn: the newest one bright while the glasses listen, what was said under a translation. */ @Composable private fun Turn(text: String, source: String?, current: Boolean) { diff --git a/app/src/main/res/values-it/strings_transcript.xml b/app/src/main/res/values-it/strings_transcript.xml new file mode 100644 index 00000000..53428fbe --- /dev/null +++ b/app/src/main/res/values-it/strings_transcript.xml @@ -0,0 +1,4 @@ + + + Condividi + diff --git a/app/src/main/res/values/strings_transcript.xml b/app/src/main/res/values/strings_transcript.xml new file mode 100644 index 00000000..96107217 --- /dev/null +++ b/app/src/main/res/values/strings_transcript.xml @@ -0,0 +1,4 @@ + + + Share + diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Captions.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Captions.kt index e11c2f97..7b87a409 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Captions.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Captions.kt @@ -20,15 +20,16 @@ enum class CaptionsPhase { IDLE, STARTING, LIVE, STOPPING } enum class CaptionsMode { CAPTIONS, TRANSLATION } -/** One speech turn as the glasses last sent it: [text] to read, [source] what was said when it is a translation. */ -data class CaptionLine(val id: String, val text: String, val source: String?, val done: Boolean) +/** One speech turn as the glasses last sent it: [text] to read, [source] what was said when it is a translation. [time] is when it arrived. */ +data class CaptionLine(val id: String, val text: String, val source: String?, val done: Boolean, val time: Long) -/** [refusal] is the app manager's code when the glasses would not start it. */ +/** [refusal] is the app manager's code when the glasses would not start it. [earlier] is the turns of the session before. */ data class CaptionsState( val phase: CaptionsPhase = CaptionsPhase.IDLE, val mode: CaptionsMode = CaptionsMode.CAPTIONS, val refusal: Long? = null, val lines: List = emptyList(), + val earlier: List = emptyList(), ) /** @@ -64,7 +65,7 @@ class Captions(private val connection: GlassesConnection) { if (connection.state.value.link != LinkState.READY) return val data = intent("action.lst_lc.self_lang", self) + intent("action.lst_lc.other_lang", other) + intent("action.lst_lc.request_id", UUID.randomUUID().toString()) + intent("action.lst_lc.action", action(mode, "start")) - _state.value = CaptionsState(CaptionsPhase.STARTING, mode) + _state.value = _state.value.restart(CaptionsPhase.STARTING, mode) connection.submit { it.sendOnService(SVC_COMPANION, MT_START_APP, Wire.fieldVarint(1, APP_TRANSLATION) + data) } settleAfter { if (it.phase == CaptionsPhase.STARTING) it.copy(phase = CaptionsPhase.LIVE) else it } } @@ -98,19 +99,23 @@ class Captions(private val connection: GlassesConnection) { MT_TWO_WAY_TRANSLATION -> turn(fields, _state.value.mode)?.let { line -> _state.update { now -> val at = now.lines.indexOfLast { it.id == line.id } - val lines = if (at >= 0) now.lines.toMutableList().also { it[at] = line } else (now.lines + line).takeLast(MAX_LINES) + val lines = if (at >= 0) now.lines.toMutableList().also { it[at] = line.copy(time = now.lines[at].time) } else (now.lines + line).takeLast(MAX_LINES) now.copy(phase = if (now.phase == CaptionsPhase.IDLE) CaptionsPhase.LIVE else now.phase, lines = lines) } } // Also how a session the wearer started on the glasses shows here. Its category is 1 for captions as well. MT_INVITE_TO_JOIN -> _state.update { - if (it.phase == CaptionsPhase.IDLE) CaptionsState(CaptionsPhase.LIVE) else it.copy(phase = CaptionsPhase.LIVE) + if (it.phase == CaptionsPhase.IDLE) it.restart(CaptionsPhase.LIVE) else it.copy(phase = CaptionsPhase.LIVE) } MT_SESSION_STATUS -> if (fields.varint(1) == STATUS_IDLE) ended() MT_END_FROM_FRAME, MT_STOPPING_FROM_FRAME -> ended() } } + // The turns so far become the earlier session's, so a restart keeps the transcript. + private fun CaptionsState.restart(phase: CaptionsPhase, mode: CaptionsMode = CaptionsMode.CAPTIONS) = + CaptionsState(phase, mode, earlier = lines.ifEmpty { earlier }) + // A start makes the glasses end the session before it first; that end is not this one's. private fun ended() { if (_state.value.phase == CaptionsPhase.STARTING) return @@ -132,7 +137,7 @@ class Captions(private val connection: GlassesConnection) { if (text.isEmpty() || text == PENDING && heard.isEmpty()) return null val id = s(1).ifEmpty { "t${f.varint(6)}" } val done = f.varint(13) != 0L || s(11).isEmpty() && s(9).isEmpty() && s(5).isEmpty() - return CaptionLine(id, text, heard.takeIf { mode == CaptionsMode.TRANSLATION && it.isNotEmpty() && it != text }, done) + return CaptionLine(id, text, heard.takeIf { mode == CaptionsMode.TRANSLATION && it.isNotEmpty() && it != text }, done, System.currentTimeMillis()) } private fun settleAfter(change: (CaptionsState) -> CaptionsState) { From bf30bb0c4a87367665990bd574807920aa72f98d Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 09:40:09 +0200 Subject: [PATCH 48/90] Display: type to Meta AI from the phone. With a Display connected, a question typed on the Meta AI page runs as a turn on the glasses: ev337 StartRemoteInteraction binds a new interaction, ev52 gives its words, and the answer goes as for a spoken turn, so the Display shows it with its card. Without a Display, or with no assistant channel open, the app answers it as before. Not tried on the Display yet. --- .../com/macedonga/visor/metaai/MetaAiModel.kt | 4 +- .../visor/core/assistant/Assistant.kt | 37 ++++++++++++++++++- .../macedonga/visor/core/assistant/Oxide.kt | 1 + 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/metaai/MetaAiModel.kt b/app/src/main/kotlin/com/macedonga/visor/metaai/MetaAiModel.kt index 3aeab54d..3b38e2fe 100644 --- a/app/src/main/kotlin/com/macedonga/visor/metaai/MetaAiModel.kt +++ b/app/src/main/kotlin/com/macedonga/visor/metaai/MetaAiModel.kt @@ -59,7 +59,9 @@ class MetaAiModel @Inject constructor( val forgotten: StateFlow = _forgotten.asStateFlow() private var forgetting: Job? = null - fun ask(question: String, conversation: String?) = assistant.askFromPhone(question, conversation) + /** On a Display the question runs as a turn on the glasses; otherwise, or when they cannot take it, it is answered here. */ + fun ask(question: String, conversation: String?): String = + assistant.askOnGlasses(question, conversation) ?: assistant.askFromPhone(question, conversation) fun askAgain(id: String) = assistant.askAgain(id) diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt index ec7844f5..a8f91b23 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt @@ -764,7 +764,8 @@ class Assistant( private suspend fun answer(l: Link, interaction: String, transcript: String, visual: Visual) { val started = System.currentTimeMillis() - record(TurnRecord(interaction, started, transcript)) + // A typed turn is in the diary already, with its source and conversation. + if (diary.turn(interaction) == null) record(TurnRecord(interaction, started, transcript)) _phase.value = AssistantPhase.THINKING var problem = "" // During the Live every question comes with the picture: the wearer is showing it. @@ -935,11 +936,15 @@ class Assistant( return packets.sumOf { it.durationUs } / 1000 } + /** ev52 FinalTranscription: the words of a turn. */ + private fun finalTranscription(interaction: String, words: String): ByteArray = + ThriftWriter().str(1, words).i32(2, 0).str(4, "").bool(7, false).str(8, interaction).done() + /** ev71, ev52 and ev342, which open every answer. ev52 and ev342 go at a tagged type 0, as the app sends them. */ private suspend fun preamble(l: Link, interaction: String, transcript: String, text: String) { val m0 = checkNotNull(l.m0) val e71 = ThriftWriter().str(1, interaction).done() - val e52 = ThriftWriter().str(1, transcript.ifEmpty { text }).i32(2, 0).str(4, "").bool(7, false).str(8, interaction).done() + val e52 = finalTranscription(interaction, transcript.ifEmpty { text }) val e342 = ThriftWriter().str(1, interaction).bool(2, false).str(3, REPLY_INTENT).bool(4, false).done() for ((type, event, body) in listOf( Triple(MT_NET_EVENT, EV_START_OF_USER_SPEECH, e71), @@ -1062,6 +1067,34 @@ class Assistant( return id } + /** + * A question typed in the app, run as a turn on the glasses when they are a Display: 337 starts the turn, 52 gives + * its words, and the answer goes as for any turn. Null when the glasses cannot take it, so the app answers it. + */ + fun askOnGlasses(question: String, conversation: String?): String? { + val text = question.trim() + val l = link?.takeIf { display() && it.m0 != null && it.s0 != null } ?: return null + if (text.isEmpty()) return null + val id = UUID.randomUUID().toString() + // Taken before the glasses could open a microphone for the turn, so the phone does not answer its own recognition. + l.taken.add(id) + record(TurnRecord(id, System.currentTimeMillis(), text, source = TurnSource.PHONE, conversation = conversation.orEmpty())) + scope.launch { + try { + val m0 = checkNotNull(l.m0) + l.mux.send(m0, netEvent(EV_START_REMOTE_INTERACTION, ThriftWriter().str(1, id).done()), messageType = MT_NET_EVENT) + delay(20) + l.mux.send(m0, netEvent(EV_FINAL_TRANSCRIPTION, finalTranscription(id, text)), messageType = 0) + } catch (error: Exception) { + say("The typed question did not reach the glasses: ${error.message}") + update(id) { it.copy(problem = error.message.orEmpty()) } + return@launch + } + answer(l, id, text, Visual(null, "", "", false, "")) + } + return id + } + private fun liveAiNote(): String = if (liveAiDetailed()) LIVE_AI_NOTE_DETAILED else LIVE_AI_NOTE /** [id]'s question again, with the model and settings as they are now; the new answer replaces the old one. */ diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt index cb7fa1d0..153694e6 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt @@ -35,6 +35,7 @@ internal const val MT_ENVELOPE_ACK = 9001 // table beside the names; all 206 are in events.tsv). Several are not what lensbreak called them, // but every send below is byte for byte the one the glasses accepted from it. internal const val EV_FINAL_TRANSCRIPTION = 52 +internal const val EV_START_REMOTE_INTERACTION = 337 internal const val EV_VOICE_ACTION_RESPONSE = 53 internal const val EV_FULFILLMENT_REQUEST = 57 internal const val EV_ACTION_FULFILLMENT_REQUEST = 55 From 98ec7d8cec656c226691e6fcfbeb22750fc3e6a3 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 09:45:14 +0200 Subject: [PATCH 49/90] Reconnect after Visor is updated. An update kills the glasses service, and glasses already in reach raise no new presence event, so Visor stayed off until it was opened. A MY_PACKAGE_REPLACED receiver now asks Android to observe the companion presence again and, when a pair is set up, starts the connection as the presence service does. --- app/src/main/AndroidManifest.xml | 9 ++++++ .../macedonga/visor/connection/AfterUpdate.kt | 31 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/AfterUpdate.kt diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 7939a0b4..292e5b06 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -161,6 +161,15 @@ android:exported="false" android:foregroundServiceType="connectedDevice" /> + + + + + + + Date: Fri, 23 Oct 2026 09:45:28 +0200 Subject: [PATCH 50/90] Display: answer the tunnel's HTTP requests. Besides IP packets, the Display sends single HTTP requests on svc41111 (type 8, an HttpRequest: url, method, headers, redirects). Visor makes them with the phone's own connection and answers on the same channel as the Display's reader of HttpResponse expects: the response (type 11: status, message, headers), the body in 1024-byte pieces (12) and an empty end (13); a request the phone cannot make gets the response's error field and the end. Each one shows on the Requests page. The reply types come from that reader; a request body streamed after the request (12, ended by 13) is a guess. Not tried on the Display yet. --- .../visor/connection/PhoneHttpRelay.kt | 149 ++++++++++++++++++ .../visor/connection/PhoneInternet.kt | 4 + 2 files changed, 153 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/PhoneHttpRelay.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneHttpRelay.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneHttpRelay.kt new file mode 100644 index 00000000..960b7218 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneHttpRelay.kt @@ -0,0 +1,149 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.log.EventLog +import java.io.ByteArrayOutputStream +import java.net.HttpURLConnection +import java.net.URL +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch + +/** + * svc41111, internet_proxy_remote, HTTP relay: the Display sends an HttpRequest (type 8) on a channel of its own, and the + * phone answers on that channel with an HttpResponse (type 11), the body in chunks (12), then an empty end (13). The reply + * types follow the Display's reader of HttpResponse (GreatwhiteSystemUIRelease, X/C0OQ.java). The request type 8 and a + * streamed request body (chunks until 13) are not in the research; they are guessed from that reader. + */ +@Singleton +class PhoneHttpRelay @Inject constructor( + private val requests: RequestLog, + private val log: Provider, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Internet: a request failed: $e") }) + // Requests whose body the Display streams after them, by channel. + private val streaming = ConcurrentHashMap() + + fun handle(message: DataXMessage, connection: DataXConnection) { + val channel = message.info.channelId + when (message.info.messageType) { + MT_REQUEST -> { + val request = runCatching { Request.decode(message.payload) }.getOrElse { + scope.launch { answer(channel, connection, error = true) } + return + } + if (request.streamed) streaming[channel] = Streamed(request, connection, ByteArrayOutputStream()) + else scope.launch { serve(channel, request, null, connection) } + } + MT_BODY -> streaming[channel]?.body?.write(message.payload) + MT_END -> streaming.remove(channel)?.let { scope.launch { serve(channel, it.request, it.body.toByteArray(), it.mux) } } + } + } + + fun forget(channel: Int) { + streaming.remove(channel) + } + + private suspend fun serve(channel: Int, request: Request, body: ByteArray?, mux: DataXConnection) { + val id = requests.start(request.url, SOURCE_TUNNEL, request.method) + var answered = false + try { + val url = URL(request.url) + require(url.protocol == "http" || url.protocol == "https") { "not an http link" } + val conn = (url.openConnection() as HttpURLConnection).apply { + requestMethod = request.method + instanceFollowRedirects = !request.disableRedirects + connectTimeout = CONNECT_MS + readTimeout = READ_MS + for ((name, value) in request.headers) setRequestProperty(name, value) + if (body != null) { + doOutput = true + outputStream.use { it.write(body) } + } + } + val code = conn.responseCode + mux.send(DataXConnection.mirror(channel), response(code, conn.responseMessage.orEmpty(), conn), messageType = MT_RESPONSE) + answered = true + var sent = 0L + (if (code >= 400) conn.errorStream else conn.inputStream)?.use { input -> + val buffer = ByteArray(CHUNK) + while (true) { + val n = input.read(buffer) + if (n < 0) break + mux.send(DataXConnection.mirror(channel), buffer.copyOf(n), messageType = MT_BODY) + sent += n + } + } + mux.send(DataXConnection.mirror(channel), ByteArray(0), messageType = MT_END) + requests.finish(id, "HTTP $code", failed = code >= 400, bytes = sent) + } catch (e: Exception) { + requests.finish(id, "failed: ${e.message}", failed = true) + answer(channel, mux, error = !answered) + } + } + + /** The reply that ends a request the phone could not serve: an error and the end, so the Display stops waiting. */ + private suspend fun answer(channel: Int, mux: DataXConnection, error: Boolean) { + val reply = DataXConnection.mirror(channel) + runCatching { + if (error) mux.send(reply, Wire.fieldVarint(4, ERROR_FAILED), messageType = MT_RESPONSE) + mux.send(reply, ByteArray(0), messageType = MT_END) + } + } + + private fun response(code: Int, message: String, conn: HttpURLConnection): ByteArray { + var out = Wire.fieldVarint(1, code.toLong()) + Wire.fieldBytes(2, message.toByteArray()) + for ((name, values) in conn.headerFields) { + if (name == null) continue + for (value in values) out += Wire.fieldBytes(3, Wire.fieldBytes(1, name.toByteArray()) + Wire.fieldBytes(2, value.toByteArray())) + } + return out + } + + private class Request( + val url: String, + val method: String, + val headers: List>, + val streamed: Boolean, + val disableRedirects: Boolean, + ) { + companion object { + // The request's uuid (field 1) is not needed: each request has its own channel. + fun decode(payload: ByteArray): Request { + val f = Wire.parse(payload) + return Request( + url = f.bytes(2).decodeToString(), + method = f.bytes(7).decodeToString().ifEmpty { "GET" }, + headers = f.repeatedBytes(5).map { h -> Wire.parse(h).let { it.bytes(1).decodeToString() to it.bytes(2).decodeToString() } }, + streamed = f.varint(6) != 0L, + disableRedirects = f.varint(8) != 0L, + ) + } + } + } + + private class Streamed(val request: Request, val mux: DataXConnection, val body: ByteArrayOutputStream) + + companion object { + private const val MT_REQUEST = 8 + private const val MT_RESPONSE = 11 + private const val MT_BODY = 12 + private const val MT_END = 13 + // HttpResponse.error for a request the phone could not make. The Display's error codes are not in the research. + private const val ERROR_FAILED = 1L + // The chunk size the data fetcher uses by default, which the Display already reads. + private const val CHUNK = 1024 + private const val CONNECT_MS = 10_000 + private const val READ_MS = 30_000 + // The request log's source for traffic of the tunnel rather than of the data fetcher. + private const val SOURCE_TUNNEL = -1 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt index ae38c9e5..3975c90a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt @@ -40,11 +40,13 @@ import kotlinx.coroutines.withTimeoutOrNull * each IP packet as one message (type 1), framed `1c fb`, its length, its ethertype and two spare bytes; replies go * back the same way on the same channel. The phone ends the connections itself: each TCP connection from the Display * becomes a socket of the phone's, each UDP flow a datagram socket, so the Display's traffic is the phone's own. + * Other types go to [PhoneHttpRelay]: a single HTTP request (type 8) answered on its own channel. */ @Singleton class PhoneInternet @Inject constructor( private val requests: RequestLog, private val log: Provider, + private val relay: PhoneHttpRelay, ) : Responder { // A connection that fails ends alone; nothing in here may take the app down. private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Internet: a connection failed: $e") }) @@ -55,6 +57,7 @@ class PhoneInternet @Inject constructor( val channel = info.channelId if (info.flags.isClosed && message.payload.isEmpty()) { sessions.remove(channel)?.close() + relay.forget(channel) return } when (info.messageType) { @@ -64,6 +67,7 @@ class PhoneInternet @Inject constructor( log.get().add("Internet: the Display's apps are using the phone's internet") } MT_FRAME -> sessions.getOrPut(channel) { Session(connection, DataXConnection.mirror(channel)) }.inbound(message.payload) + else -> relay.handle(message, connection) } } From 18a48c1a64e6085e6591464b5cecf849d2bd7e7e Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 09:47:58 +0200 Subject: [PATCH 51/90] Display: the glasses' own tools for Visor's model, and their device intents handed back. The tools the Display names in each question (ev57 field 54) are offered to the model with Visor's own descriptions and schemas for the known ones (volume, brightness, do not disturb, battery, music, apps, browser, maps, navigation, the camera); a call goes to the glasses as an MCP tools/call (ev311) and its result comes back on ev312. A volume or brightness intent the glasses understood but left to the server (ev57 field 55) is handed back to them with ev380; when they answer ev381 with success, the wearer hears it is done, otherwise the model answers as before. Only on a Display. Not tried on the Display yet. --- .../visor/core/assistant/Assistant.kt | 88 ++++++++++++++++- .../macedonga/visor/core/assistant/Brain.kt | 9 +- .../visor/core/assistant/GlassesTools.kt | 98 +++++++++++++++++++ .../macedonga/visor/core/assistant/Oxide.kt | 6 ++ .../macedonga/visor/core/protocol/Thrift.kt | 24 +++++ 5 files changed, 220 insertions(+), 5 deletions(-) create mode 100644 core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/GlassesTools.kt diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt index a8f91b23..9ab22952 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt @@ -2,6 +2,7 @@ package com.macedonga.visor.core.assistant import android.content.Context import com.macedonga.visor.core.assistant.mcp.ToolAction +import com.macedonga.visor.core.assistant.mcp.ToolResult import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.connection.GlassesLink import com.macedonga.visor.core.media.AirshipReceiver @@ -24,6 +25,7 @@ import java.time.LocalDate import java.util.Locale import java.util.UUID import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job @@ -47,6 +49,7 @@ import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import kotlinx.coroutines.withTimeout import kotlinx.coroutines.withTimeoutOrNull +import org.json.JSONObject enum class AssistantPhase { OFF, WAITING, REGISTERING, LISTENING, THINKING, SPEAKING } @@ -280,6 +283,10 @@ class Assistant( // The text the glasses read in each question's photo, from their ev65. private val ocr = ConcurrentHashMap>() fun ocrOf(interaction: String): CompletableDeferred = ocr.getOrPut(interaction) { CompletableDeferred() } + // The glasses' tool calls waiting for their ev312, by JSON-RPC id; and device intents waiting for their ev381. + val calls = ConcurrentHashMap>() + val handovers = ConcurrentHashMap>>() + val callIds = AtomicInteger(1) } /** A photo this side is waiting for, for one interaction. */ @@ -571,10 +578,26 @@ class Assistant( say("The glasses took the photo: ${describe(event.body)}") } CustomWakeWord.EV_RESULT -> wakeWord.onResult(runCatching { Thrift.loads(event.body) }.getOrDefault(emptyMap())) + EV_MCP_REQUEST_ACK -> Unit + EV_MCP_RESPONSE -> glassesReply(l, event.body) + EV_REMOTE_INTENT_RESPONSE -> handoverReply(l, event.body) else -> say("Assistant: ${eventName(event.id)} (${event.id}), ${event.body.size} bytes ${describe(event.body)}") } } + /** A tool call's answer, ev312 {1 jsonContent}: handed to the call waiting for its JSON-RPC id. */ + private fun glassesReply(l: Link, body: ByteArray) { + val json = runCatching { Thrift.loads(body)[1].asText() }.getOrDefault("") + val id = runCatching { JSONObject(json).opt("id")?.toString() }.getOrNull() ?: return say("Assistant: an unreadable tool answer") + l.calls[id]?.complete(json) + } + + /** ev381 {1 requestId, 2 success, 3 message}: the glasses' answer to a device intent they were asked to run. */ + private fun handoverReply(l: Link, body: ByteArray) { + val f = runCatching { Thrift.loads(body) }.getOrNull() ?: return + l.handovers[f[1].asText()]?.complete((f[2] == true) to f[3].asText()) + } + /** * ev59 MwaVoiceInteractionCloseRequest {1 interactionId, 2 audioLogData {3 transcription}}: the glasses end a turn. One * they never asked about they did themselves, as "play something": it goes in the diary with its words. @@ -689,6 +712,9 @@ class Assistant( say("Group conversation: $intent") return } + // The Display's own tools for this turn, as Meta's server names them (ev57 field 54). + val advertised = (m[54] as? List<*>).orEmpty().flatMap { server -> (server.asStruct()[2] as? List<*>).orEmpty().map { it.asText() } } + val glasses = if (display()) glassesToolbox(advertised) { full, input -> glassesCall(l, full, input) } else null when { // The glasses' own words for giving up. waiting != null && intent in CANCEL_INTENTS -> { @@ -700,9 +726,13 @@ class Assistant( sendReply(l, interaction, transcript, waiting.conversation, slot(slots, "SL:CONTENT_EXACT").ifEmpty { transcript }) } intent == "IN:REPLY_MESSAGE" -> scope.launch { replyRequested(l, interaction, transcript, slot(slots, "SL:CONTENT_EXACT")) } + // A device action the glasses understood: they run it, and the model answers only if they do not. + display() && intent in DEVICE_INTENTS -> scope.launch { + if (!handOver(l, interaction, transcript, body, intent)) answer(l, interaction, transcript, visual, glasses) + } // Left to the server by the glasses: the phone's player does them at once. phone != null && intent in MEDIA_CONTROLS -> scope.launch { mediaControl(l, interaction, transcript, phone, MEDIA_CONTROLS.getValue(intent)) } - else -> scope.launch { answer(l, interaction, transcript, visual) } + else -> scope.launch { answer(l, interaction, transcript, visual, glasses) } } } @@ -746,6 +776,50 @@ class Assistant( ) } + /** + * A tool call on the glasses: ev311 {1 jsonContent} with a JSON-RPC id, answered by ev312 with the same id. Only + * while the turn is open, which is while the model is asked. + */ + private suspend fun glassesCall(l: Link, full: String, input: JSONObject): ToolResult { + val m0 = l.m0 ?: return ToolResult("the glasses are not connected", true) + val id = l.callIds.getAndIncrement().toString() + val reply = CompletableDeferred() + l.calls[id] = reply + try { + val params = JSONObject().put("name", full).put("arguments", input) + val request = JSONObject().put("jsonrpc", "2.0").put("id", id.toInt()).put("method", "tools/call").put("params", params) + l.mux.send(m0, netEvent(EV_MCP_REQUEST, ThriftWriter().str(1, request.toString()).done()), messageType = MT_NET_EVENT) + val json = withTimeoutOrNull(MCP_WAIT_MS) { reply.await() } + say(if (json == null) "The glasses did not answer $full" else "The glasses answered $full") + return json?.let(::glassesResult) ?: ToolResult("no answer from the glasses", true) + } finally { + l.calls.remove(id) + } + } + + /** + * A device intent the glasses understood, handed back as ev380 {1 requestId, 2 interactionId, 3 intent}, the intent + * as they sent it. True when they ran it and the wearer hears it done; false leaves the turn to the model. + */ + private suspend fun handOver(l: Link, interaction: String, transcript: String, body: ByteArray, intent: String): Boolean { + val m0 = l.m0 ?: return false + val raw = Thrift.rawStruct(body, 55) ?: return false + val requestId = UUID.randomUUID().toString() + val answer = CompletableDeferred>() + l.handovers[requestId] = answer + val (done, message) = try { + l.mux.send(m0, netEvent(EV_REMOTE_INTENT_REQUEST, ThriftWriter().str(1, requestId).str(2, interaction).struct(3, raw).done()), messageType = MT_NET_EVENT) + withTimeoutOrNull(INTENT_WAIT_MS) { answer.await() } ?: (false to "no answer") + } finally { + l.handovers.remove(requestId) + } + say("Glasses intent $intent handed back: ${if (done) "done" else "not done"} ($message)") + if (!done) return false + if (diary.turn(interaction) == null) record(TurnRecord(interaction, System.currentTimeMillis(), transcript)) + speakTurn(l, interaction, transcript, "Tell the wearer in two or three words that it is done.") + return true + } + /** Speak, as this interaction's answer, a sentence the model writes from [what] in the glasses' language. */ private suspend fun speakTurn(l: Link, interaction: String, transcript: String, what: String) { _phase.value = AssistantPhase.THINKING @@ -762,7 +836,7 @@ class Assistant( // --- answering ----------------------------------------------------------- - private suspend fun answer(l: Link, interaction: String, transcript: String, visual: Visual) { + private suspend fun answer(l: Link, interaction: String, transcript: String, visual: Visual, glasses: GlassesToolbox? = null) { val started = System.currentTimeMillis() // A typed turn is in the diary already, with its source and conversation. if (diary.turn(interaction) == null) record(TurnRecord(interaction, started, transcript)) @@ -793,7 +867,7 @@ class Assistant( // The glasses give up after their processing timeout; speech and streaming need the last seconds of it. val budget = (processingTimeoutS() ?: FACTORY_PROCESSING_S) * 1000 - DELIVERY_MS val left = (budget - (System.currentTimeMillis() - started)).coerceAtLeast(MIN_MODEL_MS) - withTimeout(left) { brain.answer(transcript, l.locale, pastOf(interaction), image, note) } + withTimeout(left) { brain.answer(transcript, l.locale, pastOf(interaction), image, note, glasses = glasses) } } catch (error: Exception) { problem = error.message ?: error::class.simpleName.orEmpty() Reply(fallback(l.locale, store.config.value.apiKey.isBlank() && store.config.value.provider == Provider.CLAUDE)) @@ -1215,6 +1289,8 @@ class Assistant( val WAKE_WORD_ONLY = Regex("^(hey|hi|ehi|ei|ok|okay)?[ ,]*(meta|mata|metta|matta|meda)[.!?]*$", RegexOption.IGNORE_CASE) const val MIN_MODEL_MS = 6_000L const val PHOTO_WAIT_MS = 8_000L + const val MCP_WAIT_MS = 8_000L + const val INTENT_WAIT_MS = 5_000L // The "Sharing" photo came 0.2 s before its question (issue #18); a low-res one is 13 KB. const val EARLY_IMAGE_MS = 10_000L const val EARLY_MAX_BYTES = 2_000_000L @@ -1233,6 +1309,12 @@ class Assistant( /** 1026 is either the bare locale string or `{1: locale}`. */ private val CANCEL_INTENTS = setOf("IN:CANCEL", "IN:NEVER_MIND", "IN:STOP", "IN:NEGATE_INTENT", "IN:CANCEL_MESSAGE") +/** The glasses' own device intents Visor hands back to them (ev380) on a Display: volume and brightness only. */ +private val DEVICE_INTENTS = setOf( + "IN:SET_VOLUME", "IN:INCREASE_VOLUME", "IN:DECREASE_VOLUME", "IN:MUTE_VOLUME", "IN:UNMUTE_VOLUME", + "IN:SET_BRIGHTNESS", "IN:INCREASE_BRIGHTNESS", "IN:DECREASE_BRIGHTNESS", "IN:MAXIMIZE_BRIGHTNESS", +) + /** The text of [name] in the glasses' slot map (field 26): each slot is a list of {1 name, 4 text, 5 start, 6 end}. */ internal fun slot(slots: Any?, name: String): String = ((slots as? Map<*, *>)?.get(name) as? List<*>)?.firstOrNull().asStruct()[4].asText().trim() diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Brain.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Brain.kt index bce559c8..c04b91c8 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Brain.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Brain.kt @@ -37,7 +37,7 @@ private fun Exchange.told(): String = if (actions.isEmpty()) answer else data class Reply(val text: String, val remembered: List = emptyList(), val actions: List = emptyList()) /** What one answer may use and what it did with it; [photo] is the glasses' JPEG that came with the question. */ -private class Turn(val box: Toolbox, val photo: ByteArray? = null) { +private class Turn(val box: Toolbox, val photo: ByteArray? = null, val glasses: GlassesToolbox? = null) { val remembered = ArrayList() val actions = ArrayList() } @@ -79,11 +79,12 @@ class Brain( image: ByteArray? = null, note: String = "", audience: Audience = Audience.GLASSES, + glasses: GlassesToolbox? = null, ): Reply = withContext(Dispatchers.IO) { val c = config() check(c.apiKey.isNotBlank() || c.provider == Provider.OPENAI_COMPATIBLE) { "no API key is set" } // The tools never stand in the way of an answer: without them the model still answers. - val turn = Turn(hub?.let { runCatching { it.toolbox() }.getOrNull() } ?: Toolbox(emptyList(), ""), image) + val turn = Turn(hub?.let { runCatching { it.toolbox() }.getOrNull() } ?: Toolbox(emptyList(), ""), image, glasses) val system = (if (audience == Audience.PHONE) PHONE_PROMPT else SYSTEM_PROMPT) + " It is ${ZonedDateTime.now().format(DateTimeFormatter.ofPattern("EEEE d MMMM yyyy, HH:mm", Locale.ENGLISH))}." + (if (audience == Audience.GLASSES) " The glasses are set to ${locale.getDisplayLanguage(Locale.ENGLISH)}." else "") + @@ -160,6 +161,8 @@ class Brain( action?.let(turn.actions::add) return result } + val glasses = turn.glasses + if (glasses != null) glasses.tools.find { it.short == name }?.let { return glasses.call(it.full, input) } val tool = turn.box.find(name) ?: return ToolResult("unknown tool", true) val (result, action) = hub?.call(tool, input, turn.photo) ?: return ToolResult("unavailable", true) action?.let(turn.actions::add) @@ -199,6 +202,7 @@ class Brain( } if (turn != null) phoneTools?.tools?.forEach { (name, help, input) -> tools.put(JSONObject().put("name", name).put("description", help).put("input_schema", input)) } turn?.box?.tools?.forEach { t -> tools.put(JSONObject().put("name", t.name).put("description", t.help()).put("input_schema", t.tool.schema)) } + turn?.glasses?.tools?.forEach { t -> tools.put(JSONObject().put("name", t.short).put("description", t.help).put("input_schema", t.schema)) } var said = "" repeat(MAX_TOOL_ROUNDS) { // Current models think before they answer, and the thinking counts toward max_tokens. @@ -255,6 +259,7 @@ class Brain( if (turn != null) phoneTools?.tools?.forEach { (name, help, input) -> put(function(name, help, input)) } // Some servers behind this API refuse a schema that names its own dialect. turn?.box?.tools?.forEach { t -> put(function(t.name, t.help(), JSONObject(t.tool.schema.toString()).apply { remove("\$schema") })) } + turn?.glasses?.tools?.forEach { t -> put(function(t.short, t.help, t.schema)) } }.takeIf { it.length() > 0 } var said = "" repeat(MAX_TOOL_ROUNDS) { diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/GlassesTools.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/GlassesTools.kt new file mode 100644 index 00000000..758cdb8c --- /dev/null +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/GlassesTools.kt @@ -0,0 +1,98 @@ +package com.macedonga.visor.core.assistant + +import com.macedonga.visor.core.assistant.mcp.ToolResult +import org.json.JSONArray +import org.json.JSONObject + +/* + * The Display's own tools, offered to the model for one turn. Meta's server names them on ev57 field 54; a call goes + * out as ev311 tools/call and comes back on ev312, with ev387 only acknowledging it. Names and help are Visor's own. + */ + +/** One of the glasses' tools as the model sees it: the short name it calls, the glasses' full name, help and schema. */ +class GlassesTool(val short: String, val full: String, val help: String, val schema: JSONObject) + +/** The glasses' tools for one turn, and the call that runs one of them. */ +class GlassesToolbox(val tools: List, val call: suspend (full: String, input: JSONObject) -> ToolResult) + +private class Known(val help: String, val props: Map = emptyMap(), val required: List = emptyList()) + +/** Keyed by the method name. A prop is a JSON type, or "string:a|b" for an enum. */ +private val KNOWN: Map = mapOf( + "setVolume" to Known("Set the glasses' volume, 0 to 100.", mapOf("percent" to "integer"), listOf("percent")), + "getVolume" to Known("Read the glasses' volume, 0 to 100."), + "setDisplayBrightness" to Known("Set the display brightness, 0 to 100.", mapOf("percent" to "integer"), listOf("percent")), + "getDisplayBrightness" to Known("Read the display brightness, 0 to 100."), + "setDoNotDisturb" to Known("Turn do not disturb on or off.", mapOf("enabled" to "boolean"), listOf("enabled")), + "getDoNotDisturbState" to Known("Say whether do not disturb is on."), + "getBatteryLevel" to Known("Read the glasses' battery level."), + "getNowPlaying" to Known("Read what is playing."), + "pauseMusic" to Known("Pause the music."), + "resumeMusic" to Known("Resume the music."), + "stopMusic" to Known("Stop the music."), + "skipNext" to Known("Skip to the next track."), + "skipPrevious" to Known("Go back to the previous track."), + "setShuffle" to Known("Turn shuffle on or off.", mapOf("enabled" to "boolean"), listOf("enabled")), + "setRepeatMode" to Known("Set repeat to off, all or one.", mapOf("mode" to "string:off|all|one"), listOf("mode")), + "likeCurrentTrack" to Known("Like the track playing."), + "startApp" to Known("Open an app on the glasses by its Android package name.", mapOf("packageName" to "string"), listOf("packageName")), + "openUrl" to Known("Open a web address in the glasses' browser.", mapOf("url" to "string"), listOf("url")), + "openMap" to Known("Show a place on the glasses' map.", mapOf("query" to "string"), listOf("query")), + "searchPlaces" to Known("Search for places near the wearer.", mapOf("query" to "string", "maxResults" to "integer"), listOf("query", "maxResults")), + "navigateTo" to Known( + "Start directions to a place, by coordinates.", + mapOf( + "latitude" to "number", "longitude" to "number", "destinationName" to "string", + "travelMode" to "string:WALKING|DRIVING|BICYCLING|TRANSIT", + ), + listOf("latitude", "longitude", "destinationName", "travelMode"), + ), + "stopNavigation" to Known("Stop the navigation."), + "getNavStatus" to Known("Say whether navigation is running."), + "getNextManeuver" to Known("Read the next turn of the route."), + "camera_capture" to Known( + "Take a photo with the glasses' camera and return where it is.", + mapOf("width" to "integer", "height" to "integer", "airshipUpload" to "boolean"), + ), +) + +/** + * The glasses' tools named in [advertised] (the full names from ev57 field 54), with short names the model can send. + * Null when the glasses advertise none. + */ +internal fun glassesToolbox(advertised: List, call: suspend (String, JSONObject) -> ToolResult): GlassesToolbox? { + if (advertised.isEmpty()) return null + val tools = mutableListOf() + for (full in advertised.distinct()) { + val method = full.substringAfterLast('.') + val base = "glasses_" + method.replace(Regex("[^A-Za-z0-9_-]"), "_").take(50) + val short = if (tools.any { it.short == base }) "${base}_${tools.size}" else base + val known = KNOWN[method] + tools += if (known == null) { + GlassesTool(short, full, "The glasses' own tool $method. Pass the arguments it takes.", JSONObject().put("type", "object")) + } else { + GlassesTool(short, full, known.help, schema(known)) + } + } + return GlassesToolbox(tools, call) +} + +private fun schema(known: Known): JSONObject { + val props = JSONObject() + for ((name, spec) in known.props) { + val type = spec.substringBefore(':') + val values = spec.substringAfter(':', "").takeIf { it.isNotEmpty() }?.split('|') + props.put(name, JSONObject().put("type", type).apply { if (values != null) put("enum", JSONArray(values)) }) + } + return JSONObject().put("type", "object").put("properties", props).put("required", JSONArray(known.required)) +} + +/** A tools/call answer (ev312): a JSON-RPC error, or a result whose text the model reads. */ +internal fun glassesResult(json: String): ToolResult { + val reply = runCatching { JSONObject(json) }.getOrNull() ?: return ToolResult("unreadable answer from the glasses", true) + reply.optJSONObject("error")?.let { return ToolResult(it.optString("message").ifEmpty { "failed" }, true) } + val result = reply.optJSONObject("result") ?: return ToolResult("no result from the glasses", true) + val content = result.optJSONArray("content") ?: JSONArray() + val text = (0 until content.length()).mapNotNull { content.optJSONObject(it)?.optString("text")?.takeIf { s -> s.isNotEmpty() } } + return ToolResult(text.joinToString("\n").ifEmpty { "done" }, result.optBoolean("isError")) +} diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt index 153694e6..7885c8bf 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt @@ -63,6 +63,12 @@ internal const val EV_ON_PHONE_UNDERSTANDING = 342 internal const val EV_TTS_START_INFO = 388 internal const val EV_TTS_END_INFO = 389 internal const val EV_KEEP_AIR_SESSION_ALIVE = 392 +// The glasses' own tools and intents, run for a Display turn (see GlassesTools.kt). +internal const val EV_REMOTE_INTENT_REQUEST = 380 +internal const val EV_REMOTE_INTENT_RESPONSE = 381 +internal const val EV_MCP_REQUEST = 311 +internal const val EV_MCP_RESPONSE = 312 +internal const val EV_MCP_REQUEST_ACK = 387 /** The glasses' name for a NetEvent id. */ internal fun eventName(id: Int): String = EVENT_NAMES[id] ?: "event $id" diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Thrift.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Thrift.kt index 5b318d5f..4aa557a4 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Thrift.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Thrift.kt @@ -228,6 +228,30 @@ object Thrift { fun loads(b: ByteArray): Map = readStruct(b, 0).fields + /** The bytes of top-level struct field [fid] as written, to pass a sub-struct on unread. */ + fun rawStruct(b: ByteArray, fid: Int): ByteArray? { + var i = 0 + var cur = 0 + while (i < b.size) { + val h = b[i].toInt() and 0xFF + if (h == 0) break + i += 1 + val delta = (h shr 4) and 0x0F + val t = h and 0x0F + if (delta == 0) { + val v = uvarint(b, i) + i = v.offset + cur = zz(v.value).toInt() + } else { + cur += delta + } + val end = readValue(b, i, t).offset + if (cur == fid) return if (t == STRUCT) b.copyOfRange(i, end) else null + i = end + } + return null + } + // `thrift.py` also has `render`, a debug pretty-printer whose output is Python repr, and // a __main__ block that dumps a file from the command line. Neither is ported. From 4021a436c849554de05fce5f3fd1fd0041619795 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 09:53:20 +0200 Subject: [PATCH 52/90] Live AI: the session keep-alive names its interaction. KeepWearableAirSessionAlive (ev392) now carries field 4, interactionId, which the Display's and the Meta app's schemas both have; the Display keeps the session per interaction. Live AI and the custom wake word were checked against the Display's research: no product gate excludes the Display, and the enrolment events match its schema. --- .../main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt index ec1ecf84..0e7eb4c1 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt @@ -83,7 +83,7 @@ internal class LiveAi(private val context: Context, private val scope: Coroutine delay(20) host.event(EV_SERVER_CONNECTION_STATUS, ThriftWriter().str(1, id).bool(2, true).bool(3, false).i32(4, 2).done()) delay(20) - host.event(EV_KEEP_AIR_SESSION_ALIVE, ThriftWriter().str(1, "AsrStartListening").i32(2, 20_000).i32(3, nonce.getAndIncrement()).done()) + host.event(EV_KEEP_AIR_SESSION_ALIVE, ThriftWriter().str(1, "AsrStartListening").i32(2, 20_000).i32(3, nonce.getAndIncrement()).str(4, id).done()) ears?.stop() ears = GlassesEars(context, locale, continuous = true, onUtterance = ::heard, onProblem = { host.note("Live AI: $it") }).also { it.start() } keepalive?.cancel() @@ -144,7 +144,7 @@ internal class LiveAi(private val context: Context, private val scope: Coroutine while (scope.isActive && active) { delay(KEEPALIVE_MS) host.event(EV_FRAME_KEEP_ALIVE, ThriftWriter().i64(1, System.currentTimeMillis()).i32(2, nonce.getAndIncrement()).done()) - host.event(EV_KEEP_AIR_SESSION_ALIVE, ThriftWriter().str(1, "LiveAiSession").i32(2, 20_000).i32(3, nonce.getAndIncrement()).done()) + host.event(EV_KEEP_AIR_SESSION_ALIVE, ThriftWriter().str(1, "LiveAiSession").i32(2, 20_000).i32(3, nonce.getAndIncrement()).str(4, interaction).done()) } } From bfc0755035964ec2aa3eba60c0e781cb8d7582d3 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 09:54:28 +0200 Subject: [PATCH 53/90] Display: like, launch and playlists on svc9012. A like from the Display's music screen (type 15) sets the playing session's heart rating, or sends its own like action, and the liked state goes back in MediaPlayerInfo field 3 from the track's rating. A launch request (1) starts the partner app on the phone when Android allows it (Visor in front or a companion) and answers whether it started. Playlist requests (8, 12) are answered empty, as Visor has no Spotify login, and an empty Spotify list follows a subscription as the Display's own mock server sends it. A notification access request (11) opens Android's settings for it. Other requests are logged once. Not tried on the Display yet. --- .../macedonga/visor/connection/PhoneMedia.kt | 81 ++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMedia.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMedia.kt index 84ac126c..c11eba1c 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMedia.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMedia.kt @@ -1,25 +1,32 @@ package com.macedonga.visor.connection +import android.app.ActivityManager import android.content.ComponentName import android.content.Context +import android.content.Intent import android.graphics.Bitmap import android.graphics.BitmapFactory import android.media.MediaMetadata +import android.media.Rating import android.media.session.MediaController import android.media.session.MediaSessionManager import android.media.session.PlaybackState import android.net.Uri import android.os.Handler import android.os.Looper +import android.provider.Settings import androidx.core.app.NotificationManagerCompat import com.macedonga.visor.core.protocol.Wire import com.macedonga.visor.core.session.DataXConnection import com.macedonga.visor.core.session.DataXMessage import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog import com.macedonga.visor.notify.MirrorListener +import com.macedonga.visor.phone.companionAssociated import dagger.hilt.android.qualifiers.ApplicationContext import java.io.ByteArrayOutputStream import javax.inject.Inject +import javax.inject.Provider import javax.inject.Singleton import kotlinx.coroutines.CoroutineExceptionHandler import kotlinx.coroutines.CoroutineScope @@ -34,12 +41,15 @@ import kotlinx.coroutines.sync.withLock * channel, MediaPlayerInfo {1 player, 2 content type, 3 liked, 4 package} (type 100) and the cover art as a JPEG in * 1024-byte pieces (101, the last one 102; an empty 102 clears it), again whenever the playing session changes. With an * Android phone the Display's music screen takes the cover only from here; title and artist come over AVRCP. + * Playlists are answered empty (9, 13). Like (15) sets the heart rating or the app's like action. Launch (1) starts an + * app from the foreground or as a companion only. Other requests are logged once and not answered. */ @Singleton -class PhoneMedia @Inject constructor(@ApplicationContext private val context: Context) : Responder { +class PhoneMedia @Inject constructor(@ApplicationContext private val context: Context, private val log: Provider) : Responder { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main + CoroutineExceptionHandler { _, _ -> }) private val sending = Mutex() private val main = Handler(Looper.getMainLooper()) + private val unserved = mutableSetOf() @Volatile private var subscriber: Pair? = null private var sessions: MediaSessionManager? = null @@ -67,14 +77,34 @@ class PhoneMedia @Inject constructor(@ApplicationContext private val context: Co sentArt = null watch() push() + runCatching { sending.withLock { connection.send(reply, EMPTY, messageType = MT_SPOTIFY_PLAYLISTS_RESPONSE) } } } MT_INFO_REQUEST -> { val (player, name) = player(controller) connection.send(reply, Wire.fieldVarint(1, player) + Wire.fieldBytes(2, name.encodeToByteArray()), messageType = MT_INFO_RESPONSE) } + MT_TRY_LAUNCH -> { + val f = fields(message) + val pkg = f?.bytes(2)?.decodeToString().orEmpty() + val ok = pkg.isNotEmpty() && launch(pkg) + if (!ok) log.get().add("Partner app $pkg: not started from the background") + connection.send(reply, Wire.fieldVarint(1, if (ok) 1 else 0) + Wire.fieldBytes(4, f?.bytes(3) ?: EMPTY), messageType = MT_TRY_LAUNCH) + } + MT_LIKE -> scope.launch { setLiked(fields(message)?.varint(1) == 1L) } + MT_NOTIFICATION_ACCESS -> scope.launch { + if (!startFromHere(Intent(Settings.ACTION_NOTIFICATION_LISTENER_SETTINGS))) log.get().add("Notification access: Visor may not open settings from the background") + } + MT_SPOTIFY_PLAYLISTS_REQUEST -> connection.send(reply, EMPTY, messageType = MT_SPOTIFY_PLAYLISTS_RESPONSE) + MT_PLAYLIST_REQUEST -> { + val partner = fields(message)?.bytes(1) ?: EMPTY + connection.send(reply, Wire.fieldBytes(3, partner), messageType = MT_PLAYLIST_RESPONSE) + } + else -> if (unserved.add(message.info.messageType)) log.get().add("Partner ecosystem: request ${message.info.messageType} not served") } } + private fun fields(message: DataXMessage) = runCatching { Wire.parse(message.payload) }.getOrNull() + /** From the first subscription on, follows the playing session; without notification access there is none. */ private fun watch() { if (sessions != null) return @@ -109,7 +139,7 @@ class PhoneMedia @Inject constructor(@ApplicationContext private val context: Co sending.withLock { val (mux, reply) = subscriber ?: return@withLock val (player, name) = player(session) - val info = Wire.fieldVarint(1, player) + Wire.fieldVarint(2, CONTENT_UNKNOWN) + Wire.fieldVarint(3, LIKED_UNKNOWN) + + val info = Wire.fieldVarint(1, player) + Wire.fieldVarint(2, CONTENT_UNKNOWN) + Wire.fieldVarint(3, liked(session)) + Wire.fieldBytes(4, name.encodeToByteArray()) val art = session?.metadata?.let { art(it) } val newInfo = !info.contentEquals(sentInfo) @@ -158,20 +188,67 @@ class PhoneMedia @Inject constructor(@ApplicationContext private val context: Co return (PLAYERS[pkg] ?: 0L) to pkg } + // The heart rating of the track, when the session keeps one. + private fun liked(session: MediaController?): Long { + val rating = session?.metadata?.getRating(MediaMetadata.METADATA_KEY_USER_RATING) ?: return LIKED_UNKNOWN + if (rating.ratingStyle != Rating.RATING_HEART || !rating.isRated) return LIKED_UNKNOWN + return if (rating.hasHeart()) LIKED_YES else LIKED_NO + } + + // Heart rating when the session takes it, else its like action; the glasses get no reply, only the new state. + private fun setLiked(wanted: Boolean) { + val session = controller ?: return + val action = session.playbackState?.customActions?.firstOrNull { a -> + val words = a.action + " " + a.name + LIKE_WORDS.any { words.contains(it, ignoreCase = true) } && !NOT_LIKE.containsMatchIn(words) + } + when { + session.ratingType == Rating.RATING_HEART -> session.transportControls.setRating(Rating.newHeartRating(wanted)) + action == null -> log.get().add("Like: ${session.packageName} has no like control") + liked(session) != (if (wanted) LIKED_YES else LIKED_NO) -> session.transportControls.sendCustomAction(action.action, null) + } + push() + } + + // Starts the partner app as the Meta app would: only from the foreground or as a companion, else it stays put. + private fun launch(pkg: String): Boolean { + val intent = context.packageManager.getLaunchIntentForPackage(pkg) ?: return false + return startFromHere(intent) + } + + private fun startFromHere(intent: Intent): Boolean = + (onScreen() || companionAssociated(context)) && runCatching { context.startActivity(intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)) }.isSuccess + + private fun onScreen(): Boolean = ActivityManager.RunningAppProcessInfo() + .also(ActivityManager::getMyMemoryState).importance == ActivityManager.RunningAppProcessInfo.IMPORTANCE_FOREGROUND + companion object { const val SVC = 9012 + private const val MT_TRY_LAUNCH = 1 private const val MT_INFO_REQUEST = 3 private const val MT_INFO_RESPONSE = 4 private const val MT_SUBSCRIBE = 5 + private const val MT_SPOTIFY_PLAYLISTS_REQUEST = 8 + private const val MT_SPOTIFY_PLAYLISTS_RESPONSE = 9 + private const val MT_NOTIFICATION_ACCESS = 11 + private const val MT_PLAYLIST_REQUEST = 12 + private const val MT_PLAYLIST_RESPONSE = 13 + private const val MT_LIKE = 15 private const val MT_PLAYER_INFO = 100 private const val MT_ART_PIECE = 101 private const val MT_ART_LAST = 102 private const val CONTENT_UNKNOWN = 0L private const val LIKED_UNKNOWN = 0L + private const val LIKED_YES = 1L + private const val LIKED_NO = 2L private const val NO_ACCESS = 7L private const val CHUNK = 1024 private const val MAX_SIDE = 300 private const val JPEG_QUALITY = 70 + private val EMPTY = ByteArray(0) + private val LIKE_WORDS = listOf("like", "thumbs", "heart", "favorite", "favourite") + // Thumbs down and dislike sit beside the like action in many players. + private val NOT_LIKE = Regex("dislike|unlike|down|remove", RegexOption.IGNORE_CASE) private val PLAYERS = mapOf( "com.spotify.music" to 1L, "com.apple.android.music" to 2L, "com.amazon.mp3" to 3L, "com.audible.application" to 4L, "com.clearchannel.iheartradio.controller" to 5L, "com.calm.android" to 6L, From 73594b6c9f1558542b9f149a991ae3fc69ddbc9b Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:01:08 +0200 Subject: [PATCH 54/90] Firmware updates from Meta. The glasses' firmware row opens a sheet that asks Meta (FetchConstellationUpdates, with the user's token, for the pair's serial, device type and build) whether a newer full image is offered, and installs it on confirmation: svc29 StartApp for the OTA app, then OtaUpdateInstall with the offer's download URL, versions, payload offset and size (the FILE_SIZE line of its install headers, as the Meta app reads it), headers and KEK, and progress until the glasses report the install complete. The glasses download the image over their own Wi-Fi; nothing goes over Bluetooth. The pack offer query and this one share their request code. Not tried on the glasses yet. --- .../visor/firmware/FirmwareUpdate.kt | 223 ++++++++++++++++++ .../macedonga/visor/settings/GlassesPage.kt | 49 +++- .../main/res/values-it/strings_firmware.xml | 15 ++ app/src/main/res/values/strings_firmware.xml | 15 ++ .../core/identity/firmware/FirmwareOffer.kt | 99 ++++++++ .../core/identity/language/LanguagePacks.kt | 56 +++-- 6 files changed, 433 insertions(+), 24 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/firmware/FirmwareUpdate.kt create mode 100644 app/src/main/res/values-it/strings_firmware.xml create mode 100644 app/src/main/res/values/strings_firmware.xml create mode 100644 core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/FirmwareOffer.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/firmware/FirmwareUpdate.kt b/app/src/main/kotlin/com/macedonga/visor/firmware/FirmwareUpdate.kt new file mode 100644 index 00000000..51a6fdaf --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/firmware/FirmwareUpdate.kt @@ -0,0 +1,223 @@ +package com.macedonga.visor.firmware + +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.firmware.FirmwareOffer +import com.macedonga.visor.core.identity.firmware.fetchFirmwareOffer +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.ServiceData +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.login.MetaLogin +import java.util.UUID +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull + +sealed interface FirmwareWork { + data object Idle : FirmwareWork + /** [percent] is null until the glasses report some progress. */ + data class Installing(val percent: Int? = null) : FirmwareWork + data object Installed : FirmwareWork + data class Failed(val reason: String) : FirmwareWork +} + +/** + * The glasses' firmware from Meta. [check] asks Meta for the newest full image; [install] sends its address to the + * glasses over svc29 (the OTA service), and the glasses fetch the image over their own Wi-Fi. If they ask for the + * image over Bluetooth instead, the install stops: that fallback is not built. + */ +@Singleton +class FirmwareUpdate @Inject constructor( + private val login: MetaLogin, + private val connection: GlassesConnection, + private val log: EventLog, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + + private val _checking = MutableStateFlow(false) + val checking: StateFlow = _checking.asStateFlow() + /** Meta answered for the glasses' current build, offer or none. */ + private val _checked = MutableStateFlow(false) + val checked: StateFlow = _checked.asStateFlow() + private val _offer = MutableStateFlow(null) + val offer: StateFlow = _offer.asStateFlow() + private val _problem = MutableStateFlow("") + val problem: StateFlow = _problem.asStateFlow() + private val _work = MutableStateFlow(FirmwareWork.Idle) + val work: StateFlow = _work.asStateFlow() + private var job: Job? = null + + init { + // An offer is for one pair's build: a different pair starts over. + scope.launch { + connection.pair.filterNotNull().distinctUntilChanged().drop(1).collect { + _offer.value = null + _checked.value = false + _problem.value = "" + } + } + } + + fun check() { + if (_checking.value || job?.isActive == true) return + scope.launch { + _checking.value = true + try { + val info = connection.state.value.info ?: error("connect the glasses once, so they say which they are") + // Meta/greatwhite/greatwhite:14/UKQ1.250303.001/69951180085500080:user/release-keys + val parts = info.build.split('/') + val firmware = parts.getOrNull(4)?.substringBefore(':').orEmpty() + val type = parts.getOrNull(4)?.substringAfter(':', "user").orEmpty() + val product = parts.getOrNull(1).orEmpty() + if (firmware.isEmpty() || product.isEmpty()) error("the glasses' build is unknown") + val found = fetchFirmwareOffer(login.source.get(), info.serial, "ota.$product.$type", firmware) + _offer.value = found + _problem.value = "" + _checked.value = true + log.add("Firmware: " + (found?.let { "${it.targetVersion} is offered" } ?: "no newer build is offered")) + } catch (error: Throwable) { + if (error is kotlinx.coroutines.CancellationException) throw error + _problem.value = error.message ?: error::class.java.simpleName + log.add("Firmware: check failed: ${error.message}") + } finally { + _checking.value = false + } + } + } + + fun install() { + val offer = _offer.value ?: return + if (job?.isActive == true) return + job = scope.launch { + try { + if (connection.state.value.link != LinkState.READY) error("the glasses are not connected") + _work.value = FirmwareWork.Installing() + log.add("Firmware: installing ${offer.targetVersion} on the glasses") + val why = runCatching { installOn(offer) }.exceptionOrNull()?.let { it.message ?: it::class.java.simpleName } + if (why != null) error(why) + _offer.value = null + _work.value = FirmwareWork.Installed + log.add("Firmware: ${offer.targetVersion} is installed, the glasses restart") + } catch (error: kotlinx.coroutines.CancellationException) { + _work.value = FirmwareWork.Idle + throw error + } catch (error: Throwable) { + log.add("Firmware: install failed: ${error.message}") + _work.value = FirmwareWork.Failed(error.message ?: error::class.java.simpleName) + } + } + } + + fun cancel() { + job?.cancel() + } + + /** Back to the start after an install ended. */ + fun dismiss() { + if (job?.isActive != true) _work.value = FirmwareWork.Idle + } + + private suspend fun installOn(offer: FirmwareOffer) = coroutineScope { + val fromGlasses = Channel(Channel.UNLIMITED) + // Subscribed before the first message goes out, so no answer is missed. + val listening = launch(start = CoroutineStart.UNDISPATCHED) { + connection.serviceData.collect { if (it.serviceId == SVC_OTA) fromGlasses.send(it) } + } + suspend fun next(type: Int, ms: Long): ServiceData? = withTimeoutOrNull(ms) { + var m: ServiceData + do { + m = fromGlasses.receive() + when (m.messageType) { + MT_ERROR -> error(errorWords(m.payload)) + MT_FILE_STATUS -> error("the glasses asked for the image over Bluetooth, not over their Wi-Fi") + MT_PROGRESS -> progress(m.payload) + } + } while (m.messageType != type) + m + } + try { + val install = UUID.randomUUID().toString() + val megabytes = (offer.size / 1_000_000 + 1) * 2 + val start = Wire.fieldVarint(1, APP_OTA) + Wire.fieldBytes(2, Wire.fieldBytes(1, Wire.fieldVarint(1, megabytes))) + + intent("ota_type", Wire.fieldBytes(5, str("firmware"))) + intent("attempt_id", Wire.fieldBytes(5, str(install))) + connection.submit { it.sendOnService(SVC_OTA, MT_START_APP, start) } + val started = next(MT_START_APP_RESPONSE, ANSWER_MS) ?: error("the glasses' OTA service did not answer") + val code = runCatching { Wire.parse(started.payload).varint(1) }.getOrDefault(0) + // Already started: its workflow is up, and still takes the request. + if (code != SNAPP_SUCCESS && code != SNAPP_ALREADY_STARTED) error("the glasses would not start an install (code $code)") + + val key = Wire.fieldBytes(1, str(offer.sessionId)) + Wire.fieldBytes(2, str(install)) + val request = Wire.fieldBytes(1, key) + + Wire.fieldBytes(3, str(offer.baseVersion)) + + Wire.fieldBytes(4, str(offer.targetVersion)) + + Wire.fieldVarint(5, payloadSize(offer)) + + Wire.fieldVarint(6, offer.payloadOffset) + + (if (offer.metaHeaders.isNotEmpty()) Wire.fieldBytes(7, str(offer.metaHeaders)) else ByteArray(0)) + + Wire.fieldBytes(10, str(offer.url)) + + (offer.kekId?.let { Wire.fieldVarint(11, it) } ?: ByteArray(0)) + + (if (offer.kekHash.isNotEmpty()) Wire.fieldBytes(12, str(offer.kekHash)) else ByteArray(0)) + connection.submit { it.sendOnService(SVC_OTA, MT_INSTALL, request) } + _work.value = FirmwareWork.Installing() + next(MT_INSTALL_COMPLETE, INSTALL_MS) ?: error("the glasses did not finish installing it") + } finally { + listening.cancel() + } + } + + // The payload's size is the FILE_SIZE line of the install headers, as the Meta app reads it; else the image's size. + private fun payloadSize(offer: FirmwareOffer): Long = + offer.metaHeaders.lineSequence().firstOrNull { it.startsWith("FILE_SIZE") }?.substringAfter('=')?.trim()?.toLongOrNull() ?: offer.size + + /** Progress for the sheet: the percentage the glasses give, when it is in range. */ + private fun progress(payload: ByteArray) { + val f = runCatching { Wire.parse(payload) }.getOrNull() ?: return + if (!f.has(3)) return + val percent = f.float(3).toInt() + if (percent in 0..100) _work.value = FirmwareWork.Installing(percent) + } + + private fun errorWords(payload: ByteArray): String = runCatching { + val f = Wire.parse(payload) + val stage = STAGES.getOrElse(f.varint(2).toInt() - 1) { "stage ${f.varint(2)}" } + "the glasses stopped the install at $stage, code ${f.varint(3)}, error ${f.varint(5)}" + + if (f.varint(4) == 1L) " (retry)" else "" + }.getOrDefault("the glasses stopped the install") + + private fun str(s: String) = s.encodeToByteArray() + + private fun intent(key: String, value: ByteArray) = Wire.fieldBytes(3, Wire.fieldBytes(1, str(key)) + value) + + private companion object { + const val SVC_OTA = 29 + const val MT_INSTALL = 1 + const val MT_PROGRESS = 2 + const val MT_ERROR = 3 + const val MT_FILE_STATUS = 4 + const val MT_INSTALL_COMPLETE = 7 + const val MT_START_APP = 2000 + const val MT_START_APP_RESPONSE = 2001 + // The OTA app, as the Meta app numbers it for svc29 starts. + const val APP_OTA = 1L + const val SNAPP_SUCCESS = 1L + const val SNAPP_ALREADY_STARTED = 12L + const val ANSWER_MS = 15_000L + // A download over the glasses' Wi-Fi and the A/B apply: a guess, no timing was read. + const val INSTALL_MS = 60 * 60_000L + // OtaUpdateInstallStage, in the order the Meta app numbers them from 1. + val STAGES = listOf("transfer", "apply", "reboot", "stopped", "startup", "complete", "download") + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt index 211e43d8..ad40de60 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt @@ -32,6 +32,8 @@ import com.macedonga.visor.WifiNetworksKey import com.macedonga.visor.R import com.macedonga.visor.connection.GlassesService import com.macedonga.visor.connection.LinkKind +import com.macedonga.visor.firmware.FirmwareUpdate +import com.macedonga.visor.firmware.FirmwareWork import com.macedonga.visor.home.pairName import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.connection.GlassesLink @@ -62,6 +64,7 @@ class GlassesPageModel @Inject constructor( val connection: GlassesConnection, val settings: GlassesSettings, val login: MetaLogin, + val firmware: FirmwareUpdate, val kind: LinkKind, private val store: IdentityStore, val looks: Looks, @@ -126,6 +129,13 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( val serial = glasses.target?.serial ?: info?.serial var renaming by remember { mutableStateOf(false) } var forgetting by remember { mutableStateOf(false) } + var firmwareOpen by remember { mutableStateOf(false) } + var confirmingFirmware by remember { mutableStateOf(false) } + val fwChecking by model.firmware.checking.collectAsStateWithLifecycle() + val fwChecked by model.firmware.checked.collectAsStateWithLifecycle() + val fwOffer by model.firmware.offer.collectAsStateWithLifecycle() + val fwProblem by model.firmware.problem.collectAsStateWithLifecycle() + val fwWork by model.firmware.work.collectAsStateWithLifecycle() SettingsPage(name, light = 0.6f) { item(key = "power") { @@ -174,7 +184,7 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( }, look?.let { l -> l.colorName ?: l.color }?.let { v -> @Composable { ValueRow(stringResource(R.string.glasses_color), v, onClick = null) } }, look?.let { l -> l.lensName ?: l.lens }?.let { v -> @Composable { ValueRow(stringResource(R.string.glasses_lenses), v, onClick = null) } }, - { ValueRow(stringResource(R.string.glasses_firmware), info?.build?.let(::firmwareWords) ?: stringResource(R.string.settings_unknown), onClick = null) }, + { ValueRow(stringResource(R.string.glasses_firmware), info?.build?.let(::firmwareWords) ?: stringResource(R.string.settings_unknown), onClick = { firmwareOpen = true }) }, { ValueRow(stringResource(R.string.glasses_serial), serial ?: stringResource(R.string.settings_unknown), onClick = null) }, device?.thermal?.let { t -> @Composable { ValueRow(stringResource(R.string.glasses_heat), heatWords(t.health, t.throttling), onClick = null) } }, device?.storage?.let { s -> @Composable { ValueRow(stringResource(R.string.glasses_storage), stringResource(R.string.glasses_storage_free, gb(s.freeMb), gb(s.totalMb)), onClick = null) } }, @@ -274,6 +284,43 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( } } } + if (firmwareOpen) { + VisorSheet({ firmwareOpen = false; confirmingFirmware = false; model.firmware.dismiss() }) { + SheetTitle(stringResource(R.string.firmware_title)) + Rows( + { ValueRow(stringResource(R.string.firmware_current), info?.build?.let(::firmwareWords) ?: stringResource(R.string.settings_unknown), onClick = null) }, + fwOffer?.let { o -> @Composable { ValueRow(stringResource(R.string.firmware_update), o.displayName.ifEmpty { o.targetVersion }, onClick = null) } }, + ) + val status: String? = when (val w = fwWork) { + is FirmwareWork.Failed -> stringResource(R.string.firmware_failed, w.reason) + FirmwareWork.Installed -> stringResource(R.string.firmware_installed) + is FirmwareWork.Installing -> w.percent?.let { stringResource(R.string.firmware_progress, it) } ?: stringResource(R.string.firmware_installing) + FirmwareWork.Idle -> when { + fwChecking -> stringResource(R.string.firmware_checking) + fwProblem.isNotEmpty() -> fwProblem + fwChecked && fwOffer == null -> stringResource(R.string.firmware_up_to_date) + else -> null + } + } + Column(Modifier.padding(horizontal = SIDE, vertical = 16.dp)) { + if (status != null) Footnote(status) + val offer = fwOffer + val installing = fwWork is FirmwareWork.Installing + if (confirmingFirmware && offer != null) { + Row(Modifier.padding(top = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp)) { + ActionPill(stringResource(R.string.firmware_install), { confirmingFirmware = false; model.firmware.install() }) + ActionPill(stringResource(R.string.language_cancel), { confirmingFirmware = false }, quiet = true) + } + Footnote(stringResource(R.string.firmware_confirm, offer.displayName.ifEmpty { offer.targetVersion })) + } else { + Row(Modifier.padding(top = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp)) { + ActionPill(stringResource(R.string.firmware_check), { model.firmware.check() }, quiet = true, enabled = !fwChecking && !installing) + if (offer != null) ActionPill(stringResource(R.string.firmware_install), { confirmingFirmware = true }, enabled = !installing) + } + } + } + } + } } private const val NAME_KEY = "device_custom_name" diff --git a/app/src/main/res/values-it/strings_firmware.xml b/app/src/main/res/values-it/strings_firmware.xml new file mode 100644 index 00000000..548872e2 --- /dev/null +++ b/app/src/main/res/values-it/strings_firmware.xml @@ -0,0 +1,15 @@ + + + Firmware + Versione + Aggiornamento + Cerca aggiornamenti + Ricerca + Aggiornato + Installa + Installare %1$s? Gli occhiali si riavviano al termine. + Installazione + Installazione, %1$d%% + Installato. Gli occhiali si riavviano. + Errore: %1$s + diff --git a/app/src/main/res/values/strings_firmware.xml b/app/src/main/res/values/strings_firmware.xml new file mode 100644 index 00000000..d3c7dcf6 --- /dev/null +++ b/app/src/main/res/values/strings_firmware.xml @@ -0,0 +1,15 @@ + + + Firmware + Version + Update + Check for update + Checking + Up to date + Install + Install %1$s? The glasses restart when it is done. + Installing + Installing, %1$d%% + Installed. The glasses restart. + Failed: %1$s + diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/FirmwareOffer.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/FirmwareOffer.kt new file mode 100644 index 00000000..11e897e6 --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/FirmwareOffer.kt @@ -0,0 +1,99 @@ +package com.macedonga.visor.core.identity.firmware + +import com.macedonga.visor.core.identity.HttpPost +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.language.constellationPost +import com.macedonga.visor.core.identity.token.Token +import com.macedonga.visor.core.identity.token.TokenError +import java.util.UUID +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.longOrNull +import kotlinx.serialization.json.put + +/** + * A full firmware image Meta offers for the glasses. The glasses fetch [url] themselves over their own Wi-Fi; the + * install message carries the fields the Meta app puts in it. + */ +data class FirmwareOffer( + val baseVersion: String, + val targetVersion: String, + /** Meta's name for the build; empty when Meta gives none. */ + val displayName: String, + val url: String, + val size: Long, + val checksum: String, + /** From the offer's install options: the payload's headers and offset inside the image. */ + val metaHeaders: String, + val payloadOffset: Long, + val kekId: Long?, + val kekHash: String, + /** The session the offer was asked under; the install names its update by it. */ + val sessionId: String, +) + +private const val UPDATES_DOC = "155904535318377635033579516369" + +/** + * FetchConstellationUpdates, the Meta app's own query for the glasses' firmware: the full image newer than + * [firmware], or null when there is none. [token] is the AR token from the login. + */ +fun fetchFirmwareOffer( + token: Token, + serial: String, + deviceType: String, + firmware: String, + http: HttpPost = UrlConnectionPost, +): FirmwareOffer? { + val sessionId = UUID.randomUUID().toString() + val info = buildJsonObject { + put("device_type", deviceType) + put("device_serial", serial) + put("device_firmware_version", firmware) + put("device_ota_session_id", sessionId) + put("device_feature_list", JsonArray(emptyList())) + put("device_artifact_list", JsonArray(emptyList())) + put("device_firmware_full_update_only", true) + put("device_firmware_rollout_token", JsonNull) + put("device_manage_mode", JsonNull) + // No constellation device is paired, so the app sends all three as null. + put("constellation_device_type", JsonNull) + put("constellation_device_firmware_version", JsonNull) + put("constellation_device_serial", JsonNull) + } + val (status, json) = constellationPost(token, "FetchConstellationUpdates", UPDATES_DOC, info, http) + val responses = json.path("data", "xar_constellation_updates")?.get("constellation_updates_response_info") as? JsonArray + ?: throw TokenError("Meta answered $status without updates") + val offer = responses.firstNotNullOfOrNull { (it.jsonObject["firmware_update_info"] as? JsonObject) } ?: return null + val url = offer.text("file_uri") ?: throw TokenError("Meta's firmware offer has no file") + // The options come as an object or as the JSON text of one. + val options = when (val raw = offer["extra_install_options"]) { + is JsonObject -> raw + is JsonPrimitive -> runCatching { Json.parseToJsonElement(raw.content).jsonObject }.getOrNull() + else -> null + } + return FirmwareOffer( + baseVersion = offer.text("base_version").orEmpty(), + targetVersion = offer.text("target_version").orEmpty(), + displayName = offer.text("build_number_display_name").orEmpty(), + url = url, + size = (offer["file_size"] as? JsonPrimitive)?.longOrNull ?: offer.text("file_size")?.toLongOrNull() ?: 0, + checksum = offer.text("file_checksum").orEmpty(), + metaHeaders = options?.text("headers").orEmpty(), + payloadOffset = (options?.get("payloadOffset") as? JsonPrimitive)?.longOrNull ?: 0, + kekId = (offer["kek_id"] as? JsonPrimitive)?.longOrNull, + kekHash = offer.text("kek_hash").orEmpty(), + sessionId = sessionId, + ) +} + +private fun JsonObject.path(vararg keys: String): JsonObject? = + keys.fold(this as JsonObject?) { o, k -> o?.get(k) as? JsonObject } + +private fun JsonObject.text(key: String): String? = (this[key] as? JsonPrimitive)?.contentOrNull diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt index da57cabf..63fd8350 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt @@ -39,7 +39,6 @@ const val LANGUAGE_FEATURE = "com.meta.constellation.feature.air.language_" /** Ray-Ban Meta Gen 2 on a user build, as the OTA service names it. */ const val GEN2_DEVICE_TYPE = "ota.hammerhead.user" -private const val GRAPH_URL = "https://ar-genai.graph.meta.com/graphql" private const val FETCH_UPDATES_DOC = "10649732309712548951633872342" /** @@ -69,13 +68,41 @@ fun fetchPackOffers( put("device_firmware_rollout_token", JsonNull) put("device_manage_mode", JsonNull) } + val (status, json) = constellationPost(token, "FetchConstellationFeatureUpdates", FETCH_UPDATES_DOC, info, http) + val responses = json.path("data", "xar_constellation_feature_updates")?.get("constellation_updates_response_info") as? JsonArray + ?: throw TokenError("Meta answered $status without updates") + return responses.flatMap { r -> + (r.jsonObject["resource_updates_info"] as? JsonArray).orEmpty().flatMap { u -> + val feature = u.jsonObject.path("feature")?.text("feature_name").orEmpty() + (u.jsonObject["feature_resource_updates_info"] as? JsonArray).orEmpty().mapNotNull { e -> + val o = e.jsonObject + PackOffer( + feature = feature, + resource = o.text("resource_name") ?: return@mapNotNull null, + version = o.text("resource_version").orEmpty(), + url = o.text("file_uri") ?: return@mapNotNull null, + size = (o["file_size"] as? JsonPrimitive)?.longOrNull ?: o.text("file_size")?.toLongOrNull() ?: 0, + sha256 = o.text("file_checksum").orEmpty(), + appId = o.text("app_id").orEmpty(), + type = o.text("resource_type").orEmpty(), + ) + } + } + } +} + +/** + * One FetchConstellation query as the Meta app sends it, with [info] as its `constellation_updates_request_info`. + * Returns the HTTP status and the answer's JSON. + */ +internal fun constellationPost(token: Token, friendlyName: String, docId: String, info: JsonObject, http: HttpPost): Pair { val variables = buildJsonObject { put("fetch_request", buildJsonObject { put("constellation_updates_request_info", JsonArray(listOf(info))) }) } val body = pyUrlencode( linkedMapOf( - "fb_api_req_friendly_name" to "FetchConstellationFeatureUpdates", - "client_doc_id" to FETCH_UPDATES_DOC, + "fb_api_req_friendly_name" to friendlyName, + "client_doc_id" to docId, "variables" to variables.toString(), "method" to "post", "locale" to "en_US", @@ -95,28 +122,11 @@ fun fetchPackOffers( val json = runCatching { Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject } .getOrElse { throw TokenError("Meta answered ${reply.status} with no JSON") } json["errors"]?.let { throw TokenError("Meta refused: ${it.toString().take(200)}") } - val responses = json.path("data", "xar_constellation_feature_updates")?.get("constellation_updates_response_info") as? JsonArray - ?: throw TokenError("Meta answered ${reply.status} without updates") - return responses.flatMap { r -> - (r.jsonObject["resource_updates_info"] as? JsonArray).orEmpty().flatMap { u -> - val feature = u.jsonObject.path("feature")?.text("feature_name").orEmpty() - (u.jsonObject["feature_resource_updates_info"] as? JsonArray).orEmpty().mapNotNull { e -> - val o = e.jsonObject - PackOffer( - feature = feature, - resource = o.text("resource_name") ?: return@mapNotNull null, - version = o.text("resource_version").orEmpty(), - url = o.text("file_uri") ?: return@mapNotNull null, - size = (o["file_size"] as? JsonPrimitive)?.longOrNull ?: o.text("file_size")?.toLongOrNull() ?: 0, - sha256 = o.text("file_checksum").orEmpty(), - appId = o.text("app_id").orEmpty(), - type = o.text("resource_type").orEmpty(), - ) - } - } - } + return reply.status to json } +private const val GRAPH_URL = "https://ar-genai.graph.meta.com/graphql" + private fun JsonObject.path(vararg keys: String): JsonObject? = keys.fold(this as JsonObject?) { o, k -> o?.get(k) as? JsonObject } From e61c3c435d138be06c01ff8310de64bc8f8ae48b Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:12:59 +0200 Subject: [PATCH 55/90] Display: smart replies on chat notifications, and their answers sent. The Display's SystemUI keeps a chat notification only when its app id is one it knows, so Messenger and Instagram notifications now carry com.facebook.messenger and com.instagram, as the Meta app sends them, instead of the phone's package names; icon requests for those ids load the installed app's icon. A reply picked on the Display for a Messenger, WhatsApp or Instagram chat (svc70 type 8, provider 1, 2 or 4) goes out through that notification's own reply action, which is now kept for every chat, not only for texts. The Display's Phone app reads contacts from RLDrive contacts_native, which Visor already writes. Not tried on the Display yet. --- .../macedonga/visor/connection/PhoneMessages.kt | 15 +++++++++------ .../com/macedonga/visor/notify/MirrorListener.kt | 12 +++++++----- .../com/macedonga/visor/notify/NotifyIcons.kt | 10 +++++++++- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt index f8fdf3fe..cedc9005 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt @@ -68,14 +68,15 @@ class PhoneMessages @Inject constructor( // The Phone app's recent calls, asked once per provider, each waiting for its answer. if (type == MT_LOAD_CALL_LOG) return calls.loadCallLog(request, reply, connection) if (type !in SERVED) return log.get().add("Messages: type $type not served") - // Only the phone's texts; the other providers are the Meta app's. + // The phone's texts, and replies to a chat notification (Messenger, WhatsApp, Instagram) through its reply action. val provider = request.varint(1) - if (provider != NATIVE) return log.get().add("Messages: provider $provider not served") + val chat = type == MT_OUTGOING && provider in CHAT + if (provider != NATIVE && !chat) return log.get().add("Messages: provider $provider not served") scope.launch { when (type) { MT_LOAD_THREADS -> threads(request, reply, connection) MT_LOAD_MESSAGES -> messages(request, reply, connection) - MT_OUTGOING -> send(request, reply, connection) + MT_OUTGOING -> send(request, reply, connection, chat) } } } @@ -140,16 +141,17 @@ class PhoneMessages @Inject constructor( } /** OutgoingMessageRequest {2 request id, 3 thread id, 4 text, 7 participants}: OutgoingMessageResponse {1, 2 status, 3 error}. */ - private suspend fun send(request: Wire.Fields, reply: Int, mux: DataXConnection) { + private suspend fun send(request: Wire.Fields, reply: Int, mux: DataXConnection, chat: Boolean) { val requestId = request.bytes(2) val text = request.bytes(4).decodeToString() val thread = request.bytes(3).decodeToString() val to = request.repeatedBytes(7).map { it.decodeToString() }.filter { it.isNotBlank() }.ifEmpty { addressesOf(thread) } - // A conversation with its notification up answers through the Messages app, as RCS when it is one. - val viaApp = text.isNotBlank() && replies.has(thread) + // A conversation with its notification up answers through the Messages app, as RCS when it is one. A chat only does that. + val viaApp = text.isNotBlank() && (chat || replies.has(thread)) val id = requests.start("${if (viaApp) "messages" else "sms"}://send/${to.joinToString(",").ifEmpty { thread }}", SOURCE_SMS) val error = when { viaApp -> replies.reply(context, thread, text) + chat -> if (text.isBlank()) "Empty message" else "No reply in that conversation" !allowed(Manifest.permission.SEND_SMS) -> "SMS not allowed in Visor" to.isEmpty() -> "No recipient" to.size > 1 -> "Group texts are not supported" @@ -354,6 +356,7 @@ class PhoneMessages @Inject constructor( private const val MT_CHUNK = 32 private val SERVED = setOf(MT_LOAD_THREADS, MT_LOAD_MESSAGES, MT_OUTGOING) private const val NATIVE = 3L + private val CHAT = setOf(1L, 2L, 4L) private const val DEFAULT_THREADS = 10 private const val DEFAULT_MESSAGES = 25 private const val MAX_MESSAGES = 100 diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt b/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt index 670f4ff3..1c490894 100644 --- a/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt +++ b/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt @@ -238,10 +238,12 @@ class MirrorListener : NotificationListenerService(), Messaging { // A text names its sender by number and its thread as the SMS store does, as svc70 lists them. val number = if (native) person?.uri?.takeIf { it.startsWith("tel:") }?.let { Uri.decode(it.substringAfter("tel:")) } ?: numberOf(sender) else null val thread = number?.takeIf { !group }?.let { runCatching { Telephony.Threads.getOrCreateThreadId(this, it).toString() }.getOrNull() } - if (thread != null) replyAction(sbn.notification)?.let { textReplies.put(thread, sbn.key, it) } + // The thread the glasses answer with (FoA 2) names the notification's reply action, for every provider. + val threadId = thread ?: sbn.notification.shortcutId ?: sbn.key + replyAction(sbn.notification)?.let { textReplies.put(threadId, sbn.key, it) } return GlassesNotification( key = sbn.key, - pkg = if (native) NATIVE_APP else sbn.packageName, + pkg = APP_IDS[provider] ?: sbn.packageName, appName = appName, title = sender, text = text, @@ -250,7 +252,7 @@ class MirrorListener : NotificationListenerService(), Messaging { provider = provider, sender = sender, senderId = number ?: person?.key ?: person?.uri?.substringAfter(':').orEmpty(), - threadId = thread ?: sbn.notification.shortcutId ?: sbn.key, + threadId = threadId, isGroup = group, groupName = extras.getCharSequence(Notification.EXTRA_CONVERSATION_TITLE)?.toString().orEmpty(), ) @@ -324,8 +326,8 @@ class MirrorListener : NotificationListenerService(), Messaging { private const val PROVIDER_NATIVE = 3 private const val TYPE_INCOMING_MESSAGE = 1 - // The app id the glasses take for the phone's own texts. - private const val NATIVE_APP = "com.android.sms" + // The app id the glasses gate on, per provider, as the Meta app sends it. Messenger and Instagram are not their package names. + private val APP_IDS = mapOf(1 to "com.facebook.messenger", 2 to "com.whatsapp", 3 to "com.android.sms", 4 to "com.instagram") // The chat apps the glasses announce themselves, by their NotificationProvider. private val PROVIDERS = mapOf( diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/NotifyIcons.kt b/app/src/main/kotlin/com/macedonga/visor/notify/NotifyIcons.kt index b287b200..f9ba9a3e 100644 --- a/app/src/main/kotlin/com/macedonga/visor/notify/NotifyIcons.kt +++ b/app/src/main/kotlin/com/macedonga/visor/notify/NotifyIcons.kt @@ -40,14 +40,22 @@ class NotifyIcons @Inject constructor( /** The app's icon as a 78 x 78 JPEG, or null when the package has none. */ private fun render(pkg: String): ByteArray? = runCatching { - val d = context.packageManager.getApplicationIcon(pkg) + val d = runCatching { context.packageManager.getApplicationIcon(pkg) }.getOrNull() + ?: context.packageManager.getApplicationIcon(installed(pkg) ?: return null) val bmp = Bitmap.createBitmap(ICON, ICON, Bitmap.Config.ARGB_8888) d.setBounds(0, 0, ICON, ICON) d.draw(Canvas(bmp)) ByteArrayOutputStream().also { bmp.compress(Bitmap.CompressFormat.JPEG, 100, it) }.toByteArray() }.getOrNull() + // Chat notifications carry the app ids the glasses gate on, not the phone's packages. + private fun installed(appId: String): String? = when (appId) { + "com.android.sms" -> android.provider.Telephony.Sms.getDefaultSmsPackage(context) + else -> APP_PACKAGES[appId] + } + private companion object { + val APP_PACKAGES = mapOf("com.facebook.messenger" to "com.facebook.orca", "com.instagram" to "com.instagram.android") const val MT_RESPONSE = 2 const val MT_ICON_REQUEST = 9 const val MT_ICON_RESPONSE = 10 From 4a924b932cf7cdd1229d97fdccfa18d9c557d648 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:13:12 +0200 Subject: [PATCH 56/90] Display: calls placed and answered from the glasses. Visor serves the call control on svc70: a call the Display places on the phone's line (OUTGOING_CALL_REQUEST, provider 3) goes through Android's Telecom, and a WhatsApp call to a contact with a WhatsApp number opens that contact's WhatsApp call; other providers are refused with a result instead of silence (OUTGOING_CALL_RESPONSE, results from the Meta app's enum). A ringing call can be answered from the glasses (38/39), the call state is answered from the phone's line (3/9), the comms health check says whether the phone is online and Bluetooth is on (59/60), and the listeners get their acks (6/15, 13/14, 22/36). Phone access asks for calling, answering and the phone state. The call audio still rides the phone's HFP link. Not tried on the Display yet. --- app/src/main/AndroidManifest.xml | 3 + .../visor/connection/PhoneCallControl.kt | 223 ++++++++++++++++++ .../visor/connection/PhoneMessages.kt | 6 +- .../macedonga/visor/settings/PhoneAccess.kt | 4 + app/src/main/res/values-it/strings_calls.xml | 5 + app/src/main/res/values/strings_calls.xml | 5 + 6 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/PhoneCallControl.kt create mode 100644 app/src/main/res/values-it/strings_calls.xml create mode 100644 app/src/main/res/values/strings_calls.xml diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 292e5b06..f19dfd84 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -34,6 +34,9 @@ + + + diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneCallControl.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneCallControl.kt new file mode 100644 index 00000000..779879b6 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneCallControl.kt @@ -0,0 +1,223 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.app.ActivityManager +import android.bluetooth.BluetoothManager +import android.content.ContentUris +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.net.ConnectivityManager +import android.net.NetworkCapabilities +import android.net.Uri +import android.os.Bundle +import android.provider.ContactsContract +import android.telecom.TelecomManager +import android.telephony.TelephonyManager +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.phone.companionAssociated +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * svc70 call control, beside the texts: OUTGOING_CALL_REQUEST (4) by OUTGOING_CALL_RESPONSE (10) {1 request id, + * 2 result, 3 error detail}; CALL_STATE_REQUEST (3) by CALL_STATE_RESPONSE (9) {1 call states}; ANSWER_INCOMING_CALL_REQUEST + * (38) by its response (39) {1 request id, 2 success}; COMMS_HEALTH_REQUEST (59) by COMMS_HEALTH_RESPONSE (60). The other + * listeners get the ack they wait for: 6 by 15, 13 by 14, 22 by 36. Cellular calls (provider 3) go through Telecom, and + * WhatsApp calls open the contact's WhatsApp call row; any other provider is refused. + */ +@Singleton +class PhoneCallControl @Inject constructor( + @ApplicationContext private val context: Context, + private val requests: RequestLog, + private val log: Provider, +) { + suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed) return + val reply = DataXConnection.mirror(message.info.channelId) + val request = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return + when (message.info.messageType) { + MT_CALL_STATE_REQUEST -> callState(reply, connection) + MT_OUTGOING_CALL_REQUEST -> outgoing(request, reply, connection) + MT_ANSWER_INCOMING_CALL_REQUEST -> answer(request, reply, connection) + MT_COMMS_HEALTH_REQUEST -> commsHealth(request, reply, connection) + MT_CALL_ENGINE_SUBSCRIPTION -> send("calls://engine/subscribe", reply, connection, MT_CALL_ENGINE_SUBSCRIPTION_ACK, ByteArray(0), "call engine subscription acknowledged") + MT_OUTGOING_ERROR_LISTENER -> send("calls://errors/subscribe", reply, connection, MT_OUTGOING_ERROR_LISTENER_ACK, ByteArray(0), "call error listener acknowledged") + MT_CALLING_SVC_SUBSCRIBE -> send("calls://svc/subscribe", reply, connection, MT_CALLING_SVC_ACK, ByteArray(0), "calling service state acknowledged") + } + } + + /** CallState on the phone's own line: ringing in as RINGING_INBOUND, a call under way as ACTIVE, else none. */ + @Suppress("DEPRECATION") + private suspend fun callState(reply: Int, mux: DataXConnection) { + if (!allowed(Manifest.permission.READ_PHONE_STATE)) { + return send(STATE_URL, reply, mux, MT_CALL_STATE_RESPONSE, ByteArray(0), "phone state not allowed", failed = true) + } + val state = runCatching { context.getSystemService(TelephonyManager::class.java)?.callState }.getOrNull() + val entry = when (state) { + TelephonyManager.CALL_STATE_RINGING -> STATE_RINGING_INBOUND + TelephonyManager.CALL_STATE_OFFHOOK -> STATE_ACTIVE + else -> null + } + // CallState {1 call id, 2 provider, 3 state}, in GetCallStatesResponse {1}. + val body = if (entry == null) ByteArray(0) else + Wire.fieldBytes(1, Wire.fieldBytes(1, CELLULAR_CALL_ID.encodeToByteArray()) + Wire.fieldVarint(2, NATIVE) + Wire.fieldVarint(3, entry)) + send(STATE_URL, reply, mux, MT_CALL_STATE_RESPONSE, body, if (entry == null) "no call" else "call state $entry") + } + + /** OutgoingCallRequest {1 provider, 2 callee, 7 request id}. */ + private suspend fun outgoing(request: Wire.Fields, reply: Int, mux: DataXConnection) { + val provider = request.varint(1) + val callee = request.bytes(2).decodeToString() + val (result, error) = when (provider) { + NATIVE -> placeCall(callee) + WHATSAPP_PROVIDER -> startWhatsApp(callee) + else -> RESULT_UNKNOWN to "Calls on this provider are not supported" + } + val body = Wire.fieldBytes(1, request.bytes(7)) + Wire.fieldVarint(2, result) + + (error?.let { Wire.fieldBytes(3, it.encodeToByteArray()) } ?: ByteArray(0)) + send("calls://outgoing/$provider", reply, mux, MT_OUTGOING_CALL_RESPONSE, body, error ?: "call started", failed = result != RESULT_OK) + } + + /** Telecom takes the call; the call itself rides HFP to the glasses. */ + private fun placeCall(number: String): Pair { + if (!allowed(Manifest.permission.CALL_PHONE)) return ERROR_PERMISSION to "Call permission is not allowed in Visor" + if (number.isBlank()) return RESULT_UNKNOWN to "No number to call" + return try { + requireNotNull(context.getSystemService(TelecomManager::class.java)).placeCall(Uri.fromParts("tel", number, null), Bundle()) + RESULT_OK to null + } catch (e: Exception) { + RESULT_UNKNOWN to (e.message ?: e.toString()) + } + } + + /** Android has no call API for WhatsApp, so this opens the contact's WhatsApp call row, which starts the call. */ + private fun startWhatsApp(number: String): Pair { + if (!allowed(Manifest.permission.READ_CONTACTS)) return ERROR_PERMISSION to "Contacts permission is not allowed in Visor" + if (!startable()) return RESULT_UNKNOWN to "Visor cannot open WhatsApp from the background" + val row = whatsAppRow(number) ?: return RESULT_UNKNOWN to "No WhatsApp call for this number" + val intent = Intent(Intent.ACTION_VIEW, ContentUris.withAppendedId(ContactsContract.Data.CONTENT_URI, row)).setPackage(WHATSAPP_PACKAGE) + return try { + context.startActivity(intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)) + RESULT_OK to null + } catch (e: Exception) { + RESULT_UNKNOWN to (e.message ?: e.toString()) + } + } + + /** The data row holding the WhatsApp call of the contact that owns [number]. */ + private fun whatsAppRow(number: String): Long? { + if (number.isBlank()) return null + val contact = context.contentResolver.query( + Uri.withAppendedPath(ContactsContract.PhoneLookup.CONTENT_FILTER_URI, Uri.encode(number)), + arrayOf(ContactsContract.PhoneLookup._ID), null, null, null, + )?.use { if (it.moveToFirst()) it.getLong(0) else null } ?: return null + return context.contentResolver.query( + ContactsContract.Data.CONTENT_URI, + arrayOf(ContactsContract.Data._ID), + "${ContactsContract.Data.CONTACT_ID} = ? AND ${ContactsContract.Data.MIMETYPE} = ?", + arrayOf(contact.toString(), WHATSAPP_CALL), + null, + )?.use { if (it.moveToFirst()) it.getLong(0) else null } + } + + /** AnswerIncomingCallRequest {1 provider, 2 request id, 3 call id}: the ringing cellular call is taken. */ + @Suppress("DEPRECATION") + private suspend fun answer(request: Wire.Fields, reply: Int, mux: DataXConnection) { + val provider = request.varint(1) + val error = when { + provider != NATIVE -> "Answering on this provider is not supported" + !allowed(Manifest.permission.ANSWER_PHONE_CALLS) || !allowed(Manifest.permission.READ_PHONE_STATE) -> + "Call permission is not allowed in Visor" + !ringing() -> "No ringing call" + else -> try { + requireNotNull(context.getSystemService(TelecomManager::class.java)).acceptRingingCall() + null + } catch (e: Exception) { + e.message ?: e.toString() + } + } + val body = Wire.fieldBytes(1, request.bytes(2)) + Wire.fieldVarint(2, if (error == null) RESULT_OK else 0L) + send("calls://answer/$provider", reply, mux, MT_ANSWER_INCOMING_CALL_RESPONSE, body, error ?: "call answered", failed = error != null) + } + + /** CommsHealthResponse {1 request id, 2 has state, 4 network, 5 Bluetooth, 6 network time, 7 state time}. */ + private suspend fun commsHealth(request: Wire.Fields, reply: Int, mux: DataXConnection) { + val now = System.currentTimeMillis() + val network = if (networkUp()) SIGNAL_PASS else SIGNAL_FAIL + val bluetooth = if (bluetoothOn()) SIGNAL_PASS else SIGNAL_FAIL + val body = Wire.fieldBytes(1, request.bytes(1)) + Wire.fieldVarint(2, 1) + Wire.fieldVarint(4, network) + + Wire.fieldVarint(5, bluetooth) + Wire.fieldVarint(6, now) + Wire.fieldVarint(7, now) + val outcome = "network ${signal(network)}, bluetooth ${signal(bluetooth)}" + send("calls://health", reply, mux, MT_COMMS_HEALTH_RESPONSE, body, outcome, failed = network == SIGNAL_FAIL || bluetooth == SIGNAL_FAIL) + } + + @Suppress("DEPRECATION") + private fun ringing(): Boolean = runCatching { + context.getSystemService(TelephonyManager::class.java)?.callState == TelephonyManager.CALL_STATE_RINGING + }.getOrDefault(false) + + private fun networkUp(): Boolean { + val manager = context.getSystemService(ConnectivityManager::class.java) ?: return false + val caps = manager.getNetworkCapabilities(manager.activeNetwork) ?: return false + return caps.hasCapability(NetworkCapabilities.NET_CAPABILITY_VALIDATED) + } + + private fun bluetoothOn(): Boolean = context.getSystemService(BluetoothManager::class.java)?.adapter?.isEnabled == true + + /** Android lets an app start an activity from the screen or as the glasses' companion, as PhoneMedia does. */ + private fun startable(): Boolean = onScreen() || companionAssociated(context) + + private fun onScreen(): Boolean = ActivityManager.RunningAppProcessInfo() + .also(ActivityManager::getMyMemoryState).importance == ActivityManager.RunningAppProcessInfo.IMPORTANCE_FOREGROUND + + private fun allowed(permission: String) = context.checkSelfPermission(permission) == PackageManager.PERMISSION_GRANTED + + private suspend fun send(url: String, reply: Int, mux: DataXConnection, type: Int, body: ByteArray, outcome: String, failed: Boolean = false) { + val id = requests.start(url, SOURCE_CALLS) + // The glasses may have closed the channel by now; the answer is theirs to miss. + runCatching { mux.send(reply, body, messageType = type) } + requests.finish(id, outcome, failed, bytes = body.size.toLong()) + log.get().add("Calls: $outcome") + } + + private fun signal(value: Long) = if (value == SIGNAL_PASS) "pass" else "fail" + + companion object { + /** The types this class answers. PhoneMessages hands them here before its own checks. */ + val TYPES = setOf(3, 4, 6, 13, 22, 38, 59) + private const val MT_CALL_STATE_REQUEST = 3 + private const val MT_OUTGOING_CALL_REQUEST = 4 + private const val MT_CALL_ENGINE_SUBSCRIPTION = 6 + private const val MT_CALL_STATE_RESPONSE = 9 + private const val MT_OUTGOING_CALL_RESPONSE = 10 + private const val MT_OUTGOING_ERROR_LISTENER = 13 + private const val MT_OUTGOING_ERROR_LISTENER_ACK = 14 + private const val MT_CALL_ENGINE_SUBSCRIPTION_ACK = 15 + private const val MT_ANSWER_INCOMING_CALL_REQUEST = 38 + private const val MT_ANSWER_INCOMING_CALL_RESPONSE = 39 + private const val MT_CALLING_SVC_SUBSCRIBE = 22 + private const val MT_CALLING_SVC_ACK = 36 + private const val MT_COMMS_HEALTH_REQUEST = 59 + private const val MT_COMMS_HEALTH_RESPONSE = 60 + private const val RESULT_OK = 1L + private const val ERROR_PERMISSION = 4L + private const val RESULT_UNKNOWN = 12L + private const val WHATSAPP_PACKAGE = "com.whatsapp" + private const val WHATSAPP_CALL = "vnd.android.cursor.item/vnd.com.whatsapp.voip.call" + private const val NATIVE = 3L + private const val WHATSAPP_PROVIDER = 2L + private const val STATE_RINGING_INBOUND = 1L + private const val STATE_ACTIVE = 3L + private const val CELLULAR_CALL_ID = "cellular" + private const val STATE_URL = "calls://state" + private const val SIGNAL_PASS = 1L + private const val SIGNAL_FAIL = 2L + private const val SOURCE_CALLS = -5 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt index cedc9005..1561283c 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt @@ -45,7 +45,8 @@ import kotlinx.coroutines.withTimeoutOrNull * is one; any other goes as SMS, answered once the radio has sent it. Each request comes on a channel of its own and * is answered with its own type; a reply over 1000 bytes goes as type-32 chunks when the request allows them. Texts * that arrive reach the glasses as native notifications, from the SMS app's own notifications (see MirrorListener). - * The Phone app's recent calls (type 11) come on this service too; [PhoneCalls] answers them. + * The Phone app's recent calls (type 11) come on this service too; [PhoneCalls] answers them, and the call control + * (placing, answering) is [PhoneCallControl]'s. * * The RCS chats Google Messages files as MMS are marked restricted, and Android shows those rows only to the default * SMS app: a chat with nothing else in it is missing from the list here. @@ -57,10 +58,13 @@ class PhoneMessages @Inject constructor( private val log: Provider, private val replies: TextReplies, private val calls: PhoneCalls, + private val control: PhoneCallControl, ) : Responder { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Messages: failed: $e") }) override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + // Call control has types with empty payloads, so it is checked before the text's. + if (message.info.messageType in PhoneCallControl.TYPES) return control.respond(message, connection) if (message.payload.isEmpty()) return val request = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return val type = message.info.messageType diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt b/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt index c414a8fd..e8b72966 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt @@ -48,6 +48,7 @@ enum class Need(val title: Int, val why: Int) { CONTACTS(R.string.access_contacts, R.string.access_contacts_why), SMS(R.string.access_sms, R.string.access_sms_why), CALLS(R.string.access_calls, R.string.access_calls_why), + CALL_CONTROL(R.string.access_call_control, R.string.access_call_control_why), LOCATION(R.string.access_location, R.string.access_location_why), /** Location while Visor is in the background: Android asks it on its own, once [LOCATION] is given. */ @@ -66,6 +67,8 @@ enum class Need(val title: Int, val why: Int) { CONTACTS -> context.allows(Manifest.permission.READ_CONTACTS) SMS -> context.allows(Manifest.permission.READ_SMS) && context.allows(Manifest.permission.SEND_SMS) CALLS -> context.allows(Manifest.permission.READ_CALL_LOG) + CALL_CONTROL -> context.allows(Manifest.permission.CALL_PHONE) && context.allows(Manifest.permission.ANSWER_PHONE_CALLS) && + context.allows(Manifest.permission.READ_PHONE_STATE) LOCATION -> context.allows(Manifest.permission.ACCESS_FINE_LOCATION) LOCATION_ALWAYS -> context.allows(Manifest.permission.ACCESS_BACKGROUND_LOCATION) WIFI -> wifiDirectAllowed(context) @@ -81,6 +84,7 @@ enum class Need(val title: Int, val why: Int) { CONTACTS -> arrayOf(Manifest.permission.READ_CONTACTS) SMS -> arrayOf(Manifest.permission.READ_SMS, Manifest.permission.SEND_SMS) CALLS -> arrayOf(Manifest.permission.READ_CALL_LOG) + CALL_CONTROL -> arrayOf(Manifest.permission.CALL_PHONE, Manifest.permission.ANSWER_PHONE_CALLS, Manifest.permission.READ_PHONE_STATE) LOCATION -> arrayOf(Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_COARSE_LOCATION) // Android grants it only after the foreground one, so that one comes first when it is missing. LOCATION_ALWAYS -> if (context.allows(Manifest.permission.ACCESS_FINE_LOCATION)) arrayOf(Manifest.permission.ACCESS_BACKGROUND_LOCATION) diff --git a/app/src/main/res/values-it/strings_calls.xml b/app/src/main/res/values-it/strings_calls.xml new file mode 100644 index 00000000..bc1fef4a --- /dev/null +++ b/app/src/main/res/values-it/strings_calls.xml @@ -0,0 +1,5 @@ + + + Effettua e rispondi alle chiamate + Così gli occhiali possono effettuare e rispondere alle chiamate su questo telefono. + diff --git a/app/src/main/res/values/strings_calls.xml b/app/src/main/res/values/strings_calls.xml new file mode 100644 index 00000000..173f0b7c --- /dev/null +++ b/app/src/main/res/values/strings_calls.xml @@ -0,0 +1,5 @@ + + + Place and answer calls + So the glasses can place and answer calls on this phone. + From 62c7d200080925c956c3edc08e5b6daf99ad6bf0 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:21:52 +0200 Subject: [PATCH 57/90] Neural Band: firmware updates from Meta. The band's settings get a Firmware row with the version the band reports (its OTA device info, now read when the band links), and a sheet that asks Meta (FetchSystemUpdates for ota.ceres.user and the band's serial) for a newer image and installs it on confirmation: the zip is downloaded to the cache and checked, the band is held from the Display for the whole update, an update in progress is aborted first, the band's key is installed when the image needs another (FetchEncryptedKEKInfoByKEKId), then each item (the firmware, and the model when newer) goes StartItemUpdate, bulk transfer in 950-byte chunks within the band's window with acks and retries, and VerifyItemUpdate, then one CommitUpdate. Any failure aborts the update. The band's battery must be at least 10 %. The answer keys of Meta's query are read from the Meta app's field ids and are the least certain part. Not tried on the band yet. --- .../com/macedonga/visor/band/BandLink.kt | 32 +- .../com/macedonga/visor/band/BandUpdate.kt | 364 ++++++++++++++++++ .../com/macedonga/visor/settings/BandPart.kt | 51 +++ .../visor/settings/SettingsScreen.kt | 1 + .../res/values-it/strings_bandfirmware.xml | 16 + .../main/res/values/strings_bandfirmware.xml | 16 + .../visor/core/identity/firmware/BandOffer.kt | 96 +++++ .../core/identity/language/LanguagePacks.kt | 5 + .../com/macedonga/visor/core/protocol/Band.kt | 99 +++++ 9 files changed, 679 insertions(+), 1 deletion(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/band/BandUpdate.kt create mode 100644 app/src/main/res/values-it/strings_bandfirmware.xml create mode 100644 app/src/main/res/values/strings_bandfirmware.xml create mode 100644 core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/BandOffer.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt index 099ce92d..c851f5cc 100644 --- a/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt @@ -30,7 +30,9 @@ import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.delay import kotlinx.coroutines.withTimeoutOrNull +import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharedFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.combine @@ -55,6 +57,8 @@ data class BandState( val protocol: Int? = null, // A lesson has the band's gestures coming to the phone. val gestures: Boolean = false, + // The firmware's version, from the band's OTA device info. + val firmware: Long? = null, ) /** @@ -80,6 +84,10 @@ class BandLink @Inject constructor( private val held = MutableStateFlow(true) private val lesson = MutableStateFlow(false) private val borrowing = MutableStateFlow(false) + private val updating = MutableStateFlow(false) + private val _answers = MutableSharedFlow(extraBufferCapacity = 64) + /** Every RPC answer while the band is up, raw: a firmware update reads its own answers here. */ + val answers: SharedFlow = _answers private val _gestures = kotlinx.coroutines.flow.MutableSharedFlow(extraBufferCapacity = 32) val gestures: kotlinx.coroutines.flow.SharedFlow = _gestures @@ -97,7 +105,7 @@ class BandLink @Inject constructor( scope.launch { // A lesson or an errand takes the band from the Display; when it ends, the band goes back. Never otherwise: // the Display reaches the band on its own, and a phone holding it while the Display restarts keeps it off. - val stepBack = combine(lesson, borrowing) { learning, errand -> !learning && !errand } + val stepBack = combine(lesson, borrowing, updating) { learning, errand, update -> !learning && !errand && !update } combine(connection.running, connection.pair, bands.all, stepBack, held) { running, pair, _, away, hold -> bands.forGlasses(pair)?.takeIf { running && hold && !away } }.distinctUntilChanged().collect { band -> gate.withLock { switchTo(band) } } @@ -120,6 +128,23 @@ class BandLink @Inject constructor( held.value = true } + /** Holds the band for [block], which needs it for minutes, then gives it back. */ + suspend fun holdFor(block: suspend () -> T): T { + updating.value = true + try { + if (withTimeoutOrNull(BORROW_MS) { _state.first { it.link == LinkState.READY } } == null) error("the band is not reached") + return block() + } finally { + updating.value = false + } + } + + /** The next seq for a request built by hand. */ + fun nextSeq(): Int = seq.getAndIncrement() + + /** Sends RPC requests on the band now; false when the band is not up. */ + fun sendRaw(requests: List): Boolean = send { requests } + /** Have the band's gestures come here: the band is taken from the Display for as long as it lasts. */ fun startLesson() { if (lesson.value) return @@ -267,6 +292,7 @@ class BandLink @Inject constructor( SessionCommand { s -> val mux = s.connection ?: return@SessionCommand if (info) mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.deviceInfoRequest(seq.getAndIncrement())) + if (info) mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.otaDeviceInfo(seq.getAndIncrement())) mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.batteryRequest(seq.getAndIncrement())) mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.configRequest(seq.getAndIncrement())) }, @@ -275,6 +301,10 @@ class BandLink @Inject constructor( private fun onBand(event: ServiceData) { val type = event.messageType and 0xFF if (Band.isGesture(type)) return onGesture(type, event.payload) + if (type == Band.RESPONSE) { + _answers.tryEmit(event.payload) + runCatching { Band.otaInfo(event.payload) }.getOrNull()?.let { info -> _state.update { it.copy(firmware = info.firmware) } } + } val battery = runCatching { when (type) { Band.RESPONSE -> Band.response(event.payload).also { r -> diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandUpdate.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandUpdate.kt new file mode 100644 index 00000000..e5cdb24a --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandUpdate.kt @@ -0,0 +1,364 @@ +package com.macedonga.visor.band + +import android.content.Context +import android.util.Base64 +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.firmware.BandOffer +import com.macedonga.visor.core.identity.firmware.fetchBandKek +import com.macedonga.visor.core.identity.firmware.fetchBandOffer +import com.macedonga.visor.core.identity.token.Token +import com.macedonga.visor.core.protocol.Band +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.login.MetaLogin +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.File +import java.io.FileOutputStream +import java.net.HttpURLConnection +import java.net.URL +import java.security.MessageDigest +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull +import org.json.JSONObject + +sealed interface BandWork { + data object Idle : BandWork + data object Checking : BandWork + /** [percent] is null until the band acks the first chunk. */ + data class Installing(val percent: Int? = null) : BandWork + data object Installed : BandWork + data class Failed(val reason: String) : BandWork +} + +/** + * The Neural Band's firmware from Meta. [check] asks Meta for the newest image; [install] downloads it with the user's + * login, checks it against its manifest, and sends it over the band's RPC: Ota on RpcRequest 7 (answers 8) and the bulk + * file transfer on 9 (answers 10). The band is held for the whole update, and any failure aborts it. + */ +@Singleton +class BandUpdate @Inject constructor( + @ApplicationContext private val context: Context, + private val login: MetaLogin, + private val connection: GlassesConnection, + private val bands: Bands, + private val link: BandLink, + private val log: EventLog, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val _checked = MutableStateFlow(false) + /** Meta answered for the band's current build, offer or none. */ + val checked: StateFlow = _checked.asStateFlow() + private val _offer = MutableStateFlow(null) + val offer: StateFlow = _offer.asStateFlow() + private val _work = MutableStateFlow(BandWork.Idle) + val work: StateFlow = _work.asStateFlow() + private val transfers = AtomicInteger(1) + private var job: Job? = null + + init { + // An offer is for one band's build: another pair starts over. + scope.launch { + connection.pair.filterNotNull().distinctUntilChanged().drop(1).collect { + _offer.value = null + _checked.value = false + } + } + } + + fun check() { + if (job?.isActive == true) return + job = scope.launch { + _work.value = BandWork.Checking + try { + val serial = serial() + val firmware = link.holdFor { withSession { deviceInfo().firmware } } + // The Meta app never sends a version below 1. + val found = fetchBandOffer(login.source.get(), serial, firmware.coerceAtLeast(1).toString()) + _offer.value = found + _checked.value = true + _work.value = BandWork.Idle + log.add("Band firmware: " + (found?.let { "${it.targetVersion} is offered" } ?: "no newer build is offered")) + } catch (error: kotlinx.coroutines.CancellationException) { + _work.value = BandWork.Idle + throw error + } catch (error: Throwable) { + log.add("Band firmware: check failed: ${error.message}") + _work.value = BandWork.Failed(reason(error)) + } + } + } + + fun install() { + val offer = _offer.value ?: return + if (job?.isActive == true) return + job = scope.launch { + val zip = File(File(context.cacheDir, "band-update").apply { mkdirs() }, "band.zip") + try { + val serial = serial() + val token = login.source.get() + download(offer, zip) + val blobs = unpack(zip) + _work.value = BandWork.Installing() + log.add("Band firmware: installing ${offer.targetVersion}") + link.holdFor { installOn(offer, serial, token, blobs) } + _offer.value = null + _work.value = BandWork.Installed + log.add("Band firmware: ${offer.targetVersion} is installed") + } catch (error: kotlinx.coroutines.CancellationException) { + _work.value = BandWork.Idle + throw error + } catch (error: Throwable) { + log.add("Band firmware: install failed: ${error.message}") + _work.value = BandWork.Failed(reason(error)) + } finally { + zip.delete() + } + } + } + + fun cancel() { + job?.cancel() + } + + /** Back to the start after an install ended. */ + fun dismiss() { + if (job?.isActive != true) _work.value = BandWork.Idle + } + + private fun serial(): String = bands.forGlasses(connection.pair.value)?.serial ?: error("no band is set up with these glasses") + + private suspend fun installOn(offer: BandOffer, serial: String, token: Token, blobs: List) = withSession { + val target = offer.targetVersion.toLongOrNull() ?: error("the offer's version is not a number") + var begun = false + try { + if (battery() < MIN_BATTERY) error("the band's battery is below $MIN_BATTERY%") + var info = deviceInfo() + if (info.unavailable != Band.OTA_NO_REASON) error("the band cannot update now (reason ${info.unavailable})") + if (info.state == Band.OTA_IN_PROGRESS) { + fire(Band.otaAbort(link.nextSeq())) + delay(ABORT_MS) + info = deviceInfo() + if (info.state == Band.OTA_IN_PROGRESS) error("the band is still in an update") + } + if (info.state == Band.OTA_UNAVAILABLE) error("the band is not ready for an update") + + val items = ArrayList() + if (info.firmware != target) { + val firmware = blobs.firstOrNull { it.manifestType == MANIFEST_FIRMWARE } ?: error("the zip has no firmware") + items += Item(Band.OTA_ITEM_FIRMWARE, firmware) + } + blobs.firstOrNull { it.manifestType == MANIFEST_ML }?.let { ml -> + if (ml.bytes.size >= MIN_ML_BYTES && target > info.ml) items += Item(Band.OTA_ITEM_ML_MODEL, ml) + } + if (items.isEmpty()) { + log.add("Band firmware: already on $target") + } else { + offer.kekId?.takeIf { it != info.kek }?.let { kekId -> + val wrapped = fetchBandKek(token, serial, kekId) + if (wrapped.algorithm != null && wrapped.algorithm != "AES_128") log.add("Band firmware: the key names ${wrapped.algorithm}") + begun = true + val answer = call { Band.otaInstallKek(it, kekId, Base64.decode(wrapped.encrypted, Base64.NO_WRAP)) } + expectOk(answer, "its key") + } + val total = items.sumOf { it.blob.bytes.size.toLong() } + val done = AtomicLong() + for (item in items) { + begun = true + val size = item.blob.bytes.size.toLong() + expectOk(call { Band.otaStartItem(it, item.type, target, size) }, "the update") + transfer(item.blob) { n -> + val now = done.addAndGet(n.toLong()) + _work.value = BandWork.Installing((now * 100 / total).toInt()) + } + expectOk(call(VERIFY_MS) { Band.otaVerifyItem(it, item.type, target) }, "the image") + } + expectOk(call(VERIFY_MS) { Band.otaCommit(it) }, "the commit") + } + } catch (error: Throwable) { + if (begun) fire(Band.otaAbort(link.nextSeq())) + throw error + } + } + + /** One image over the bulk transfer: the info request, then 950-byte chunks in the band's window. */ + private suspend fun Session.transfer(blob: Blob, sent: (Int) -> Unit) { + val id = transfers.getAndIncrement() + val size = blob.bytes.size.toLong() + val info = Band.bulk(call { Band.bulkInfo(it, blob.md5, size, id) }) as? Band.Bulk.Info + ?: error("the band did not answer the transfer") + if (info.transferId != id) error("the band answered another transfer") + when (info.state) { + Band.TRANSFER_FULL -> return + Band.TRANSFER_NEW, Band.TRANSFER_PARTIAL -> Unit + else -> error("the band would not take the file (state ${info.state})") + } + val window = (info.memory / Band.CHUNK_BYTES).toInt().coerceAtLeast(1) + sendChunks(id, blob.bytes, info.resume.toInt(), window, sent) + } + + private suspend fun Session.sendChunks(id: Int, bytes: ByteArray, from: Int, window: Int, sent: (Int) -> Unit) = coroutineScope { + val slots = Semaphore(window) + var offset = from + while (offset < bytes.size) { + slots.acquire() + val at = offset + val end = minOf(bytes.size, at + Band.CHUNK_BYTES) + val piece = bytes.copyOfRange(at, end) + launch { + try { + chunk(id, at, piece) + sent(piece.size) + } finally { + slots.release() + } + } + offset = end + } + } + + /** Sends one chunk and waits for its ack; unanswered, it goes again up to three times. */ + private suspend fun Session.chunk(id: Int, at: Int, piece: ByteArray) { + repeat(CHUNK_SENDS) { + val key = id to at + val ack = CompletableDeferred() + acks[key] = ack + try { + if (!link.sendRaw(listOf(Band.bulkChunk(link.nextSeq(), id, at.toLong(), piece)))) error("the band is not up") + if (withTimeoutOrNull(CHUNK_MS) { ack.await() } != null) return + } finally { + acks.remove(key) + } + } + error("the band did not ack the chunk at $at") + } + + /** The answers of one held update: each request's by seq, each chunk's ack by its transfer and offset. */ + private inner class Session { + private val replies = ConcurrentHashMap>() + val acks = ConcurrentHashMap, CompletableDeferred>() + + /** Sends one request and waits for its answer. */ + suspend fun call(timeout: Long = REPLY_MS, build: (Int) -> ByteArray): ByteArray { + val n = link.nextSeq() + val answer = CompletableDeferred() + replies[n] = answer + try { + if (!link.sendRaw(listOf(build(n)))) error("the band is not up") + return withTimeoutOrNull(timeout) { answer.await() } ?: error("the band did not answer") + } finally { + replies.remove(n) + } + } + + fun fire(request: ByteArray) { + link.sendRaw(listOf(request)) + } + + fun route(payload: ByteArray) { + when (val bulk = runCatching { Band.bulk(payload) }.getOrNull()) { + is Band.Bulk.Ack -> acks.remove(bulk.transferId to bulk.offset.toInt())?.complete(Unit) + is Band.Bulk.Nack -> { + val refused = IllegalStateException("the band refused a chunk: ${bulk.reason.ifEmpty { "no reason" }}") + acks.values.forEach { it.completeExceptionally(refused) } + acks.clear() + } + else -> runCatching { Band.seqOf(payload) }.getOrNull()?.let { replies.remove(it)?.complete(payload) } + } + } + } + + private suspend fun withSession(block: suspend Session.() -> T): T = coroutineScope { + val session = Session() + val router = launch(start = CoroutineStart.UNDISPATCHED) { link.answers.collect { session.route(it) } } + try { + session.block() + } finally { + router.cancel() + } + } + + private suspend fun Session.deviceInfo(): Band.OtaInfo = + Band.otaInfo(call { Band.otaDeviceInfo(it) }) ?: error("the band gave no device info") + + private suspend fun Session.battery(): Int = + Band.response(call { Band.batteryRequest(it) }).battery?.percent ?: error("the band gave no battery") + + private fun Session.expectOk(answer: ByteArray, what: String) { + val code = Band.otaResult(answer) ?: error("the band gave no answer for $what") + if (code != Band.OTA_RESULT_OK) error("the band refused $what (code $code)") + } + + private suspend fun download(offer: BandOffer, file: File) = withContext(Dispatchers.IO) { + val c = URL(offer.url).openConnection() as HttpURLConnection + try { + c.connectTimeout = 30_000 + c.readTimeout = 60_000 + if (c.responseCode != 200) error("the download answered ${c.responseCode}") + c.inputStream.use { input -> FileOutputStream(file).use { input.copyTo(it) } } + } finally { + c.disconnect() + } + if (offer.checksum.isNotEmpty() && sha256(file.readBytes()) != offer.checksum.lowercase()) error("the download did not match Meta's checksum") + } + + /** The zip's images, each checked against the SHA-256 in its manifest. They stay encrypted; the band decrypts them. */ + private fun unpack(zip: File): List = ZipFile(zip).use { z -> + val manifest = JSONObject(z.read(z.getEntry("manifest.json") ?: error("the zip has no manifest")).decodeToString()) + val items = manifest.getJSONArray("items") + (0 until items.length()).map { i -> + val item = items.getJSONObject(i) + val path = item.getString("path") + val bytes = z.read(z.getEntry(path) ?: error("the zip has no $path")) + if (sha256(bytes) != item.getString("digest").lowercase()) error("$path does not match its digest") + Blob(item.getInt("item_type"), bytes, MessageDigest.getInstance("MD5").digest(bytes)) + } + } + + private fun ZipFile.read(entry: ZipEntry): ByteArray = getInputStream(entry).use { it.readBytes() } + + private fun sha256(bytes: ByteArray): String = MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) } + + private fun reason(error: Throwable): String = error.message ?: error::class.java.simpleName + + /** An image from the zip; [manifestType] is its item type in the manifest (1 firmware, 2 ML model). */ + private class Blob(val manifestType: Int, val bytes: ByteArray, val md5: ByteArray) + + /** An image as the band's item: [type] is the wire's item type (0 firmware, 2 ML model). */ + private class Item(val type: Int, val blob: Blob) + + private companion object { + const val MANIFEST_FIRMWARE = 1 + const val MANIFEST_ML = 2 + const val MIN_BATTERY = 10 + const val MIN_ML_BYTES = 10240 + // A chunk is sent once and at most three more times when unanswered. + const val CHUNK_SENDS = 4 + // Guesses: nothing in the Meta app's code sets these times. + const val CHUNK_MS = 3_000L + const val REPLY_MS = 5_000L + const val VERIFY_MS = 60_000L + const val ABORT_MS = 2_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt index 9d83e69b..155af1b8 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt @@ -1,15 +1,23 @@ package com.macedonga.visor.settings +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.padding import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.collectAsStateWithLifecycle import com.macedonga.visor.R import com.macedonga.visor.band.BandSetup +import com.macedonga.visor.band.BandUpdate +import com.macedonga.visor.band.BandWork import com.macedonga.visor.core.session.LinkState import kotlinx.coroutines.launch @@ -29,6 +37,7 @@ internal fun BandPart(model: SettingsViewModel) { val problem = (setup as? BandSetup.Status.Failed)?.let { failWords(it) } val scope = rememberCoroutineScope() var refused by remember { mutableStateOf(false) } + var firmwareOpen by remember { mutableStateOf(false) } @Composable fun SetUpRow(title: Int) = ValueRow( stringResource(title), @@ -43,6 +52,7 @@ internal fun BandPart(model: SettingsViewModel) { } Rows( { ValueRow(stringResource(R.string.band_battery), battery?.let { batteryWords(it, charging) }, onClick = null) }, + { ValueRow(stringResource(R.string.band_firmware), band.firmware?.toString() ?: stringResource(R.string.settings_unknown), onClick = { firmwareOpen = true }) }, fitWords(device?.bandFit)?.let { fit -> row { ValueRow( @@ -68,6 +78,47 @@ internal fun BandPart(model: SettingsViewModel) { }, { SetUpRow(R.string.band_set_up_again) }, ) + if (firmwareOpen) BandFirmwareSheet(model.bandUpdate, band.firmware) { firmwareOpen = false } +} + +/** The band's firmware: its version, a check for a newer build and the install, as the glasses' sheet does it. */ +@Composable +private fun BandFirmwareSheet(update: BandUpdate, version: Long?, onClose: () -> Unit) { + val checked by update.checked.collectAsStateWithLifecycle() + val offer by update.offer.collectAsStateWithLifecycle() + val work by update.work.collectAsStateWithLifecycle() + var confirming by remember { mutableStateOf(false) } + val busy = work == BandWork.Checking || work is BandWork.Installing + VisorSheet({ onClose(); confirming = false; update.dismiss() }) { + SheetTitle(stringResource(R.string.band_firmware_title)) + val found = offer + Rows( + { ValueRow(stringResource(R.string.band_firmware_current), version?.toString() ?: stringResource(R.string.settings_unknown), onClick = null) }, + found?.let { o -> @Composable { ValueRow(stringResource(R.string.band_firmware_update), o.displayName.ifEmpty { o.targetVersion }, onClick = null) } }, + ) + val status: String? = when (val w = work) { + is BandWork.Failed -> stringResource(R.string.band_firmware_failed, w.reason) + BandWork.Installed -> stringResource(R.string.band_firmware_installed) + is BandWork.Installing -> w.percent?.let { stringResource(R.string.band_firmware_progress, it) } ?: stringResource(R.string.band_firmware_installing) + BandWork.Checking -> stringResource(R.string.band_firmware_checking) + BandWork.Idle -> if (checked && found == null) stringResource(R.string.band_firmware_up_to_date) else null + } + Column(Modifier.padding(horizontal = SIDE, vertical = 16.dp)) { + if (status != null) Footnote(status) + if (confirming && found != null) { + Row(Modifier.padding(top = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp)) { + ActionPill(stringResource(R.string.band_firmware_install), { confirming = false; update.install() }) + ActionPill(stringResource(R.string.language_cancel), { confirming = false }, quiet = true) + } + Footnote(stringResource(R.string.band_firmware_confirm, found.displayName.ifEmpty { found.targetVersion })) + } else { + Row(Modifier.padding(top = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp)) { + ActionPill(stringResource(R.string.band_firmware_check), { update.check() }, quiet = true, enabled = !busy) + if (found != null) ActionPill(stringResource(R.string.band_firmware_install), { confirming = true }, enabled = !busy) + } + } + } + } } @Composable diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt index 6e86dc83..6eb7356a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt @@ -177,6 +177,7 @@ class SettingsViewModel @Inject constructor( val band: com.macedonga.visor.band.BandLink, val bands: com.macedonga.visor.band.Bands, val bandSetup: com.macedonga.visor.band.BandSetup, + val bandUpdate: com.macedonga.visor.band.BandUpdate, val developer: com.macedonga.visor.outside.DeveloperMode, ) : ViewModel() diff --git a/app/src/main/res/values-it/strings_bandfirmware.xml b/app/src/main/res/values-it/strings_bandfirmware.xml new file mode 100644 index 00000000..dd2f8119 --- /dev/null +++ b/app/src/main/res/values-it/strings_bandfirmware.xml @@ -0,0 +1,16 @@ + + + Firmware + Firmware del braccialetto + Versione + Aggiornamento + Cerca aggiornamenti + Ricerca + Aggiornato + Installa + Installare %1$s sul braccialetto? + Installazione + Installazione, %1$d%% + Installato + Errore: %1$s + diff --git a/app/src/main/res/values/strings_bandfirmware.xml b/app/src/main/res/values/strings_bandfirmware.xml new file mode 100644 index 00000000..9b7eaca8 --- /dev/null +++ b/app/src/main/res/values/strings_bandfirmware.xml @@ -0,0 +1,16 @@ + + + Firmware + Band firmware + Version + Update + Check for update + Checking + Up to date + Install + Install %1$s on the band? + Installing + Installing, %1$d%% + Installed + Failed: %1$s + diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/BandOffer.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/BandOffer.kt new file mode 100644 index 00000000..af0afb0b --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/BandOffer.kt @@ -0,0 +1,96 @@ +package com.macedonga.visor.core.identity.firmware + +import com.macedonga.visor.core.identity.HttpPost +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.language.graphPost +import com.macedonga.visor.core.identity.token.Token +import com.macedonga.visor.core.identity.token.TokenError +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.longOrNull +import kotlinx.serialization.json.put + +/** A Neural Band firmware image Meta offers: the zip's address, size, SHA-256 and the KEK it is wrapped with. */ +data class BandOffer( + val baseVersion: String, + val targetVersion: String, + val displayName: String, + val url: String, + val size: Long, + val checksum: String, + val kekId: Long?, +) + +/** The band's wrapped KEK as Meta gives it, base64, and the algorithm it names. */ +data class BandKek(val encrypted: String, val algorithm: String?) + +private const val SYSTEM_UPDATES_DOC = "29744781653741296994778085945" +private const val KEK_DOC = "313699925716490204858433749875" +private const val BAND_DEVICE_TYPE = "ota.ceres.user" + +/** + * FetchSystemUpdates, the Meta app's own query for the band's firmware: the full image newer than [firmware] (the + * band's version as a decimal string), or null when there is none. [token] is the AR token from the login. + */ +fun fetchBandOffer( + token: Token, + serial: String, + firmware: String, + http: HttpPost = UrlConnectionPost, +): BandOffer? { + val variables = buildJsonObject { + put("fetch_request", buildJsonObject { + put("topology_fetch_update_info", JsonArray(listOf(buildJsonObject { + put("custom_ota_fetch_type", "CERES_USER_INSIGHTS_NO_DEVICE_IDENTITY") + put("full_update_only", true) + put("device_serial_number", serial) + put("device_type", BAND_DEVICE_TYPE) + put("fw_version", firmware) + }))) + }) + } + val (status, json) = graphPost(token, "FetchSystemUpdates", SYSTEM_UPDATES_DOC, variables, http) + val root = json["data"]?.jsonObject?.get("xar_fetch_constellation_system_updates") as? JsonObject + ?: throw TokenError("Meta answered $status without updates") + val answers = (root["fetch_update_info_list"] as? JsonArray) ?: emptyList() + val offer = answers.firstNotNullOfOrNull { (it as? JsonObject)?.get("fetch_update_response") as? JsonObject } ?: return null + val url = offer.text("file_uri") ?: throw TokenError("Meta's band update has no file") + return BandOffer( + baseVersion = offer.text("base_version").orEmpty(), + targetVersion = offer.text("target_version").orEmpty(), + displayName = offer.text("build_number_display_name").orEmpty(), + url = url, + size = offer.long("file_size") ?: 0, + checksum = offer.text("file_checksum").orEmpty(), + kekId = offer.long("kek_id"), + ) +} + +/** FetchEncryptedKEKInfoByKEKId: the band's KEK for [kekId], wrapped for the band with [serial]. */ +fun fetchBandKek( + token: Token, + serial: String, + kekId: Long, + http: HttpPost = UrlConnectionPost, +): BandKek { + val variables = buildJsonObject { + put("request", buildJsonObject { + put("device_serial_number", serial) + put("kek_id", kekId) + }) + } + val (status, json) = graphPost(token, "FetchEncryptedKEKInfoByKEKId", KEK_DOC, variables, http) + val node = json["data"]?.jsonObject?.get("xar_fetch_encrypted_kek_info_by_kek_id") as? JsonObject + ?: throw TokenError("Meta answered $status without a key") + val encrypted = node.text("encrypted_kek") ?: throw TokenError("Meta's key has no value") + return BandKek(encrypted, node.text("encryption_algo")) +} + +private fun JsonObject.text(key: String): String? = (this[key] as? JsonPrimitive)?.contentOrNull + +private fun JsonObject.long(key: String): Long? = + (this[key] as? JsonPrimitive)?.let { it.longOrNull ?: it.contentOrNull?.toLongOrNull() } diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt index 63fd8350..0cdd9542 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt @@ -99,6 +99,11 @@ internal fun constellationPost(token: Token, friendlyName: String, docId: String val variables = buildJsonObject { put("fetch_request", buildJsonObject { put("constellation_updates_request_info", JsonArray(listOf(info))) }) } + return graphPost(token, friendlyName, docId, variables, http) +} + +/** One persisted query on Meta's AR endpoint, as the Meta app sends it: the HTTP status and the answer's JSON. */ +internal fun graphPost(token: Token, friendlyName: String, docId: String, variables: JsonObject, http: HttpPost): Pair { val body = pyUrlencode( linkedMapOf( "fb_api_req_friendly_name" to friendlyName, diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt index 3aab2056..e0914b4b 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt @@ -64,6 +64,105 @@ object Band { /** RpcRequest {1: seq, 17: HapticsReq {3: StopReq {}}}. */ fun stopHaptics(seq: Int): ByteArray = request(seq, 17, Wire.fieldBytes(3, ByteArray(0))) + // The firmware update: Ota on RpcRequest 7 (answers in RpcResponse 8), the bulk file transfer on 9 (answers in 10). + const val OTA_ITEM_FIRMWARE = 0 + const val OTA_ITEM_ML_MODEL = 2 + const val OTA_UNAVAILABLE = 0 + // otaUnavailableReason when nothing stands in the way. + const val OTA_NO_REASON = 0 + const val OTA_IN_PROGRESS = 2 + const val OTA_RESULT_OK = 0 + const val TRANSFER_OTA = 1 + const val TRANSFER_NEW = 1 + const val TRANSFER_PARTIAL = 2 + const val TRANSFER_FULL = 3 + const val CHUNK_BYTES = 950 + + /** The band's OTA device info: the firmware and ML versions by item, the KEK id, the OTA state and why it is unavailable. */ + class OtaInfo(val firmware: Long, val ml: Long, val kek: Long, val state: Int, val unavailable: Int) + + /** BulkDataTransferResp: the info answer, a chunk's ack or a nack. */ + sealed interface Bulk { + class Info(val state: Int, val memory: Long, val resume: Long, val transferId: Int) : Bulk + class Ack(val transferId: Int, val offset: Long) : Bulk + class Nack(val transferId: Int, val reason: String) : Bulk + } + + /** RpcRequest {1: seq, 7: OtaReq {1: DeviceInfoRequest {}}}. */ + fun otaDeviceInfo(seq: Int): ByteArray = ota(seq, 1, ByteArray(0)) + + /** OtaReq {2: StartItemUpdateRequest {1: ItemInfo, 2: size, 4: kekVersion}}; the version check (3) is left as the band has it. */ + fun otaStartItem(seq: Int, itemType: Int, version: Long, size: Long): ByteArray = + ota(seq, 2, Wire.fieldBytes(1, itemInfo(itemType, version)) + Wire.fieldVarint(2, size) + Wire.fieldVarint(4, 0)) + + /** OtaReq {3: VerifyItemUpdateRequest {1: ItemInfo}}. */ + fun otaVerifyItem(seq: Int, itemType: Int, version: Long): ByteArray = ota(seq, 3, Wire.fieldBytes(1, itemInfo(itemType, version))) + + /** OtaReq {4: CommitUpdateRequest {}}: one commit for all items, the reboot not deferred. */ + fun otaCommit(seq: Int): ByteArray = ota(seq, 4, ByteArray(0)) + + /** OtaReq {5: AbortUpdateRequest {}}. */ + fun otaAbort(seq: Int): ByteArray = ota(seq, 5, ByteArray(0)) + + /** OtaReq {6: InstallKEKRequest {1: kekVersion, 3: encryptedKek}}; the key type is 0, the default, so not sent. */ + fun otaInstallKek(seq: Int, kekVersion: Long, encryptedKek: ByteArray): ByteArray = + ota(seq, 6, Wire.fieldVarint(1, kekVersion) + Wire.fieldBytes(3, encryptedKek)) + + /** RpcRequest {1: seq, 9: BulkDataTransferReq {1: InfoRequest {1 MD5, 2 size, 3 type, 4 transferID}}}. */ + fun bulkInfo(seq: Int, md5: ByteArray, size: Long, transferId: Int): ByteArray = request( + seq, 9, + Wire.fieldBytes(1, Wire.fieldBytes(1, md5) + Wire.fieldVarint(2, size) + Wire.fieldVarint(3, TRANSFER_OTA.toLong()) + Wire.fieldVarint(4, transferId.toLong())), + ) + + /** RpcRequest {1: seq, 9: BulkDataTransferReq {2: ChunkData {1 transferID, 2 chunkOffset, 3 chunkData}}}. */ + fun bulkChunk(seq: Int, transferId: Int, offset: Long, data: ByteArray): ByteArray = request( + seq, 9, Wire.fieldBytes(2, Wire.fieldVarint(1, transferId.toLong()) + Wire.fieldVarint(2, offset) + Wire.fieldBytes(3, data)), + ) + + /** The seq an answer echoes. */ + fun seqOf(payload: ByteArray): Int = Wire.parse(payload).varint(1).toInt() + + /** The OTA device info in an answer (OtaResp 2), or null when it is another answer. */ + fun otaInfo(payload: ByteArray): OtaInfo? { + val ota = otaAnswer(payload) ?: return null + if (!ota.has(2)) return null + val info = Wire.parse(ota.bytes(2)) + val items = info.repeatedBytes(5).map { Wire.parse(it) }.associate { it.varint(1).toInt() to it.varint(2) } + return OtaInfo( + firmware = items[OTA_ITEM_FIRMWARE] ?: 0, + ml = items[OTA_ITEM_ML_MODEL] ?: 0, + kek = info.varint(6), + state = info.varint(4).toInt(), + unavailable = info.varint(7).toInt(), + ) + } + + /** The result code of a GeneralResponse (1), VerifyItemUpdateResponse (3) or CommitUpdateResponse (4); null for other answers. */ + fun otaResult(payload: ByteArray): Int? { + val ota = otaAnswer(payload) ?: return null + val field = listOf(1, 3, 4).firstOrNull { ota.has(it) } ?: return null + return Wire.parse(Wire.parse(ota.bytes(field)).bytes(1)).varint(1).toInt() + } + + /** The bulk answer in an RpcResponse 10, or null for another answer. */ + fun bulk(payload: ByteArray): Bulk? { + val rpc = Wire.parse(payload) + if (!rpc.has(10)) return null + val answer = Wire.parse(rpc.bytes(10)) + return when { + answer.has(1) -> Wire.parse(answer.bytes(1)).let { Bulk.Info(it.varint(1).toInt(), it.varint(2), it.varint(3), it.varint(5).toInt()) } + answer.has(2) -> Wire.parse(answer.bytes(2)).let { Bulk.Ack(it.varint(1).toInt(), it.varint(2)) } + answer.has(3) -> Wire.parse(answer.bytes(3)).let { Bulk.Nack(it.varint(1).toInt(), it.bytes(2).decodeToString()) } + else -> null + } + } + + private fun otaAnswer(payload: ByteArray): Wire.Fields? = Wire.parse(payload).let { if (it.has(8)) Wire.parse(it.bytes(8)) else null } + + private fun ota(seq: Int, field: Int, body: ByteArray): ByteArray = request(seq, 7, Wire.fieldBytes(field, body)) + + private fun itemInfo(itemType: Int, version: Long): ByteArray = Wire.fieldVarint(1, itemType.toLong()) + Wire.fieldVarint(2, version) + fun isGesture(type: Int): Boolean = type == 13 || type in 60..200 fun gesture(type: Int, payload: ByteArray): Gesture { From 22a9817a56307d77caa56a8d1d8ced586a7764ed Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:23:12 +0200 Subject: [PATCH 58/90] Display: RCS chats in the Messages app from Google Messages' notifications. Android hides the RCS rows Google Messages files as restricted MMS from every app but the default SMS app, so chats made only of them were missing. Visor now keeps, in memory only, the conversations the default SMS app's notifications show (title, numbers, and up to 50 messages each, for 30 chats), and lists those that match no thread of the SMS store, with the notification's own thread id, so a reply still goes through its reply action. A thread the store returns empty is answered from them too. The MMS send path was reviewed against the MMS encapsulation and needed no change. --- .../visor/connection/PhoneMessages.kt | 56 ++++++++++++------ .../macedonga/visor/notify/MirrorListener.kt | 57 ++++++++++++++++--- .../com/macedonga/visor/notify/RecentChats.kt | 53 +++++++++++++++++ 3 files changed, 143 insertions(+), 23 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/notify/RecentChats.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt index 1561283c..eab52331 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt @@ -22,6 +22,7 @@ import com.macedonga.visor.core.session.DataXConnection import com.macedonga.visor.core.session.DataXMessage import com.macedonga.visor.core.session.Responder import com.macedonga.visor.log.EventLog +import com.macedonga.visor.notify.RecentChats import com.macedonga.visor.notify.TextReplies import dagger.hilt.android.qualifiers.ApplicationContext import java.nio.ByteBuffer @@ -48,8 +49,8 @@ import kotlinx.coroutines.withTimeoutOrNull * The Phone app's recent calls (type 11) come on this service too; [PhoneCalls] answers them, and the call control * (placing, answering) is [PhoneCallControl]'s. * - * The RCS chats Google Messages files as MMS are marked restricted, and Android shows those rows only to the default - * SMS app: a chat with nothing else in it is missing from the list here. + * Google Messages files RCS chats as MMS marked restricted, which Android shows only to the default SMS app. Those chats + * come from what the SMS app's notifications showed ([RecentChats], this process only), listed and answered alongside. */ @Singleton class PhoneMessages @Inject constructor( @@ -59,6 +60,7 @@ class PhoneMessages @Inject constructor( private val replies: TextReplies, private val calls: PhoneCalls, private val control: PhoneCallControl, + private val recent: RecentChats, ) : Responder { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Messages: failed: $e") }) @@ -94,19 +96,27 @@ class PhoneMessages @Inject constructor( return requests.finish(id, "SMS not allowed", failed = true) } val count = request.varint(3).toInt().takeIf { it > 0 } ?: DEFAULT_THREADS - val threads = conversations(count, null) + val stored = conversations(count, null) + // The remembered chats the store does not hold, by thread id or by a number it has. + val listed = stored.map { t -> + Listed(t.id.toString(), t.addresses, t.date, t.snippet.ifBlank { messagesOf(t.id, 1).firstOrNull()?.text.orEmpty() }, "", t.addresses.size > 1) + } + recent.chats().filter { c -> + stored.none { s -> s.id.toString() == c.id || s.addresses.any { a -> c.numbers.any { RecentChats.sameNumber(a, it) } } } + }.map { c -> + Listed(c.id, c.numbers, c.messages.firstOrNull()?.time ?: 0L, c.messages.firstOrNull()?.text.orEmpty(), c.title, c.group) + } + val threads = listed.sortedByDescending { it.date }.take(count) val names = HashMap() var body = Wire.fieldBytes(1, requestId) for (t in threads) { - val name = t.addresses.joinToString(", ") { a -> names.getOrPut(a) { nameOf(a) } } - val latest = t.snippet.ifBlank { messagesOf(t.id, 1).firstOrNull()?.text.orEmpty() } + val name = if (t.group && t.title.isNotBlank()) t.title else (t.numbers.joinToString(", ") { a -> names.getOrPut(a) { nameOf(a) } }.ifEmpty { t.title }) // SgThread: 1 id, 2 name, 3 type (1 group), 4 latest message, 5 its time, 6 Participant {1 id}. body += Wire.fieldBytes( 2, - Wire.fieldBytes(1, t.id.toString().encodeToByteArray()) + Wire.fieldBytes(2, name.encodeToByteArray()) + - (if (t.addresses.size > 1) Wire.fieldVarint(3, THREAD_GROUP) else ByteArray(0)) + - Wire.fieldBytes(4, latest.encodeToByteArray()) + Wire.fieldVarint(5, t.date) + - t.addresses.fold(ByteArray(0)) { acc, a -> acc + Wire.fieldBytes(6, Wire.fieldBytes(1, a.encodeToByteArray())) }, + Wire.fieldBytes(1, t.id.encodeToByteArray()) + Wire.fieldBytes(2, name.encodeToByteArray()) + + (if (t.group) Wire.fieldVarint(3, THREAD_GROUP) else ByteArray(0)) + + Wire.fieldBytes(4, t.snippet.encodeToByteArray()) + Wire.fieldVarint(5, t.date) + + t.numbers.fold(ByteArray(0)) { acc, a -> acc + Wire.fieldBytes(6, Wire.fieldBytes(1, a.encodeToByteArray())) }, ) } body += Wire.fieldVarint(3, threads.maxOfOrNull { it.date } ?: 0) @@ -119,15 +129,17 @@ class PhoneMessages @Inject constructor( val requestId = request.bytes(2) val numbers = request.repeatedBytes(9).map { it.decodeToString() }.filter { it.isNotBlank() } val chunked = request.varint(8) != 0L - val id = requests.start("sms://thread/${request.bytes(4).decodeToString()}", SOURCE_SMS) + val key = request.bytes(4).decodeToString() + val id = requests.start("sms://thread/$key", SOURCE_SMS) if (!allowed(Manifest.permission.READ_SMS)) { answer(mux, reply, MT_LOAD_MESSAGES, Wire.fieldBytes(1, requestId) + refused(), chunked) return requests.finish(id, "SMS not allowed", failed = true) } - val thread = request.bytes(4).decodeToString().toLongOrNull() + val thread = key.toLongOrNull() ?: numbers.takeIf { it.isNotEmpty() }?.let { runCatching { Telephony.Threads.getOrCreateThreadId(context, it.toSet()) }.getOrNull() } val count = request.varint(3).toInt().takeIf { it > 0 }?.coerceAtMost(MAX_MESSAGES) ?: DEFAULT_MESSAGES - val rows = if (thread != null) messagesOf(thread, count) else emptyList() + // A store thread that came back empty, or a chat the store does not hold, is answered from what its notifications showed. + val rows = (thread?.let { messagesOf(it, count) }.orEmpty()).ifEmpty { recentMessages(key, thread, numbers, count) } var body = Wire.fieldBytes(1, requestId) for (m in rows.asReversed()) { // SgMessage: 1 id, 2 text, 3 time, 4 the address (the other side's, in or out), 7 attachment, 19 box, 20 service. @@ -139,11 +151,21 @@ class PhoneMessages @Inject constructor( Wire.fieldBytes(19, folder(m.box).encodeToByteArray()) + Wire.fieldVarint(20, m.service), ) } - body += Wire.fieldVarint(5, RESPONSE_FILE_BUS) + Wire.fieldBytes(7, (thread ?: 0).toString().encodeToByteArray()) + body += Wire.fieldVarint(5, RESPONSE_FILE_BUS) + Wire.fieldBytes(7, (thread?.toString() ?: key.ifEmpty { "0" }).encodeToByteArray()) answer(mux, reply, MT_LOAD_MESSAGES, body, chunked) requests.finish(id, "${rows.size} messages", failed = false, bytes = body.size.toLong()) } + /** A chat from the notifications, newest first, in the same shape as the store's messages. */ + private fun recentMessages(key: String, thread: Long?, numbers: List, count: Int): List { + val chat = recent.find(key) ?: thread?.let { recent.find(it.toString()) } ?: recent.withNumber(numbers) ?: return emptyList() + return chat.messages.take(count).map { m -> + val other = if (m.outgoing) chat.numbers.firstOrNull().orEmpty() else m.number + val box = if (m.outgoing) Telephony.Sms.MESSAGE_TYPE_SENT else Telephony.Sms.MESSAGE_TYPE_INBOX + Message("recent://${chat.id}/${m.time}", m.text, m.time, other, box, SERVICE_SMS, false) + } + } + /** OutgoingMessageRequest {2 request id, 3 thread id, 4 text, 7 participants}: OutgoingMessageResponse {1, 2 status, 3 error}. */ private suspend fun send(request: Wire.Fields, reply: Int, mux: DataXConnection, chat: Boolean) { val requestId = request.bytes(2) @@ -215,13 +237,15 @@ class PhoneMessages @Inject constructor( /** The numbers of a thread's other side. */ fun addressesOf(thread: String): List { - val id = thread.toLongOrNull() ?: return emptyList() - if (!allowed(Manifest.permission.READ_SMS)) return emptyList() - return conversations(1, id).firstOrNull()?.addresses.orEmpty() + val stored = thread.toLongOrNull()?.takeIf { allowed(Manifest.permission.READ_SMS) }?.let { conversations(1, it).firstOrNull()?.addresses } + return stored ?: recent.find(thread)?.numbers.orEmpty() } private class Conversation(val id: Long, val date: Long, val addresses: List, val snippet: String) + /** A thread for the list: from the store, or a remembered chat (its id is the notification's thread). */ + private class Listed(val id: String, val numbers: List, val date: Long, val snippet: String, val title: String, val group: Boolean) + /** Conversations newest first, from Android's thread list, which holds SMS, MMS and the RCS chats Messages files as MMS. */ private fun canonical(): Map { val canonical = HashMap() diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt b/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt index 1c490894..cfd932fa 100644 --- a/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt +++ b/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt @@ -67,6 +67,9 @@ class MirrorListener : NotificationListenerService(), Messaging { @Inject lateinit var textReplies: TextReplies + @Inject + lateinit var recent: RecentChats + // Each conversation sent to the glasses, with the app's reply action, newest last. private val replyable = LinkedHashMap>() @@ -108,6 +111,9 @@ class MirrorListener : NotificationListenerService(), Messaging { override fun onNotificationPosted(sbn: StatusBarNotification, rankingMap: RankingMap) { if (!wanted(sbn, rankingMap)) return apps.saw(sbn.packageName) + // The Messages app's conversations are kept whether or not the glasses listen: the Display's list reads them. + val sms = sbn.packageName == Telephony.Sms.getDefaultSmsPackage(this) + if (sms) remember(sbn) if (!choice.enabled.value || !ready() || sbn.packageName in apps.muted.value) return val n = sbn.notification val extras = n.extras @@ -116,7 +122,6 @@ class MirrorListener : NotificationListenerService(), Messaging { val appName = runCatching { packageManager.getApplicationLabel(packageManager.getApplicationInfo(sbn.packageName, 0)).toString() }.getOrDefault(sbn.packageName) val r = Ranking() // The phone's texts, SMS or RCS, go as native messages, which the glasses' Messages app files by thread. - val sms = sbn.packageName == Telephony.Sms.getDefaultSmsPackage(this) (if (sms) PROVIDER_NATIVE else PROVIDERS[sbn.packageName])?.let { provider -> message(sbn, provider, appName)?.let { m -> deliver(m) @@ -227,19 +232,16 @@ class MirrorListener : NotificationListenerService(), Messaging { // A reply sent from the notification is added to it as the user's own message: not news for the glasses. @Suppress("DEPRECATION") val user = extras.getParcelable(Notification.EXTRA_MESSAGING_PERSON) as? Person - val fromMe = last != null && last.getCharSequence("sender") == null && - (person == null || user != null && person.key == user.key && person.name == user.name) - if (fromMe) return null + if (last != null && fromUser(last, user)) return null if (sentAt[sbn.key] == time) return null sentAt[sbn.key] = time while (sentAt.size > 300) sentAt.remove(sentAt.keys.first()) val group = extras.getBoolean(Notification.EXTRA_IS_GROUP_CONVERSATION) val native = provider == PROVIDER_NATIVE // A text names its sender by number and its thread as the SMS store does, as svc70 lists them. - val number = if (native) person?.uri?.takeIf { it.startsWith("tel:") }?.let { Uri.decode(it.substringAfter("tel:")) } ?: numberOf(sender) else null - val thread = number?.takeIf { !group }?.let { runCatching { Telephony.Threads.getOrCreateThreadId(this, it).toString() }.getOrNull() } + val number = if (native) numberOf(person, sender) else null // The thread the glasses answer with (FoA 2) names the notification's reply action, for every provider. - val threadId = thread ?: sbn.notification.shortcutId ?: sbn.key + val threadId = threadOf(sbn, number, group) replyAction(sbn.notification)?.let { textReplies.put(threadId, sbn.key, it) } return GlassesNotification( key = sbn.key, @@ -258,6 +260,47 @@ class MirrorListener : NotificationListenerService(), Messaging { ) } + /** The other side's number: the person's tel: uri, else the sender's number or contact. */ + private fun numberOf(person: android.app.Person?, sender: String): String? = + person?.uri?.takeIf { it.startsWith("tel:") }?.let { Uri.decode(it.substringAfter("tel:")) } ?: numberOf(sender) + + /** The SMS store's thread for a one-to-one text, else the glasses' key for the conversation. */ + private fun threadOf(sbn: StatusBarNotification, number: String?, group: Boolean): String = + number?.takeIf { !group }?.let { runCatching { Telephony.Threads.getOrCreateThreadId(this, it).toString() }.getOrNull() } + ?: sbn.notification.shortcutId ?: sbn.key + + /** A message the user sent: no sender, and the user's own person when one is given. */ + private fun fromUser(m: android.os.Bundle, user: Person?): Boolean { + @Suppress("DEPRECATION") + val person = m.getParcelable("sender_person") as? android.app.Person + return m.getCharSequence("sender") == null && (person == null || user != null && person.key == user.key && person.name == user.name) + } + + /** Keeps a Messages conversation as its notification shows it, for the Display's list (see RecentChats). */ + private fun remember(sbn: StatusBarNotification) { + val extras = sbn.notification.extras + @Suppress("DEPRECATION") + val user = extras.getParcelable(Notification.EXTRA_MESSAGING_PERSON) as? Person + val group = extras.getBoolean(Notification.EXTRA_IS_GROUP_CONVERSATION) + @Suppress("DEPRECATION") + val bundles = extras.getParcelableArray(Notification.EXTRA_MESSAGES).orEmpty().filterIsInstance() + val numbers = LinkedHashSet() + val messages = bundles.mapNotNull { m -> + val text = m.getCharSequence("text")?.toString()?.takeIf { it.isNotBlank() } ?: return@mapNotNull null + @Suppress("DEPRECATION") + val person = m.getParcelable("sender_person") as? android.app.Person + val outgoing = fromUser(m, user) + val sender = m.getCharSequence("sender")?.toString() ?: person?.name?.toString().orEmpty() + val number = if (outgoing) null else numberOf(person, sender) + number?.let { numbers += it } + RecentChats.Message(text, sender, number.orEmpty(), m.getLong("time").takeIf { it > 0 } ?: sbn.postTime, outgoing) + } + if (messages.isEmpty()) return + val title = extras.getCharSequence(Notification.EXTRA_CONVERSATION_TITLE)?.takeIf { group } ?: extras.getCharSequence(Notification.EXTRA_TITLE) + val id = threadOf(sbn, numbers.singleOrNull(), group) + recent.remember(id, title?.toString().orEmpty(), group, numbers.toList(), messages) + } + /** A contact's number by their name, or the name itself when it is a number. */ private fun numberOf(name: String): String? { if (name.isNotEmpty() && name.all { it.isDigit() || it in "+ -()" }) return name.filter { it.isDigit() || it == '+' } diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/RecentChats.kt b/app/src/main/kotlin/com/macedonga/visor/notify/RecentChats.kt new file mode 100644 index 00000000..e13757ee --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/notify/RecentChats.kt @@ -0,0 +1,53 @@ +package com.macedonga.visor.notify + +import javax.inject.Inject +import javax.inject.Singleton + +/** + * The SMS app's conversations as its notifications show them, for this process only. Google Messages files RCS chats + * as restricted MMS rows that the phone's texts list cannot read, so the Display's list takes these instead. Each chat + * keeps its newest messages, and the least recently updated chat goes first. + */ +@Singleton +class RecentChats @Inject constructor() { + class Message(val text: String, val sender: String, val number: String, val time: Long, val outgoing: Boolean) + + class Chat(val id: String, val title: String, val group: Boolean, val numbers: List, val messages: List) + + private val chats = LinkedHashMap() + + /** Adds [incoming] (any order) to chat [id], newest first, without repeats. */ + @Synchronized + fun remember(id: String, title: String, group: Boolean, numbers: List, incoming: List) { + val old = chats.remove(id) + val messages = ((old?.messages.orEmpty()) + incoming) + .distinctBy { it.time to it.text } + .sortedByDescending { it.time } + .take(MAX_MESSAGES) + chats[id] = Chat(id, title, group, ((old?.numbers.orEmpty()) + numbers).distinct(), messages) + while (chats.size > MAX_CHATS) chats.remove(chats.keys.first()) + } + + /** The chats, newest message first. */ + @Synchronized + fun chats(): List = chats.values.sortedByDescending { it.messages.firstOrNull()?.time ?: 0L } + + @Synchronized + fun find(id: String): Chat? = chats[id] + + /** The chat with a participant in [numbers], if any. */ + @Synchronized + fun withNumber(numbers: List): Chat? = + chats.values.firstOrNull { c -> c.numbers.any { n -> numbers.any { sameNumber(n, it) } } } + + companion object { + private const val MAX_CHATS = 30 + private const val MAX_MESSAGES = 50 + + /** Two numbers are one line when their last ten digits match. */ + fun sameNumber(a: String, b: String): Boolean { + val x = a.filter(Char::isDigit).takeLast(10) + return x.isNotEmpty() && x == b.filter(Char::isDigit).takeLast(10) + } + } +} From dc87dfcd42d7a14daf820ea0cb479118b8b1fe84 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:26:10 +0200 Subject: [PATCH 59/90] Display: capture state on svc2000. The glasses open media_capture_state on connect and send ConcurrentSignals (type 1): "ready", then video, hyperlapse, slow motion and dynamic photo started and stopped. Visor answers nothing, as the Meta app does, and asks the glasses for the capture list again after a stop, so the auto import finds the new capture sooner. svc1999 media_backsync is opened only by the phone for AI-edit results, so Visor leaves it alone. A capture with no photo or video in it is now named once in the event log instead of being skipped silently. Not tried on the Display yet. --- .../visor/connection/CaptureState.kt | 30 +++++++++++++++++++ .../macedonga/visor/di/ConnectionModule.kt | 2 ++ .../macedonga/visor/core/media/Captures.kt | 1 + .../macedonga/visor/core/media/MediaImport.kt | 6 ++++ 4 files changed, 39 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/CaptureState.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/CaptureState.kt b/app/src/main/kotlin/com/macedonga/visor/connection/CaptureState.kt new file mode 100644 index 00000000..1f66cc90 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/CaptureState.kt @@ -0,0 +1,30 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.media.MediaImport +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * svc2000, media_capture_state: the glasses open it on connect and send a ConcurrentSignal (type 1) whose signal type + * is "ready" first, then video, hyperlapse, slow motion or dynamic photo started and stopped. Nothing is asked of the + * phone, so nothing is answered. A stop refreshes the capture list, and AutoImport takes the new capture from there. + */ +@Singleton +class CaptureState @Inject constructor(private val media: Provider) : Responder { + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed || message.info.messageType != MT_CONCURRENT_SIGNAL) return + // ConcurrentSignal field 6 is signal_type; field 5 is the source. + val signalType = Wire.parse(message.payload).bytes(6).decodeToString() + if (signalType.endsWith("_stopped")) media.get().refresh() + } + + companion object { + const val SVC = 2000 + private const val MT_CONCURRENT_SIGNAL = 1 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 9ca75ec0..1b59fb2b 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -152,6 +152,7 @@ object ConnectionModule { sharing: com.macedonga.visor.connection.PhoneSharing, calls: com.macedonga.visor.connection.PhoneCalls, media: com.macedonga.visor.connection.PhoneMedia, + captureState: com.macedonga.visor.connection.CaptureState, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -174,6 +175,7 @@ object ConnectionModule { com.macedonga.visor.connection.PhoneSharing.SVC to sharing, com.macedonga.visor.connection.PhoneCalls.SVC to calls, com.macedonga.visor.connection.PhoneMedia.SVC to media, + com.macedonga.visor.connection.CaptureState.SVC to captureState, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt index e8a76ab7..e552a1d3 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt @@ -25,6 +25,7 @@ const val KIND_STILL = 2 const val KIND_IMU = 3 const val KIND_FRAMES = 4 +// The Display's kinds are the wire CaptureType plus 0xC00 (inferred). Any other kind is classed by its assets. private const val CAPTURE_VIDEO = 3074L private const val ASSET_DELIVERED = 0x100000L diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/MediaImport.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/MediaImport.kt index bb650736..859b82a7 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/MediaImport.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/MediaImport.kt @@ -94,6 +94,9 @@ class MediaImport( // Asked for while another import ran: they go next. private val queued = java.util.Collections.synchronizedSet(LinkedHashSet()) + // Captures that are not media, logged once each. + private val noticed = java.util.Collections.synchronizedSet(HashSet()) + // The import running, and when it last said or received anything. @Volatile private var running: Job? = null @@ -453,6 +456,9 @@ class MediaImport( private fun onMedia(data: ServiceData) { when (data.messageType) { MT_CAPTURE -> parseCapture(data.payload)?.let { capture -> + if (!capture.isMedia && noticed.add(capture.handle)) { + say("Skipped a capture of kind ${capture.kindCode}: no photo or video in it, assets ${capture.assets.map { it.kind }}") + } _captures.update { list -> (list.filter { it.handle != capture.handle } + capture).sortedByDescending { it.timestampNs } } From 2c1ca0118730ea854de140af66e1064b211ccc7f Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:32:50 +0200 Subject: [PATCH 60/90] Display: answer the lease service (svc41107). The Display asks the phone for internet (type 1) and for a link (type 4) on svc41107, which Visor did not serve. Internet leases are answered as the Meta app answers them: use the glasses' own Wi-Fi (type 2, ERROR_CODE_USE_DEVICE_WIFI), and a block is kept without an answer; the phone's internet still reaches the Display's apps through the tunnel. A link lease on Bluetooth is granted (type 5); one that needs Wi-Fi Direct, a soft AP or local Wi-Fi is refused with WIFI_DIRECT_NOT_AVAILABLE until the session can move to Wi-Fi for it. Expiries (type 3) are logged. Not tried on the Display yet. --- .../visor/connection/LeaseService.kt | 85 +++++++++++++++++++ .../macedonga/visor/di/ConnectionModule.kt | 2 + 2 files changed, 87 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt new file mode 100644 index 00000000..93d37d3b --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt @@ -0,0 +1,85 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * svc41107, lease_service: the glasses ask for internet (type 1, InternetLeaseRequest) and for a link (type 4, + * LinkLeaseRequest), and end a lease (type 3, no answer). Internet leases are answered as the Meta app answers them + * (2, use the glasses' own Wi-Fi); a link lease (5) on Bluetooth is granted, one that needs Wi-Fi is refused for now. + */ +@Singleton +class LeaseService @Inject constructor( + private val log: Provider, +) : Responder { + // Leases the glasses hold, by lease id, so an expiry can be named in the log. + private val held = ConcurrentHashMap() + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed) return + val channel = DataXConnection.mirror(message.info.channelId) + val request = Wire.parse(message.payload) + val id = request.bytes(1) + val key = id.joinToString("") { "%02x".format(it) } + when (message.info.messageType) { + MT_INTERNET_REQUEST -> { + val type = INTERNET_TYPES.getOrElse(request.varint(2).toInt()) { "UNKNOWN" } + // As the Meta app does: a block is kept and not answered; any other kind is told to use the glasses' own Wi-Fi. + // The phone's internet still reaches the Display's apps through the tunnel (svc41108, svc41111). + if (type == "BLOCK") { + log.get().add("Lease: the glasses blocked internet leases") + return + } + log.get().add("Lease: the glasses asked for internet ($type); answered: use the glasses' Wi-Fi") + connection.send(channel, Wire.fieldBytes(1, id) + refusal(ERROR_USE_DEVICE_WIFI, "ERROR_CODE_USE_DEVICE_WIFI"), messageType = MT_INTERNET_RESPONSE) + } + MT_LINK_REQUEST -> { + // Field 7 is a link (2 Wi-Fi Direct, 3 SoftAP, 4 local Wi-Fi); field 6 a bandwidth (1 medium, 2 high). + val wifi = when { + request.has(7) -> request.varint(7) in 2..4 + request.has(6) -> request.varint(6) >= 1 + else -> null + } + val reply = when (wifi) { + null -> Wire.fieldBytes(1, id) + refusal(ERROR_BAD_REQUEST, "Invalid Request") + true -> Wire.fieldBytes(1, id) + refusal(ERROR_WIFI_DIRECT_NOT_AVAILABLE, "Wi-Fi is not available for this link.") + false -> Wire.fieldBytes(1, id) + } + log.get().add( + when (wifi) { + null -> "Lease: the glasses asked for a link with no requirement; refused" + true -> "Lease: the glasses asked for Wi-Fi; not available yet, refused" + false -> "Lease: the glasses asked for a link on Bluetooth; answered OK" + }, + ) + if (wifi == false) held[key] = "link" + connection.send(channel, reply, messageType = MT_LINK_RESPONSE) + } + MT_LEASE_EXPIRATION -> held.remove(key)?.let { log.get().add("Lease: a $it lease ended") } + else -> log.get().add("Lease: message type ${message.info.messageType} is not handled") + } + } + + private fun refusal(code: Int, text: String): ByteArray = + Wire.fieldBytes(2, Wire.fieldVarint(1, code.toLong()) + Wire.fieldBytes(2, text.encodeToByteArray())) + + companion object { + const val SVC = 41107 + private const val MT_INTERNET_REQUEST = 1 + private const val MT_INTERNET_RESPONSE = 2 + private const val MT_LEASE_EXPIRATION = 3 + private const val MT_LINK_REQUEST = 4 + private const val MT_LINK_RESPONSE = 5 + private val INTERNET_TYPES = listOf("AUTO_BIND", "WARM", "WARM_THEN_IDLE", "BLOCK") + private const val ERROR_BAD_REQUEST = 0 + private const val ERROR_USE_DEVICE_WIFI = 4 + private const val ERROR_WIFI_DIRECT_NOT_AVAILABLE = 104 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 1b59fb2b..2da39456 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -153,6 +153,7 @@ object ConnectionModule { calls: com.macedonga.visor.connection.PhoneCalls, media: com.macedonga.visor.connection.PhoneMedia, captureState: com.macedonga.visor.connection.CaptureState, + leases: com.macedonga.visor.connection.LeaseService, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -176,6 +177,7 @@ object ConnectionModule { com.macedonga.visor.connection.PhoneCalls.SVC to calls, com.macedonga.visor.connection.PhoneMedia.SVC to media, com.macedonga.visor.connection.CaptureState.SVC to captureState, + com.macedonga.visor.connection.LeaseService.SVC to leases, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), From 85d385c791494e93258eae93d31b28d5265b4201 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:33:44 +0200 Subject: [PATCH 61/90] Display: Mirror, the open answered and a stopped service told apart. The Meta app waits for the screencast manager's answer to the open before it sends the encoder settings, and Visor now does the same (an open left unanswered for 3 s still goes on to the settings). The watch for 0xC001, the glasses' "service not running", was set before any answer was awaited, so that reply was dropped and the start timed out as "no answer"; it is now part of each wait, and the screen says the glasses need a restart, which the firmware's init needs to start the service once the MobileConfig switch is on. Parameter-set frames reach the decoder as codec config instead of being dropped. The Display's home shows the Mirror card again. Not tried on the Display yet. --- .../macedonga/visor/display/DisplayShelf.kt | 10 +++ .../macedonga/visor/media/MirrorDecoder.kt | 7 ++- .../kotlin/com/macedonga/visor/ui/Icons.kt | 2 + .../macedonga/visor/core/media/Screencast.kt | 62 ++++++++++++------- 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt index cce3b32b..5f685bc1 100644 --- a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt +++ b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt @@ -45,6 +45,7 @@ class ShelfModel @Inject constructor( val connection: com.macedonga.visor.core.connection.GlassesConnection, val captions: com.macedonga.visor.core.connection.Captions, val captionsChoices: CaptionsChoices, + val screencast: com.macedonga.visor.core.media.Screencast, ) : ViewModel() /** The Display's tools on the home: a card each, with what it holds now. */ @@ -87,6 +88,15 @@ fun DisplayShelf(onOpen: (DisplayTool) -> Unit, modifier: Modifier = Modifier, m onClick = { onOpen(DisplayTool.GESTURES) }, ) } + item(key = "mirror") { + val mirror by model.screencast.state.collectAsStateWithLifecycle() + ToolCard( + VisorIcons.Mirror, + stringResource(R.string.mirror_title), + if (mirror.phase == com.macedonga.visor.core.media.ScreencastPhase.LIVE) stringResource(R.string.mirror_live) else stringResource(R.string.mirror_idle), + onClick = { onOpen(DisplayTool.MIRROR) }, + ) + } } } diff --git a/app/src/main/kotlin/com/macedonga/visor/media/MirrorDecoder.kt b/app/src/main/kotlin/com/macedonga/visor/media/MirrorDecoder.kt index 962f0b90..abad6971 100644 --- a/app/src/main/kotlin/com/macedonga/visor/media/MirrorDecoder.kt +++ b/app/src/main/kotlin/com/macedonga/visor/media/MirrorDecoder.kt @@ -5,20 +5,21 @@ import android.media.MediaFormat import android.view.Surface import com.macedonga.visor.core.media.ScreencastFrame -/** H.264 decoding of the Display's screen onto a Surface. One thread only; starts on a key frame. */ +/** H.264 decoding of the Display's screen onto a Surface. One thread only; starts on a parameter set or key frame. */ class MirrorDecoder(private val surface: Surface) { private var codec: MediaCodec? = null private var lastPtsUs = -1L private val info = MediaCodec.BufferInfo() fun feed(frame: ScreencastFrame) { - val c = codec ?: (if (frame.key) start() else null) ?: return + val c = codec ?: (if (frame.key || frame.config) start() else null) ?: return try { val i = c.dequeueInputBuffer(INPUT_WAIT_US) if (i >= 0) { c.getInputBuffer(i)!!.apply { clear(); put(frame.data) } lastPtsUs = maxOf(frame.timestampMs * 1000, lastPtsUs + 1) - c.queueInputBuffer(i, 0, frame.data.size, lastPtsUs, 0) + val flags = if (frame.config) MediaCodec.BUFFER_FLAG_CODEC_CONFIG else 0 + c.queueInputBuffer(i, 0, frame.data.size, lastPtsUs, flags) } drain(c) } catch (e: RuntimeException) { diff --git a/app/src/main/kotlin/com/macedonga/visor/ui/Icons.kt b/app/src/main/kotlin/com/macedonga/visor/ui/Icons.kt index 1add1630..8dffb5ee 100644 --- a/app/src/main/kotlin/com/macedonga/visor/ui/Icons.kt +++ b/app/src/main/kotlin/com/macedonga/visor/ui/Icons.kt @@ -198,6 +198,8 @@ object VisorIcons { "M7.5 11h4", "M13.5 11h3", "M7.5 14h7", ) + val Mirror = icon("M4 5.5h16v11H4z", "M9.5 19.5h5", "M12 16.5v3") + val People = icon( "M9 9m-2.5 0a2.5 2.5 0 1 0 5 0a2.5 2.5 0 1 0-5 0", "M4 18.5a5 5 0 0 1 10 0", "M16 9.5m-2 0a2 2 0 1 0 4 0a2 2 0 1 0-4 0", "M15.5 14a4.2 4.2 0 0 1 4.5 4.5", diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt index 8e5e2996..d33a2dd8 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt @@ -9,6 +9,7 @@ import java.nio.ByteBuffer import java.nio.ByteOrder import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob @@ -37,13 +38,21 @@ enum class ScreencastFault { NOT_CONNECTED, NOT_READ, NO_FLAG, FLAG_NOT_SENT, NE data class ScreencastState(val phase: ScreencastPhase = ScreencastPhase.IDLE, val fault: ScreencastFault? = null, val detail: String = "") -/** One whole H.264 frame, its parameter sets in band. */ -class ScreencastFrame(val timestampMs: Long, val key: Boolean, val data: ByteArray) +/** One whole H.264 frame, its parameter sets in band. [flags] are MediaCodec's: key frame, codec config. */ +class ScreencastFrame(val timestampMs: Long, val flags: Int, val data: ByteArray) { + val key get() = flags and FLAG_KEY != 0 + val config get() = flags and FLAG_CONFIG != 0 + + companion object { + const val FLAG_KEY = 1 + const val FLAG_CONFIG = 2 + } +} /** * The Display's own screen, streamed to the phone on svc23463 and driven on svc23464, over Wi-Fi as the Meta app runs * it: the manager opened and answered, then the encoder settings and their answer, then the frames opened. The service - * runs only while the MobileConfig switch is on, and the glasses read that switch when they start. + * is started by the glasses' init on the MobileConfig switch's sysprop, so a new switch needs the glasses restarted. */ class Screencast(private val connection: GlassesConnection, private val config: GlassesConfig, private val live: LiveStream) { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) @@ -57,9 +66,9 @@ class Screencast(private val connection: GlassesConnection, private val config: private var answer: CompletableDeferred>? = null private var stopping = CompletableDeferred() - // The frame being assembled: its time, key flag and the chunks still owed. + // The frame being assembled: its time, flags and the chunks still owed. private var timestamp = 0L - private var key = false + private var flags = 0 private var chunksLeft = 0 private var frame = ByteArrayOutputStream() @@ -100,26 +109,22 @@ class Screencast(private val connection: GlassesConnection, private val config: val why = live.overWifi { mux -> coroutineScope { val listening = launch { live.wifiData.collect { onData(it) } } - // A service that is not running answers 0xC001: the switch is on but the glasses have not restarted since. - val absent = launch { - live.notes.first { it.contains("service $SVC_HN_SCREENCAST_MANAGER answered status 0xc001") } - answer?.complete(Pair(false, NOT_RUNNING)) - } try { - // The open is not answered; the Meta app goes straight on to the settings, which are. - mux.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONTROL, byteArrayOf(0)) - delay(OPEN_MS) - val configured = ask { mux.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONFIGURE, configure()) } - if (configured?.first == true) { + // The Meta app waits for the manager's answer to the open, then to the settings, then opens the frames. + // An open left unanswered goes on to the settings, as Visor did before. + val opened = ask(OPEN_MS) { mux.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONTROL, byteArrayOf(0)) } + val got = if (opened == null || opened.first) { + ask { mux.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONFIGURE, configure()) } + } else opened + if (got?.first == true) { mux.sendOnService(SVC_HN_SCREENCAST, MT_CONTROL, byteArrayOf(0)) _state.value = ScreencastState(ScreencastPhase.LIVE) stopping.await() mux.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONTROL, byteArrayOf(3)) delay(STOP_MS) - } else refused(configured) + } else refused(got) } finally { listening.cancel() - absent.cancel() } } null @@ -127,11 +132,21 @@ class Screencast(private val connection: GlassesConnection, private val config: if (why != null) fail(ScreencastFault.NO_WIFI, why) } - /** Send, and wait for the manager's answer: success, and why not. */ - private suspend fun ask(send: suspend () -> Unit): Pair? { + /** + * Send, and wait for the manager's answer: success, and why not. A 0xC001 from the glasses also answers, as + * "not running": the switch is on but the glasses have not started the service since it was set. + */ + private suspend fun ask(waitMs: Long = ANSWER_MS, send: suspend () -> Unit): Pair? = coroutineScope { val reply = CompletableDeferred>().also { answer = it } + val absent = launch(start = CoroutineStart.UNDISPATCHED) { + live.notes.first { it.contains("service $SVC_HN_SCREENCAST_MANAGER answered status 0xc001") } + reply.complete(Pair(false, NOT_RUNNING)) + } send() - return withTimeoutOrNull(ANSWER_MS) { reply.await() }.also { answer = null } + withTimeoutOrNull(waitMs) { reply.await() }.also { + absent.cancel() + answer = null + } } private fun refused(got: Pair?) { @@ -162,14 +177,14 @@ class Screencast(private val connection: GlassesConnection, private val config: if (p.size == HEADER_BYTES) { val b = ByteBuffer.wrap(p).order(ByteOrder.LITTLE_ENDIAN) timestamp = b.getLong(0) - key = b.getInt(8) and KEY_FLAG != 0 + flags = b.getInt(8) chunksLeft = b.getInt(24) frame = ByteArrayOutputStream() return } if (chunksLeft <= 0) return frame.write(p) - if (--chunksLeft == 0) _frames.tryEmit(ScreencastFrame(timestamp, key, frame.toByteArray())) + if (--chunksLeft == 0) _frames.tryEmit(ScreencastFrame(timestamp, flags, frame.toByteArray())) } // Five little-endian int32s: bitrate, I-frame interval, 2^profile, 2^level, framerate. These are the Meta app's defaults. @@ -181,10 +196,9 @@ class Screencast(private val connection: GlassesConnection, private val config: const val MT_CONTROL = 129 const val MT_CONFIGURE = 1 const val HEADER_BYTES = 32 - const val KEY_FLAG = 1 const val ANSWER_MS = 15_000L const val STOP_MS = 300L - const val OPEN_MS = 500L + const val OPEN_MS = 3_000L const val NOT_RUNNING = -1 } } From efeb65129162871b5124270e89ca1f0f6e7a1fa5 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:40:03 +0200 Subject: [PATCH 62/90] Display: put the Meta account and WhatsApp on the glasses (svc89). A Display set up with Visor has no accounts for its own apps. The Display's settings get an Accounts page: the Meta account is minted for the glasses' accounts app with the user's existing Meta login (/app_token_exchange, no new login) and sent as MetaAccountSyncRequest (type 5, answered by 13); whether one is there is asked with type 25 (26). WhatsApp links through the glasses' own WhatsApp: type 22 has the glasses show a code to scan from WhatsApp's Linked devices on the phone, and 23 says when it is done; the glasses need internet for that. The token is never logged. Not tried on the Display yet. --- .../kotlin/com/macedonga/visor/VisorNav.kt | 4 + .../visor/connection/AccountLinking.kt | 139 ++++++++++++++++++ .../visor/settings/AccountsScreen.kt | 138 +++++++++++++++++ .../macedonga/visor/settings/GlassesPage.kt | 2 + .../main/res/values-it/strings_accounts.xml | 15 ++ app/src/main/res/values/strings_accounts.xml | 15 ++ .../visor/core/identity/token/AccountToken.kt | 81 ++++++++++ 7 files changed, 394 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt create mode 100644 app/src/main/res/values-it/strings_accounts.xml create mode 100644 app/src/main/res/values/strings_accounts.xml create mode 100644 core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/AccountToken.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt index 2e7a08e5..946ecf98 100644 --- a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt +++ b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt @@ -166,6 +166,9 @@ data object RequestLogKey : NavKey @Serializable data object WifiNetworksKey : NavKey +@Serializable +data object AccountsKey : NavKey + /** The app's graph: the home, and every screen it opens. */ @Composable fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) { @@ -287,6 +290,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) entry { com.macedonga.visor.display.GesturesScreen() } entry { com.macedonga.visor.display.CaptionsScreen() } entry { com.macedonga.visor.settings.WifiNetworksScreen() } + entry { com.macedonga.visor.settings.AccountsScreen() } entry { MetaAiScreen( onSettings = { backStack.add(AssistantSettingsKey) }, diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt b/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt new file mode 100644 index 00000000..62369a1c --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt @@ -0,0 +1,139 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.token.TokenError +import com.macedonga.visor.core.identity.token.exchangeAccountToken +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.ServiceData +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.login.MetaLogin +import java.util.UUID +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull + +/** What a link attempt came to, for the Accounts page to say. */ +enum class LinkOutcome { LINKED, LOGGED_OUT, UNREACHABLE, NO_ANSWER, REFUSED } + +/** + * svc89, wearables_whatsapp_linking: the Meta account and WhatsApp linking the Meta app runs with the Display. Visor + * opens the channel. 25 asks whether a Meta account is present (answered by 26). 5 is the Meta account sync, with the + * user's token minted for the glasses' accounts app (answered by 13, after an ack 17 Visor does not wait for). 22 is + * the WhatsApp QR pairing: the glasses show a code for WhatsApp's Linked devices and answer 23 when it is scanned. + */ +@Singleton +class AccountLinking @Inject constructor( + private val connection: GlassesConnection, + private val login: MetaLogin, + private val log: EventLog, +) { + // One exchange at a time: each answer is taken as the next one of its type. + private val lock = Mutex() + + /** Whether the glasses hold a Meta account; null when they do not answer. */ + suspend fun metaPresent(): Boolean? { + val f = ask(MT_PRESENT_REQUEST, MT_PRESENT_RESPONSE, Wire.fieldBytes(1, str(uuid())), ANSWER_MS) ?: return null + return f.varint(2) == 1L + } + + /** Mints the user's token for the accounts app and sends the Meta account to the glasses. */ + suspend fun syncMeta(): LinkOutcome { + if (!login.source.hasCachedLogin) return LinkOutcome.LOGGED_OUT + val identity = try { + withContext(Dispatchers.IO) { exchangeAccountToken(login.source.get()) } + } catch (error: TokenError) { + log.add("Accounts: ${error.message}") + return if (error.unreachable) LinkOutcome.UNREACHABLE else LinkOutcome.REFUSED + } + val request = Wire.fieldBytes(1, str(uuid())) + + Wire.fieldBytes(2, str(identity.accessToken)) + + Wire.fieldBytes(3, str(identity.appId)) + + Wire.fieldBytes(4, str(identity.userId)) + + Wire.fieldBytes(5, str(identity.analyticsClaim)) + + Wire.fieldBytes(6, str(SOURCE)) + + Wire.fieldBytes(7, str(identity.accountId)) + + Wire.fieldBytes(8, str(identity.profilePictureUrl)) + + Wire.fieldVarint(9, flag(identity.isEmployee)) + + Wire.fieldVarint(10, flag(identity.isTestUser)) + + Wire.fieldBytes(11, str(identity.name)) + + Wire.fieldBytes(12, str(uuid())) + val f = ask(MT_META_SYNC_REQUEST, MT_META_SYNC_RESPONSE, request, ANSWER_MS) + if (f == null) { + log.add("Accounts: the glasses did not answer the Meta account") + return LinkOutcome.NO_ANSWER + } + if (f.varint(2) != 1L) { + log.add("Accounts: the glasses refused the Meta account, code ${f.varint(3)}") + return LinkOutcome.REFUSED + } + log.add("Accounts: Meta account sent to the glasses") + return LinkOutcome.LINKED + } + + /** Has the glasses show their WhatsApp code; the answer comes once it is scanned from WhatsApp's Linked devices. */ + suspend fun linkWhatsApp(): LinkOutcome { + val f = ask(MT_QR_PAIRING_REQUEST, MT_QR_PAIRING_RESPONSE, Wire.fieldBytes(1, str(uuid())), SCAN_MS) + if (f == null) { + log.add("Accounts: no WhatsApp pairing answer from the glasses") + return LinkOutcome.NO_ANSWER + } + if (f.varint(2) != 1L) { + log.add("Accounts: the glasses did not pair WhatsApp, code ${f.varint(3)}") + return LinkOutcome.REFUSED + } + log.add("Accounts: WhatsApp paired on the glasses") + return LinkOutcome.LINKED + } + + /** Sends one request on svc89 and waits for the answer of [answer] type; null when none comes in [waitMs]. */ + private suspend fun ask(type: Int, answer: Int, payload: ByteArray, waitMs: Long): Wire.Fields? = lock.withLock { + coroutineScope { + val fromGlasses = Channel(Channel.UNLIMITED) + // Subscribed before the request goes out, so the answer is not missed. + val listening = launch(start = CoroutineStart.UNDISPATCHED) { + connection.serviceData.collect { if (it.serviceId == SVC && it.messageType == answer) fromGlasses.send(it) } + } + try { + connection.submit { it.sendOnService(SVC, type, payload) } + withTimeoutOrNull(waitMs) { runCatching { Wire.parse(fromGlasses.receive().payload) }.getOrNull() } + } finally { + listening.cancel() + fromGlasses.close() + } + } + } + + private fun uuid() = UUID.randomUUID().toString() + + private fun str(s: String) = s.encodeToByteArray() + + private fun flag(on: Boolean) = if (on) 1L else 0L + + companion object { + const val SVC = 89 + + // The phone's default for its account and WhatsApp waits. + private const val ANSWER_MS = 20_000L + + // Until the code is scanned; the glasses refresh it while it waits. + private const val SCAN_MS = 180_000L + + // The Meta app's source constant for the account sync. + private const val SOURCE = "306760944872162" + + private const val MT_META_SYNC_REQUEST = 5 + private const val MT_META_SYNC_RESPONSE = 13 + private const val MT_PRESENT_REQUEST = 25 + private const val MT_PRESENT_RESPONSE = 26 + private const val MT_QR_PAIRING_REQUEST = 22 + private const val MT_QR_PAIRING_RESPONSE = 23 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt new file mode 100644 index 00000000..8e9e278d --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt @@ -0,0 +1,138 @@ +package com.macedonga.visor.settings + +import androidx.compose.foundation.layout.padding +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.lifecycle.viewModelScope +import com.macedonga.visor.R +import com.macedonga.visor.connection.AccountLinking +import com.macedonga.visor.connection.LinkOutcome +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.session.LinkState +import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch + +enum class AccountRow { META, WHATSAPP } + +@HiltViewModel +class AccountsModel @Inject constructor( + val connection: GlassesConnection, + private val linking: AccountLinking, +) : ViewModel() { + + // Null until the glasses answer. + private val _meta = MutableStateFlow(null) + val meta: StateFlow = _meta.asStateFlow() + + private val _busy = MutableStateFlow(null) + val busy: StateFlow = _busy.asStateFlow() + + private val _metaResult = MutableStateFlow(null) + val metaResult: StateFlow = _metaResult.asStateFlow() + + private val _whatsappResult = MutableStateFlow(null) + val whatsappResult: StateFlow = _whatsappResult.asStateFlow() + + fun refresh() { + viewModelScope.launch { linking.metaPresent()?.let { _meta.value = it } } + } + + fun linkMeta() = task(AccountRow.META) { + _metaResult.value = linking.syncMeta() + if (_metaResult.value == LinkOutcome.LINKED) _meta.value = true + } + + fun linkWhatsApp() = task(AccountRow.WHATSAPP) { + _whatsappResult.value = linking.linkWhatsApp() + } + + private fun task(row: AccountRow, work: suspend () -> Unit) { + if (_busy.value != null) return + _busy.value = row + viewModelScope.launch { + try { + work() + } finally { + _busy.value = null + } + } + } +} + +/** The accounts the Display's apps log in with: the Meta account, and WhatsApp's link. */ +@Composable +fun AccountsScreen(model: AccountsModel = hiltViewModel()) { + val glasses by model.connection.state.collectAsStateWithLifecycle() + val ready = glasses.link == LinkState.READY + val meta by model.meta.collectAsStateWithLifecycle() + val busy by model.busy.collectAsStateWithLifecycle() + val metaResult by model.metaResult.collectAsStateWithLifecycle() + val whatsappResult by model.whatsappResult.collectAsStateWithLifecycle() + LaunchedEffect(ready) { if (ready) model.refresh() } + + SettingsPage(stringResource(R.string.accounts_title), light = 0f) { + item(key = "accounts") { + Rows( + { + RowFrame( + stringResource(R.string.accounts_meta), + onClick = null, + sub = metaResult?.let { outcomeWords(it) } ?: when (meta) { + true -> stringResource(R.string.accounts_linked) + false -> stringResource(R.string.accounts_not_linked) + null -> stringResource(R.string.settings_unknown) + }, + locked = !ready, + busy = busy == AccountRow.META, + ) { + if (meta == false) { + ActionPill( + stringResource(R.string.accounts_link), { model.linkMeta() }, + quiet = true, modifier = Modifier.padding(start = 12.dp), enabled = ready && busy == null, + ) + } + } + }, + { + RowFrame( + stringResource(R.string.accounts_whatsapp), + onClick = null, + sub = when { + busy == AccountRow.WHATSAPP -> stringResource(R.string.accounts_scan) + else -> whatsappResult?.let { outcomeWords(it) } + }, + locked = !ready, + busy = busy == AccountRow.WHATSAPP, + ) { + ActionPill( + stringResource(R.string.accounts_link), { model.linkWhatsApp() }, + quiet = true, modifier = Modifier.padding(start = 12.dp), enabled = ready && busy == null, + ) + } + }, + ) + } + } +} + +@Composable +private fun outcomeWords(outcome: LinkOutcome): String = stringResource( + when (outcome) { + LinkOutcome.LINKED -> R.string.accounts_done + LinkOutcome.LOGGED_OUT -> R.string.accounts_logged_out + LinkOutcome.UNREACHABLE -> R.string.accounts_unreachable + LinkOutcome.NO_ANSWER -> R.string.accounts_no_answer + LinkOutcome.REFUSED -> R.string.accounts_refused + }, +) diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt index ad40de60..aaa81387 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt @@ -27,6 +27,7 @@ import androidx.lifecycle.compose.LifecycleResumeEffect import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.lifecycle.viewModelScope import androidx.navigation3.runtime.NavKey +import com.macedonga.visor.AccountsKey import com.macedonga.visor.OnboardingKey import com.macedonga.visor.WifiNetworksKey import com.macedonga.visor.R @@ -247,6 +248,7 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( } }, { ValueRow(stringResource(R.string.glasses_change_owner), null, onClick = { open(OnboardingKey(login = true, address = (glasses.target?.link as? GlassesLink.Bluetooth)?.address)) }, sub = stringResource(R.string.glasses_change_owner_sub)) }, + if (hasDisplay(info?.build)) row { ValueRow(stringResource(R.string.accounts_title), null, onClick = { open(AccountsKey) }, locked = !ready) } else null, ) } } diff --git a/app/src/main/res/values-it/strings_accounts.xml b/app/src/main/res/values-it/strings_accounts.xml new file mode 100644 index 00000000..96458b2a --- /dev/null +++ b/app/src/main/res/values-it/strings_accounts.xml @@ -0,0 +1,15 @@ + + + Account + Account Meta + WhatsApp + Collega + Collegato + Non collegato + Scansiona il codice sugli occhiali da WhatsApp, Dispositivi collegati. + Fatto + Accedi prima a Meta + Meta non raggiungibile + Gli occhiali non hanno risposto + Gli occhiali lo hanno rifiutato + diff --git a/app/src/main/res/values/strings_accounts.xml b/app/src/main/res/values/strings_accounts.xml new file mode 100644 index 00000000..830ec608 --- /dev/null +++ b/app/src/main/res/values/strings_accounts.xml @@ -0,0 +1,15 @@ + + + Accounts + Meta account + WhatsApp + Link + Linked + Not linked + Scan the code on the glasses from WhatsApp, Linked devices. + Done + Log in to Meta first + Meta could not be reached + The glasses did not answer + The glasses refused it + diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/AccountToken.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/AccountToken.kt new file mode 100644 index 00000000..8093513e --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/AccountToken.kt @@ -0,0 +1,81 @@ +package com.macedonga.visor.core.identity.token + +import com.macedonga.visor.core.identity.HttpPost +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.pyUrlencode +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.jsonObject + +/** The glasses' accounts app: the Meta account the Display keeps is minted for this app id and package. */ +const val ACCOUNTS_APP_ID = "1463123261063354" +const val ACCOUNTS_PACKAGE = "com.meta.smartglass.app.accounts" + +/** What the token exchange answers for the Meta account. The access token is a secret: never log it. */ +class AccountIdentity( + val accessToken: String, + val userId: String, + val appId: String, + val accountId: String, + val name: String, + val profilePictureUrl: String, + val isEmployee: Boolean, + val isTestUser: Boolean, + val analyticsClaim: String, +) + +/** + * POST /app_token_exchange: an access token for [appId] and [packageName], minted from the user's own Meta login. The + * user id and the account fields come back with it. + */ +fun exchangeAccountToken( + userToken: Token, + appId: String = ACCOUNTS_APP_ID, + packageName: String = ACCOUNTS_PACKAGE, + http: HttpPost = UrlConnectionPost, +): AccountIdentity { + val body = pyUrlencode( + linkedMapOf( + "target_app_id" to appId, + "package_name" to packageName, + "extras" to "account_id,is_employee,is_test_user,name,profile_picture_url,analytics_claim", + ), + ).toByteArray(Charsets.US_ASCII) + val reply = try { + http.post( + "https://$DEFAULT_AR_HOST/app_token_exchange", + mapOf( + "Authorization" to "OAuth ${userToken.value}", + "Content-Type" to "application/x-www-form-urlencoded", + "User-Agent" to META_USER_AGENT, + ), + body, + ) + } catch (error: Exception) { + throw TokenError("app_token_exchange: ${error.javaClass.simpleName}: ${error.message}", unreachable = true) + } + val payload: JsonObject = try { + Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject + } catch (error: Exception) { + throw TokenError("app_token_exchange: response was not JSON") + } + payload["error"]?.let { throw TokenError("token exchange refused: ${(it as? JsonPrimitive)?.content ?: it}") } + val token = payload.text("access_token") + val userId = payload.text("user_id") + if (token.isNullOrEmpty() || userId.isNullOrEmpty()) { + throw TokenError("no access_token or user_id in the answer; keys: ${payload.sortedKeys()}") + } + return AccountIdentity( + accessToken = token, + userId = userId, + appId = payload.text("app_id") ?: appId, + accountId = payload.text("account_id").orEmpty(), + name = payload.text("name").orEmpty(), + profilePictureUrl = payload.text("profile_picture_url").orEmpty(), + isEmployee = (payload["is_employee"] as? JsonPrimitive)?.booleanOrNull == true, + isTestUser = (payload["is_test_user"] as? JsonPrimitive)?.booleanOrNull == true, + analyticsClaim = payload.text("analytics_claim").orEmpty(), + ) +} From ad33a178a022541ef6079c10ae0d75724f672b6e Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:46:29 +0200 Subject: [PATCH 63/90] Display: maps and turn-by-turn directions. Visor opens svc42 (smartglasses_navigation) on a Display as the Meta app does and drives the guidance: a route request from the Display's navigation app is routed on the phone with the OpenStreetMap community's OSRM server (routing.openstreetmap.de, walking) and answered with the route's maneuvers in the Display's 15 directions, short spoken instructions, distances and durations, and OSRM's encoded line; a navigation request starts updates every second from the phone's location (snapped position, next maneuver, remaining distance and time, on or off route), a notice at each new maneuver, a reroute after 10 s off route, then arrival or stop. Place search (svc55072) uses Android's geocoder around the phone; maps support (svc11020) gets empty answers; svc87 and svc11004 are accepted and logged. Routes and searches show on the Requests page. The Display may draw the route line from svc11004, whose format is native and not decoded, so the guidance cards are what is expected to work. Not tried on the Display yet. --- .../visor/connection/GlassesService.kt | 4 + .../macedonga/visor/connection/MapsSupport.kt | 64 ++++ .../macedonga/visor/connection/Navigation.kt | 321 ++++++++++++++++++ .../visor/connection/PhonePosition.kt | 63 ++++ .../macedonga/visor/connection/PlaceSearch.kt | 118 +++++++ .../com/macedonga/visor/connection/Routing.kt | 227 +++++++++++++ .../macedonga/visor/di/ConnectionModule.kt | 9 + .../macedonga/visor/log/RequestLogScreen.kt | 1 + .../main/res/values-it/strings_navigation.xml | 4 + .../main/res/values/strings_navigation.xml | 4 + 10 files changed, 815 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/MapsSupport.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/Navigation.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/PhonePosition.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/PlaceSearch.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/Routing.kt create mode 100644 app/src/main/res/values-it/strings_navigation.xml create mode 100644 app/src/main/res/values/strings_navigation.xml diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt index 54eeac44..d63a1de4 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt @@ -59,6 +59,9 @@ class GlassesService : Service() { @Inject lateinit var sharing: PhoneSharing + @Inject + lateinit var navigation: Navigation + @Inject lateinit var kind: LinkKind @@ -134,6 +137,7 @@ class GlassesService : Service() { connection.start(scope, targetFrom(intent)) contacts.start(scope) sharing.start(scope) + navigation.start(scope) moveToLe() holdLe() com.macedonga.visor.gallery.notePlaces(this, scope, connection, media.places) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/MapsSupport.kt b/app/src/main/kotlin/com/macedonga/visor/connection/MapsSupport.kt new file mode 100644 index 00000000..9233d1ab --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/MapsSupport.kt @@ -0,0 +1,64 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * svc11020, maps_support. The glasses ask for saved and recent places, survey and feature state, third-party routes + * and Tessa events; each request gets an empty answer with result 0, so the Display never waits on silence. Location + * signals (0) and trip feedback (1) are notes with no answer. + */ +@Singleton +class MapsSupport @Inject constructor() : Responder { + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed) return + val reply = REPLIES[message.info.messageType] ?: return + // Route details (4 -> 5) have no decoded reply, so theirs is empty. + val payload = if (message.info.messageType == REQUEST_ROUTE_DETAILS) ByteArray(0) else EMPTY_RESULT + connection.send(DataXConnection.mirror(message.info.channelId), payload, messageType = reply) + } + + companion object { + const val SVC = 11020 + private val EMPTY_RESULT = Wire.fieldVarint(1, 0) + private const val REQUEST_ROUTE_DETAILS = 4 + + /** Request type to the reply type the glasses wait for: the phone's table, reply = request + 1. */ + private val REPLIES = mapOf( + 2 to 3, + 4 to 5, + 6 to 7, + 8 to 9, + 10 to 11, + 12 to 13, + 14 to 15, + 16 to 17, + 18 to 19, + ) + } +} + +/** + * svc87 (persistent store) and svc11004 (constellation): accepted and never answered, since their schemas are native + * and undecoded. The first message of each is logged once. + */ +@Singleton +class MapsQuiet @Inject constructor(private val log: Provider) { + private val logged = ConcurrentHashMap.newKeySet() + + fun responder(svc: Int): Responder = Responder { _, _ -> + if (logged.add(svc)) log.get().add("Maps: service $svc opened, not served") + } + + companion object { + const val SVC_PERSISTENT_STORE = 87 + const val SVC_CONSTELLATION = 11004 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/Navigation.kt b/app/src/main/kotlin/com/macedonga/visor/connection/Navigation.kt new file mode 100644 index 00000000..5f12689c --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/Navigation.kt @@ -0,0 +1,321 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.settings.hasDisplay +import java.util.UUID +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlin.math.max +import kotlin.math.roundToLong +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.launch + +/** + * svc42, smartglasses_navigation, on the Display. The phone opens the channel and sends NavInitChannelRequest (type 0, + * {1 platform, Android 1}); the glasses answer (type 1), then both sides send NavMessage (type 2) {1 msgType, and a + * oneof whose case is the field number}. The glasses ask for a route (ROUTE_REQUEST, 0) and for guidance (NAV_REQUEST, + * 2); Visor answers with ROUTE_RESPONSE (1), then NAV_UPDATE (3) each second with the phone's snapped position, a + * MANEUVER_NOTIFICATION (8) per turn, REROUTE_UPDATE (7) after a re-route, and KEEPALIVE (17) on a timer. The route + * is OpenStreetMap's (see [OsrmRouter]); its line goes out as that router's encoded polyline. The map tiles are the + * glasses' own. + */ +@Singleton +class Navigation @Inject constructor( + private val glasses: Provider, + private val position: PhonePosition, + private val router: OsrmRouter, + private val requests: RequestLog, + private val log: Provider, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default + CoroutineExceptionHandler { _, e -> log.get().add("Navigation: failed: $e") }) + + @Volatile + private var session: Session? = null + + /** Opens the channel once a Display is linked, and drops the session when the link goes. */ + fun start(owner: CoroutineScope) { + owner.launch { + glasses.get().state.map { it.link == LinkState.READY && hasDisplay(it.info?.build) }.distinctUntilChanged().collect { ready -> + if (ready) { + glasses.get().submit { it.sendOnService(SVC, MT_INIT_REQUEST, Wire.fieldVarint(1, PLATFORM_ANDROID)) } + } else { + endSession() + } + } + } + } + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed) return endSession() + if (message.info.messageType != MT_NAV_MESSAGE) return + val f = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return + // The channel is Visor's own: the glasses answer on it without the originator bit, and Visor writes it with. + val s = session ?: Session(connection, message.info.channelId xor 0x8000).also { session = it } + when (f.varint(1).toInt()) { + MSG_ROUTE_REQUEST -> scope.launch { routeRequest(f, s) } + MSG_NAV_REQUEST -> startGuidance(s) + MSG_STOP_NAVIGATION -> stopGuidance(s, "stopped") + MSG_APP_START -> keepAlive(s) + MSG_APP_EXIT, MSG_APP_DIED -> endSession() + } + } + + private suspend fun routeRequest(f: Wire.Fields, s: Session) { + val req = Wire.parse(f.bytes(2)) + val requestId = req.bytes(2) + val place = Wire.parse(req.bytes(1)) + val geo = Wire.parse(place.bytes(4)) + val title = place.bytes(1).decodeToString().ifBlank { "place" } + val to = if (geo.has(1)) LatLng(geo.double(1), geo.double(2)) else null + val logId = requests.start("route://$title", SOURCE_MAPS, "walking") + if (s.guiding) return answerRoute(s, requestId, null, "Error: Navigation in progress.", logId) + val from = position.current() + if (to == null || from == null) return answerRoute(s, requestId, null, "No phone location or destination", logId) + val route = try { + router.route(from, to) + } catch (e: Exception) { + return answerRoute(s, requestId, null, "No route found", logId, e.message) + } + s.route = route + s.destination = to + s.title = title + s.spot = Spot(route.line[0], 0, 0.0) + answerRoute(s, requestId, route, null, logId) + } + + private suspend fun answerRoute(s: Session, requestId: ByteArray, route: Route?, error: String?, logId: Long, detail: String? = null) { + val body = if (route != null) { + Wire.fieldVarint(1, RESULT_SUCCESS) + Wire.fieldVarint(2, SOURCE_GLASS) + Wire.fieldBytes(3, routeDetails(route)) + Wire.fieldBytes(6, requestId) + } else { + Wire.fieldVarint(1, RESULT_ERROR) + Wire.fieldVarint(2, SOURCE_MWA) + Wire.fieldBytes(4, (error ?: "").encodeToByteArray()) + Wire.fieldBytes(6, requestId) + } + s.mux.send(s.reply, nav(MSG_ROUTE_RESPONSE, CASE_ROUTE_RESPONSE, body), MT_NAV_MESSAGE) + if (route != null) { + requests.finish(logId, "${route.steps.size} steps, ${(route.metres / 100).roundToLong() / 10.0} km", failed = false) + log.get().add("Navigation: route to ${s.title}, ${route.steps.size} maneuvers") + } else { + requests.finish(logId, detail ?: error.orEmpty(), failed = true) + log.get().add("Navigation: no route: ${error.orEmpty()}") + } + } + + private suspend fun startGuidance(s: Session) { + val route = s.route ?: return log.get().add("Navigation: guidance asked with no route") + if (s.guiding) return + s.guiding = true + s.notified = -1 + s.offTicks = 0 + s.logId = requests.start("guide://${s.title}", SOURCE_MAPS, "walking") + position.use() + s.spot = Spot(route.line[0], 0, 0.0) + sendUpdate(s, route, STATE_STARTED, NAVIGATOR_ON_ROUTE, 0) + s.ticker = scope.launch { + while (true) { + delay(TICK_MS) + if (!tick(s)) break + } + } + } + + /** One second of guidance. Returns false once guidance has ended. */ + private suspend fun tick(s: Session): Boolean { + if (!s.guiding) return false + val route = s.route ?: return false + val here = position.current() + if (here == null) { + sendUpdate(s, route, STATE_IN_PROGRESS, NAVIGATOR_BAD_GPS, max(s.notified, 0)) + return true + } + val snap = route.snap(here) + val offRoute = snap.off > OFF_ROUTE_M + s.offTicks = if (offRoute) s.offTicks + 1 else 0 + if (s.offTicks >= REROUTE_TICKS) { + reroute(s, here) + return s.guiding + } + s.spot = Spot(snap.at, snap.index, snap.along) + if (route.metres - snap.along < ARRIVE_M) { + sendUpdate(s, route, STATE_ARRIVED, NAVIGATOR_STOPPED, route.steps.lastIndex) + finish(s, "arrived") + return false + } + val steps = route.steps + val next = steps.indexOfFirst { snap.along < it.along + PASSED_M }.let { if (it < 0) steps.lastIndex else it } + sendUpdate(s, route, STATE_IN_PROGRESS, if (offRoute) NAVIGATOR_OFF_ROUTE else NAVIGATOR_ON_ROUTE, next) + if (next != s.notified) { + s.notified = next + s.mux.send(s.reply, nav(MSG_MANEUVER_NOTIFICATION, CASE_MANEUVER, notification(steps[next])), MT_NAV_MESSAGE) + } + return true + } + + private suspend fun reroute(s: Session, here: LatLng) { + val to = s.destination ?: return + s.offTicks = 0 + sendUpdate(s, s.route ?: return, STATE_IN_PROGRESS, NAVIGATOR_REROUTING, max(s.notified, 0)) + val fresh = try { + router.route(here, to) + } catch (e: Exception) { + log.get().add("Navigation: reroute failed: ${e.message}") + s.mux.send(s.reply, nav(MSG_REROUTE_UPDATE, CASE_REROUTE, Wire.fieldVarint(1, RESULT_ERROR)), MT_NAV_MESSAGE) + return + } + s.route = fresh + s.spot = Spot(fresh.line[0], 0, 0.0) + s.notified = -1 + s.mux.send(s.reply, nav(MSG_REROUTE_UPDATE, CASE_REROUTE, Wire.fieldVarint(1, RESULT_SUCCESS) + Wire.fieldBytes(2, routeDetails(fresh))), MT_NAV_MESSAGE) + log.get().add("Navigation: rerouted, ${fresh.steps.size} maneuvers") + } + + private suspend fun stopGuidance(s: Session, outcome: String) { + if (!s.guiding) return + s.ticker?.cancel() + s.ticker = null + s.route?.let { sendUpdate(s, it, STATE_STOPPED, NAVIGATOR_STOPPED, max(s.notified, 0)) } + finish(s, outcome) + } + + /** Ends guidance's bookkeeping. Only called while guidance runs. */ + private fun finish(s: Session, outcome: String) { + s.guiding = false + position.release() + requests.finish(s.logId, outcome, failed = false) + log.get().add("Navigation: $outcome") + } + + private fun keepAlive(s: Session) { + s.keepAlive?.cancel() + s.keepAlive = scope.launch { + while (true) { + s.mux.send(s.reply, nav(MSG_KEEPALIVE), MT_NAV_MESSAGE) + delay(KEEPALIVE_MS) + } + } + } + + private suspend fun endSession() { + val s = session ?: return + session = null + s.keepAlive?.cancel() + runCatching { stopGuidance(s, "ended by the link") } + } + + /** NAV_UPDATE for the spot, with the next maneuver's distance and time, as the glasses' guidance needs them. */ + private suspend fun sendUpdate(s: Session, route: Route, state: Long, navigator: Long, maneuver: Int) { + val spot = s.spot + val remaining = max(0.0, route.metres - spot.along) + val next = route.steps.getOrNull(maneuver) + val toNext = max(0.0, (next?.along ?: route.metres) - spot.along) + val perMetre = route.secondsPerMetre + val body = Wire.fieldVarint(1, state) + + Wire.fieldBytes(2, Wire.fieldDouble(1, spot.at.lat) + Wire.fieldDouble(2, spot.at.lng)) + + Wire.fieldVarint(3, spot.index.toLong()) + + Wire.fieldVarint(4, maneuver.toLong()) + + Wire.fieldVarint(5, (remaining * perMetre).roundToLong()) + + Wire.fieldVarint(6, remaining.roundToLong()) + + Wire.fieldVarint(7, (toNext * perMetre).roundToLong()) + + Wire.fieldVarint(8, toNext.roundToLong()) + + Wire.fieldBytes(10, (next?.text ?: "").encodeToByteArray()) + + Wire.fieldVarint(13, navigator) + s.mux.send(s.reply, nav(MSG_NAV_UPDATE, CASE_NAV_UPDATE, body), MT_NAV_MESSAGE) + } + + private fun routeDetails(r: Route): ByteArray { + var b = Wire.fieldVarint(1, ROUTE_ID) + Wire.fieldVarint(2, r.seconds.roundToLong()) + Wire.fieldVarint(3, r.metres.roundToLong()) + for (step in r.steps) b += Wire.fieldBytes(4, maneuver(step)) + return b + Wire.fieldBytes(5, r.encoded.encodeToByteArray()) + } + + private fun maneuver(step: Step): ByteArray = + Wire.fieldVarint(1, step.direction.toLong()) + + Wire.fieldBytes(2, step.street.encodeToByteArray()) + + Wire.fieldVarint(3, step.metres.toLong()) + + Wire.fieldVarint(4, step.seconds.toLong()) + + Wire.fieldBytes(5, step.text.encodeToByteArray()) + + private fun notification(step: Step): ByteArray = + Wire.fieldBytes(1, maneuver(step)) + Wire.fieldBytes(2, "HNNAV-${UUID.randomUUID().toString().take(8)}".encodeToByteArray()) + + private fun nav(type: Int, case: Int = 0, body: ByteArray = ByteArray(0)): ByteArray = + Wire.fieldVarint(1, type.toLong()) + if (case == 0) ByteArray(0) else Wire.fieldBytes(case, body) + + private class Spot(val at: LatLng, val index: Int, val along: Double) + + /** The channel's state: where replies go, the route and the guidance running on it. */ + private class Session(val mux: DataXConnection, val reply: Int) { + @Volatile var route: Route? = null + @Volatile var destination: LatLng? = null + @Volatile var title = "" + @Volatile var guiding = false + @Volatile var logId = 0L + @Volatile var spot = Spot(LatLng(0.0, 0.0), 0, 0.0) + @Volatile var notified = -1 + @Volatile var offTicks = 0 + var ticker: Job? = null + var keepAlive: Job? = null + } + + companion object { + const val SVC = 42 + private const val SOURCE_MAPS = -6 + private const val MT_INIT_REQUEST = 0 + private const val MT_NAV_MESSAGE = 2 + private const val PLATFORM_ANDROID = 1L + + private const val MSG_ROUTE_REQUEST = 0 + private const val MSG_ROUTE_RESPONSE = 1 + private const val MSG_NAV_REQUEST = 2 + private const val MSG_NAV_UPDATE = 3 + private const val MSG_STOP_NAVIGATION = 6 + private const val MSG_REROUTE_UPDATE = 7 + private const val MSG_MANEUVER_NOTIFICATION = 8 + private const val MSG_APP_START = 13 + private const val MSG_APP_EXIT = 14 + private const val MSG_APP_DIED = 15 + private const val MSG_KEEPALIVE = 17 + + private const val CASE_ROUTE_RESPONSE = 3 + private const val CASE_NAV_UPDATE = 5 + private const val CASE_REROUTE = 7 + private const val CASE_MANEUVER = 8 + + private const val RESULT_SUCCESS = 0L + private const val RESULT_ERROR = 1L + private const val SOURCE_GLASS = 0L + private const val SOURCE_MWA = 1L + private const val ROUTE_ID = 1L + + private const val STATE_STARTED = 0L + private const val STATE_IN_PROGRESS = 1L + private const val STATE_STOPPED = 2L + private const val STATE_ARRIVED = 3L + + private const val NAVIGATOR_ON_ROUTE = 1L + private const val NAVIGATOR_OFF_ROUTE = 2L + private const val NAVIGATOR_BAD_GPS = 3L + private const val NAVIGATOR_REROUTING = 4L + private const val NAVIGATOR_STOPPED = 5L + + /** The Meta app's keepalive period was not found; this is a guess. */ + private const val KEEPALIVE_MS = 5_000L + private const val TICK_MS = 1_000L + private const val OFF_ROUTE_M = 50.0 + private const val REROUTE_TICKS = 10 + private const val PASSED_M = 15.0 + private const val ARRIVE_M = 25.0 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhonePosition.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhonePosition.kt new file mode 100644 index 00000000..d9f5d526 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhonePosition.kt @@ -0,0 +1,63 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.annotation.SuppressLint +import android.content.Context +import android.content.pm.PackageManager +import android.location.Location +import android.location.LocationListener +import android.location.LocationManager +import android.os.Looper +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Singleton + +/** + * The phone's position for navigation and place search, from the provider [LocationShare] uses. Updates run only + * while guidance does; otherwise the system's last known fix answers. + */ +@Singleton +class PhonePosition @Inject constructor(@ApplicationContext private val context: Context) { + private val manager = context.getSystemService(LocationManager::class.java) + private val listener = LocationListener { last = it } + + @Volatile + private var last: Location? = null + private var users = 0 + + /** Asks for updates while guidance runs. Each call is paired with [release]. */ + @Synchronized + @SuppressLint("MissingPermission") + fun use() { + if (users++ == 0 && manager != null && granted()) { + manager.requestLocationUpdates(provider(manager), INTERVAL_MS, 0f, listener, Looper.getMainLooper()) + } + } + + @Synchronized + fun release() { + if (users > 0 && --users == 0) manager?.removeUpdates(listener) + } + + /** The latest fix while updates run, else the system's last known one. Null without the permission. */ + @SuppressLint("MissingPermission") + fun current(): LatLng? { + if (manager == null || !granted()) return null + val fix = if (users > 0) last else null + return (fix ?: manager.getLastKnownLocation(provider(manager)))?.let { LatLng(it.latitude, it.longitude) } + } + + private fun granted(): Boolean = + listOf(Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_COARSE_LOCATION) + .any { context.checkSelfPermission(it) == PackageManager.PERMISSION_GRANTED } + + private fun provider(m: LocationManager): String = when { + m.allProviders.contains(LocationManager.FUSED_PROVIDER) -> LocationManager.FUSED_PROVIDER + m.isProviderEnabled(LocationManager.GPS_PROVIDER) -> LocationManager.GPS_PROVIDER + else -> LocationManager.NETWORK_PROVIDER + } + + private companion object { + const val INTERVAL_MS = 1_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PlaceSearch.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PlaceSearch.kt new file mode 100644 index 00000000..a7fbe50b --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PlaceSearch.kt @@ -0,0 +1,118 @@ +package com.macedonga.visor.connection + +import android.content.Context +import android.location.Address +import android.location.Geocoder +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.IOException +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlin.math.cos +import kotlin.math.roundToLong +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +/** + * svc55072, smartglasses_place_search: the glasses search for a place (PlaceSearchRequest, types 0 and 1 {1 query, + * 2 radius in metres, 4 requestId}) and get PlaceSearchResponse (type 2) with the places nearest the phone, from the + * phone's own geocoder. Media requests (3) get an empty PlaceMediaResponse (4). + */ +@Singleton +class PlaceSearch @Inject constructor( + @ApplicationContext private val context: Context, + private val position: PhonePosition, + private val requests: RequestLog, + private val log: Provider, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Place search: failed: $e") }) + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed) return + val reply = DataXConnection.mirror(message.info.channelId) + when (message.info.messageType) { + MT_SEARCH, MT_SEARCH_REQUEST -> { + val req = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return + scope.launch { connection.send(reply, search(req), messageType = MT_SEARCH_RESPONSE) } + } + MT_MEDIA_REQUEST -> connection.send(reply, ByteArray(0), messageType = MT_MEDIA_RESPONSE) + } + } + + private suspend fun search(req: Wire.Fields): ByteArray = withContext(Dispatchers.IO) { + val query = req.bytes(1).decodeToString() + val radius = req.float(2).takeIf { it > 0f }?.toDouble() ?: DEFAULT_RADIUS_M + val logId = requests.start("place://$query", SOURCE_MAPS) + val here = position.current() + if (!Geocoder.isPresent()) { + requests.finish(logId, "no geocoder on the phone", failed = true) + return@withContext ERROR_NO_GEOCODER + } + val found = try { + lookup(query, here, radius) + } catch (e: IOException) { + requests.finish(logId, e.message ?: "search failed", failed = true) + return@withContext ERROR_NETWORK + } catch (e: IllegalArgumentException) { + requests.finish(logId, e.message ?: "search failed", failed = true) + return@withContext ERROR_NETWORK + } + requests.finish(logId, "${found.size} places", failed = false) + log.get().add("Place search: \"$query\", ${found.size} places") + var body = Wire.fieldVarint(1, RESULT_SUCCESS) + for (address in found) body += Wire.fieldBytes(2, place(address, here)) + body + } + + /** Bounded to [radius] around the phone when it has a fix. */ + @Suppress("DEPRECATION") + private fun lookup(query: String, here: LatLng?, radius: Double): List
{ + val geocoder = Geocoder(context) + if (here == null) return geocoder.getFromLocationName(query, MAX_RESULTS).orEmpty() + val dLat = radius / METRES_PER_DEGREE + val dLng = radius / (METRES_PER_DEGREE * cos(Math.toRadians(here.lat)).coerceAtLeast(MIN_COS)) + return geocoder.getFromLocationName(query, MAX_RESULTS, here.lat - dLat, here.lng - dLng, here.lat + dLat, here.lng + dLng) + .orEmpty() + .sortedBy { haversine(here, LatLng(it.latitude, it.longitude)) } + } + + /** PlaceSearchResult {1 id, 2 title, 3 Address {1 street}, 4 GeoCoordinate {1 lat, 2 lng}, 13 walking {1 metres}}. */ + private fun place(a: Address, here: LatLng?): ByteArray { + val street = if (a.maxAddressLineIndex >= 0) a.getAddressLine(0).orEmpty() else "" + val title = a.featureName ?: a.thoroughfare ?: street + var b = Wire.fieldBytes(1, "${a.latitude},${a.longitude}".encodeToByteArray()) + + Wire.fieldBytes(2, title.encodeToByteArray()) + + Wire.fieldBytes(3, Wire.fieldBytes(1, street.encodeToByteArray())) + + Wire.fieldBytes(4, Wire.fieldDouble(1, a.latitude) + Wire.fieldDouble(2, a.longitude)) + if (here != null) { + b += Wire.fieldBytes(13, Wire.fieldVarint(1, haversine(here, LatLng(a.latitude, a.longitude)).roundToLong())) + } + return b + } + + companion object { + const val SVC = 55072 + private const val SOURCE_MAPS = -6 + private const val MT_SEARCH = 0 + private const val MT_SEARCH_REQUEST = 1 + private const val MT_SEARCH_RESPONSE = 2 + private const val MT_MEDIA_REQUEST = 3 + private const val MT_MEDIA_RESPONSE = 4 + private const val RESULT_SUCCESS = 0L + private val ERROR_NO_GEOCODER = Wire.fieldVarint(1, 1) + Wire.fieldBytes(3, "No geocoder on the phone".encodeToByteArray()) + private val ERROR_NETWORK = Wire.fieldVarint(1, 1) + Wire.fieldBytes(3, "Search failed".encodeToByteArray()) + Wire.fieldVarint(4, 1) + private const val DEFAULT_RADIUS_M = 5_000.0 + private const val METRES_PER_DEGREE = 111_320.0 + private const val MIN_COS = 0.01 + private const val MAX_RESULTS = 10 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/Routing.kt b/app/src/main/kotlin/com/macedonga/visor/connection/Routing.kt new file mode 100644 index 00000000..6c0970ae --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/Routing.kt @@ -0,0 +1,227 @@ +package com.macedonga.visor.connection + +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import javax.inject.Inject +import javax.inject.Singleton +import kotlin.math.asin +import kotlin.math.cos +import kotlin.math.hypot +import kotlin.math.roundToInt +import kotlin.math.sin +import kotlin.math.sqrt +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.json.JSONObject + +data class LatLng(val lat: Double, val lng: Double) + +/** The Display's maneuver directions (the phone's D42 enum). */ +object Dir { + const val DEPART = 0 + const val ARRIVE = 1 + const val SHARP_LEFT = 2 + const val LEFT = 3 + const val SLIGHT_LEFT = 4 + const val CONTINUE = 5 + const val SLIGHT_RIGHT = 6 + const val RIGHT = 7 + const val SHARP_RIGHT = 8 + const val LEFT_FORK = 9 + const val MIDDLE_FORK = 10 + const val RIGHT_FORK = 11 + const val U_TURN = 12 + const val ARRIVE_LEFT = 13 + const val ARRIVE_RIGHT = 14 +} + +/** One maneuver. [metres] and [seconds] are the leg that leads into it; [along] is where it starts, in metres. */ +class Step(val direction: Int, val street: String, val metres: Int, val seconds: Int, val along: Double, val text: String) + +/** A snapped point on the line: where it is, the index of its segment, and metres from the start. */ +class Snap(val at: LatLng, val index: Int, val along: Double, val off: Double) + +/** A route from OSRM. [encoded] is the polyline as OSRM gives it (precision 5), passed on unchanged. */ +class Route(val encoded: String, val line: List, val steps: List, val metres: Double, val seconds: Double) { + /** Metres from the start to each point of [line], scaled to the route total OSRM gives. */ + private val along = DoubleArray(line.size).also { a -> + for (i in 1 until line.size) a[i] = a[i - 1] + haversine(line[i - 1], line[i]) + val total = a.lastOrNull() ?: 0.0 + if (total > 0) for (i in a.indices) a[i] *= metres / total + } + + val secondsPerMetre: Double get() = if (metres > 0) seconds / metres else 0.0 + + /** The point of the line nearest [p], on a flat map around [p], which is exact enough at route scale. */ + fun snap(p: LatLng): Snap { + val kx = cos(Math.toRadians(p.lat)) * METRES_PER_DEGREE + var best = Snap(line[0], 0, 0.0, Double.MAX_VALUE) + for (i in 0 until line.size - 1) { + val a = line[i] + val b = line[i + 1] + val ax = (a.lng - p.lng) * kx + val ay = (a.lat - p.lat) * METRES_PER_DEGREE + val dx = (b.lng - p.lng) * kx - ax + val dy = (b.lat - p.lat) * METRES_PER_DEGREE - ay + val len2 = dx * dx + dy * dy + val t = if (len2 == 0.0) 0.0 else (-(ax * dx + ay * dy) / len2).coerceIn(0.0, 1.0) + val off = hypot(ax + t * dx, ay + t * dy) + if (off < best.off) { + best = Snap( + LatLng(a.lat + t * (b.lat - a.lat), a.lng + t * (b.lng - a.lng)), + i, + along[i] + t * (along[i + 1] - along[i]), + off, + ) + } + } + return best + } +} + +enum class Mode(val path: String, val profile: String) { + WALK("foot", "foot"), + BIKE("bike", "bike"), + CAR("car", "driving"), +} + +/** Routes from the OpenStreetMap community's public OSRM server. */ +@Singleton +class OsrmRouter @Inject constructor() { + /** Throws when the network fails or the server finds no route. */ + suspend fun route(from: LatLng, to: LatLng, mode: Mode = Mode.WALK): Route = withContext(Dispatchers.IO) { + val url = URL( + "https://routing.openstreetmap.de/routed-${mode.path}/route/v1/${mode.profile}/" + + "${from.lng},${from.lat};${to.lng},${to.lat}?steps=true&overview=full&geometries=polyline", + ) + val conn = url.openConnection() as HttpURLConnection + try { + conn.setRequestProperty("User-Agent", "Visor") + conn.connectTimeout = TIMEOUT_MS + conn.readTimeout = TIMEOUT_MS + if (conn.responseCode != 200) throw IOException("routing server answered ${conn.responseCode}") + parse(conn.inputStream.bufferedReader().use { it.readText() }) + } finally { + conn.disconnect() + } + } + + private fun parse(body: String): Route { + val json = JSONObject(body) + if (json.optString("code") != "Ok") throw IOException(json.optString("code", "no route")) + val r = json.getJSONArray("routes").getJSONObject(0) + val steps = r.getJSONArray("legs").getJSONObject(0).getJSONArray("steps") + val out = ArrayList() + var along = 0.0 + var legMetres = 0.0 + var legSeconds = 0.0 + for (i in 0 until steps.length()) { + val s = steps.getJSONObject(i) + val m = s.getJSONObject("maneuver") + val name = s.optString("name") + val dir = direction(m.optString("type"), m.optString("modifier")) + out += Step(dir, name, legMetres.roundToInt(), legSeconds.roundToInt(), along, words(dir, name)) + legMetres = s.getDouble("distance") + legSeconds = s.getDouble("duration") + along += legMetres + } + val encoded = r.getString("geometry") + return Route(encoded, decodePolyline(encoded), out, r.getDouble("distance"), r.getDouble("duration")) + } + + private fun direction(type: String, modifier: String): Int { + val side = when (modifier) { + "sharp left" -> Dir.SHARP_LEFT + "left" -> Dir.LEFT + "slight left" -> Dir.SLIGHT_LEFT + "slight right" -> Dir.SLIGHT_RIGHT + "right" -> Dir.RIGHT + "sharp right" -> Dir.SHARP_RIGHT + "uturn" -> Dir.U_TURN + else -> Dir.CONTINUE + } + val leftish = modifier.contains("left") + val rightish = modifier.contains("right") + return when (type) { + "depart" -> Dir.DEPART + "arrive" -> when { + leftish -> Dir.ARRIVE_LEFT + rightish -> Dir.ARRIVE_RIGHT + else -> Dir.ARRIVE + } + "fork", "end of road" -> when { + leftish -> Dir.LEFT_FORK + rightish -> Dir.RIGHT_FORK + else -> if (type == "fork") Dir.MIDDLE_FORK else side + } + "merge", "roundabout", "rotary", "roundabout turn", "exit roundabout", "exit rotary" -> Dir.MIDDLE_FORK + "on ramp", "off ramp" -> when { + leftish -> Dir.LEFT_FORK + rightish -> Dir.RIGHT_FORK + else -> Dir.CONTINUE + } + else -> side + } + } + + private fun words(dir: Int, street: String): String { + val onto = if (street.isBlank()) "" else " onto $street" + return when (dir) { + Dir.DEPART -> if (street.isBlank()) "Start" else "Start on $street" + Dir.ARRIVE -> "You have arrived" + Dir.ARRIVE_LEFT -> "You have arrived, on the left" + Dir.ARRIVE_RIGHT -> "You have arrived, on the right" + Dir.SHARP_LEFT -> "Turn sharp left$onto" + Dir.LEFT -> "Turn left$onto" + Dir.SLIGHT_LEFT -> "Bear left$onto" + Dir.SLIGHT_RIGHT -> "Bear right$onto" + Dir.RIGHT -> "Turn right$onto" + Dir.SHARP_RIGHT -> "Turn sharp right$onto" + Dir.U_TURN -> "Make a U-turn" + Dir.LEFT_FORK -> "Keep left$onto" + Dir.RIGHT_FORK -> "Keep right$onto" + Dir.MIDDLE_FORK -> "Go through the junction$onto" + else -> if (street.isBlank()) "Continue straight" else "Continue on $street" + } + } + + private companion object { + const val TIMEOUT_MS = 15_000 + } +} + +/** Google's encoded polyline, precision 5. */ +fun decodePolyline(s: String): List { + val out = ArrayList() + var i = 0 + var lat = 0 + var lng = 0 + fun value(): Int { + var result = 0 + var shift = 0 + var b: Int + do { + b = s[i++].code - 63 + result = result or ((b and 0x1f) shl shift) + shift += 5 + } while (b >= 0x20) + return if ((result and 1) != 0) (result shr 1).inv() else result shr 1 + } + while (i < s.length) { + lat += value() + lng += value() + out += LatLng(lat / 1e5, lng / 1e5) + } + return out +} + +fun haversine(a: LatLng, b: LatLng): Double { + val dLat = Math.toRadians(b.lat - a.lat) + val dLng = Math.toRadians(b.lng - a.lng) + val h = sin(dLat / 2) * sin(dLat / 2) + cos(Math.toRadians(a.lat)) * cos(Math.toRadians(b.lat)) * sin(dLng / 2) * sin(dLng / 2) + return 2 * EARTH_RADIUS_M * asin(sqrt(h)) +} + +private const val EARTH_RADIUS_M = 6_371_000.0 +private const val METRES_PER_DEGREE = 111_320.0 diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 2da39456..16030a83 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -154,6 +154,10 @@ object ConnectionModule { media: com.macedonga.visor.connection.PhoneMedia, captureState: com.macedonga.visor.connection.CaptureState, leases: com.macedonga.visor.connection.LeaseService, + navigation: com.macedonga.visor.connection.Navigation, + placeSearch: com.macedonga.visor.connection.PlaceSearch, + mapsSupport: com.macedonga.visor.connection.MapsSupport, + mapsQuiet: com.macedonga.visor.connection.MapsQuiet, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -178,6 +182,11 @@ object ConnectionModule { com.macedonga.visor.connection.PhoneMedia.SVC to media, com.macedonga.visor.connection.CaptureState.SVC to captureState, com.macedonga.visor.connection.LeaseService.SVC to leases, + com.macedonga.visor.connection.Navigation.SVC to navigation, + com.macedonga.visor.connection.PlaceSearch.SVC to placeSearch, + com.macedonga.visor.connection.MapsSupport.SVC to mapsSupport, + com.macedonga.visor.connection.MapsQuiet.SVC_PERSISTENT_STORE to mapsQuiet.responder(com.macedonga.visor.connection.MapsQuiet.SVC_PERSISTENT_STORE), + com.macedonga.visor.connection.MapsQuiet.SVC_CONSTELLATION to mapsQuiet.responder(com.macedonga.visor.connection.MapsQuiet.SVC_CONSTELLATION), SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), diff --git a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt index 949663c0..a44579b4 100644 --- a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt @@ -95,6 +95,7 @@ private fun RequestRow(r: GlassesRequest) { -3 -> stringResource(R.string.requests_location) -4 -> stringResource(R.string.requests_sms) -5 -> stringResource(R.string.requests_calls) + -6 -> stringResource(R.string.navigation_requests_maps) else -> stringResource(R.string.requests_source, r.source) }, r.options.ifEmpty { null }, diff --git a/app/src/main/res/values-it/strings_navigation.xml b/app/src/main/res/values-it/strings_navigation.xml new file mode 100644 index 00000000..d7a90c5d --- /dev/null +++ b/app/src/main/res/values-it/strings_navigation.xml @@ -0,0 +1,4 @@ + + + mappe + diff --git a/app/src/main/res/values/strings_navigation.xml b/app/src/main/res/values/strings_navigation.xml new file mode 100644 index 00000000..186630f1 --- /dev/null +++ b/app/src/main/res/values/strings_navigation.xml @@ -0,0 +1,4 @@ + + + maps + From ffa6d619d5c3c5a91e3471a1e6fc18b5a7ceefb2 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:50:52 +0200 Subject: [PATCH 64/90] Display: web apps. Visor serves svc55074 for the Display's browser as the Meta app does: web app management (type 15: create, update, delete, list and the developer mode status, answered with 16) against Visor's own list, cookies (11, answered with 9) and Web Bluetooth scans (17, answered with 20 and no devices). The list goes to the glasses as dev_mode_web_app_ entries of the partner_app_settings setting (a PartnerAppSettings map: title and URL), with developer_mode on. The Display's settings get a Web apps page: add a page by name and https address, remove it, open it on the glasses (a svc56 lease for the browser's service, then OPEN on svc55074), and a row that turns on the browser's MobileConfig switches for web apps (web_on_hn web_app and web_app_connections) when they are off, which needs a restart. The settings catalogue now names partner_app_settings' item fields. Not tried on the Display yet. --- .../kotlin/com/macedonga/visor/VisorNav.kt | 2 + .../com/macedonga/visor/connection/WebApps.kt | 263 ++++++++++++++++++ .../macedonga/visor/di/ConnectionModule.kt | 2 + .../macedonga/visor/settings/GlassesPage.kt | 2 + .../macedonga/visor/settings/WebAppsScreen.kt | 157 +++++++++++ .../main/res/values-it/strings_webapps.xml | 21 ++ app/src/main/res/values/strings_webapps.xml | 21 ++ .../visor/core/settings/settings_ui.json | 26 ++ 8 files changed, 494 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt create mode 100644 app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt create mode 100644 app/src/main/res/values-it/strings_webapps.xml create mode 100644 app/src/main/res/values/strings_webapps.xml diff --git a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt index 946ecf98..a8984cdc 100644 --- a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt +++ b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt @@ -168,6 +168,7 @@ data object WifiNetworksKey : NavKey @Serializable data object AccountsKey : NavKey +data object WebAppsKey : NavKey /** The app's graph: the home, and every screen it opens. */ @Composable @@ -291,6 +292,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) entry { com.macedonga.visor.display.CaptionsScreen() } entry { com.macedonga.visor.settings.WifiNetworksScreen() } entry { com.macedonga.visor.settings.AccountsScreen() } + entry { com.macedonga.visor.settings.WebAppsScreen() } entry { MetaAiScreen( onSettings = { backStack.add(AssistantSettingsKey) }, diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt b/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt new file mode 100644 index 00000000..824ad1ac --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt @@ -0,0 +1,263 @@ +package com.macedonga.visor.connection + +import android.content.Context +import android.net.Uri +import android.util.Base64 +import android.webkit.CookieManager +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.media.ConfigState +import com.macedonga.visor.core.media.GlassesConfig +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.core.settings.GlassesSettings +import com.macedonga.visor.core.settings.ProtoField +import com.macedonga.visor.core.settings.encodeProto +import com.macedonga.visor.core.settings.parseProto +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.File +import java.util.UUID +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.withTimeoutOrNull +import org.json.JSONArray +import org.json.JSONObject + +data class WebApp(val key: String, val name: String, val url: String) + +enum class AddResult { ADDED, EXISTS, NOT_HTTPS, FULL } + +/** Whether the glasses open a page from the phone: the two MobileConfig switches the browser checks. */ +enum class OpenGate { UNREAD, UNAVAILABLE, OFF, ON } + +val OPEN_GATES = listOf("web_on_hn.web_app", "web_on_hn.web_app_connections") + +fun gateOf(cfg: ConfigState): OpenGate = when { + cfg.params.isEmpty() -> OpenGate.UNREAD + OPEN_GATES.any { it !in cfg.params } -> OpenGate.UNAVAILABLE + OPEN_GATES.all { cfg.values[it] == true } -> OpenGate.ON + else -> OpenGate.OFF +} + +/** + * svc55074, the browser's phone service: the Display's web-app manager (type 15: CREATE 1, DELETE 2, LIST 3, UPDATE 4, + * OPEN 5, DEV MODE STATUS 6, answered with 16), cookies (11, answered with 9) and a BLE scan (17, answered with 20 and + * no devices). Visor keeps its own list in a file and writes it to the glasses' partner_app_settings, one + * dev_mode_web_app_ entry each, as the Meta app does. [open] sends OPEN to the glasses. + */ +@Singleton +class WebApps @Inject constructor( + @ApplicationContext context: Context, + private val connection: Provider, + private val settings: Provider, + private val log: Provider, +) : Responder { + private val file = File(context.filesDir, "web_apps.json") + private val _apps = MutableStateFlow(load()) + val apps: StateFlow> = _apps.asStateFlow() + private val lock = Any() + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.payload.isEmpty() && message.info.flags.isClosed) return + val reply = DataXConnection.mirror(message.info.channelId) + val f = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return + when (message.info.messageType) { + MT_MANAGE -> connection.send(reply, manage(f), messageType = MT_MANAGE_RESPONSE) + MT_GET_COOKIES -> connection.send(reply, cookies(f), messageType = MT_SET_COOKIES) + // Visor scans for nothing: the page gets an answer with no devices. + MT_BLE_SCAN_START -> connection.send(reply, Wire.fieldBytes(1, f.bytes(1)), messageType = MT_BLE_SCAN_RESPONSE) + else -> log.get().add("Web apps: type ${message.info.messageType} not served") + } + } + + /** Add a page, from the Display's CREATE or the Add sheet. The glasses take it with [push]. */ + fun add(name: String, url: String): AddResult { + val uri = Uri.parse(url) + if (uri.scheme != "https" || uri.host.isNullOrEmpty()) return AddResult.NOT_HTTPS + synchronized(lock) { + val list = _apps.value + if (list.any { it.url == url }) return AddResult.EXISTS + if (list.size >= MAX_APPS) return AddResult.FULL + val key = generateSequence(0) { it + 1 }.map { "$PREFIX$it" }.first { k -> list.none { it.key == k } } + save(list + WebApp(key, name.ifEmpty { uri.host.orEmpty() }, url)) + } + log.get().add("Web apps: added ${uri.host}") + return AddResult.ADDED + } + + fun remove(key: String): Boolean = synchronized(lock) { + val list = _apps.value + val rest = list.filterNot { it.key == key } + if (rest.size == list.size) return false + save(rest) + true + } + + private fun rename(key: String, name: String): Boolean = synchronized(lock) { + if (name.isEmpty() || _apps.value.none { it.key == key }) return false + save(_apps.value.map { if (it.key == key) it.copy(name = name) else it }) + true + } + + /** + * Write the list into the glasses' partner_app_settings, with developer mode on. False when the glasses cannot take + * it now (not connected, or their settings not read yet). + */ + fun push(): Boolean { + val glasses = connection.get() + val s = settings.get() + if (glasses.state.value.link != LinkState.READY || s.state.value.readAt == null) return false + val value = Base64.encodeToString(partnerSettings(s.value(SETTING).orEmpty(), _apps.value), Base64.NO_WRAP) + val sent = s.write(SETTING, value) && s.write(DEVELOPER_MODE, "true") + if (!sent) log.get().add("Web apps: the list did not go to the glasses") + return sent + } + + /** Open [app] on the glasses: a lease on the browser's service (svc56, feature 19), then OPEN on svc55074. */ + suspend fun open(app: WebApp): Boolean { + val glasses = connection.get() + if (glasses.state.value.link != LinkState.READY) return false + coroutineScope { + val bound = async(start = CoroutineStart.UNDISPATCHED) { + withTimeoutOrNull(BOUND_MS) { + glasses.serviceData.first { + it.serviceId == SVC_LEASE && it.messageType == MT_LIFECYCLE && + runCatching { Wire.parse(it.payload).varint(2) }.getOrNull() == STATE_BOUND + } + } + } + glasses.submit { it.sendOnService(SVC_LEASE, MT_LEASE, Wire.fieldVarint(1, FEATURE_WEB_APP)) } + if (bound.await() == null) log.get().add("Web apps: the browser's service did not report bound; sending OPEN anyway") + } + delay(REGISTER_MS) + val request = Wire.fieldBytes(1, UUID.randomUUID().toString().encodeToByteArray()) + + Wire.fieldVarint(2, ACTION_OPEN) + + Wire.fieldBytes(3, app.name.encodeToByteArray()) + + Wire.fieldBytes(4, app.url.encodeToByteArray()) + glasses.submit { mux -> mux.send(mux.openChannel(SVC), request, messageType = MT_MANAGE, serviceId = SVC) } + log.get().add("Web apps: OPEN sent for ${app.name}") + return true + } + + private fun manage(f: Wire.Fields): ByteArray { + val id = f.bytes(1) + fun answer(error: String? = null, apps: List = emptyList()): ByteArray = + Wire.fieldBytes(1, id) + + Wire.fieldVarint(2, if (error == null) 1L else 0L) + + (error?.let { Wire.fieldBytes(3, it.encodeToByteArray()) } ?: ByteArray(0)) + + apps.fold(ByteArray(0)) { out, app -> out + Wire.fieldBytes(4, info(app)) } + val key = f.bytes(5).decodeToString() + val name = f.bytes(3).decodeToString() + return when (f.varint(2)) { + ACTION_CREATE -> when (add(name, f.bytes(4).decodeToString())) { + AddResult.ADDED, AddResult.EXISTS -> { + push() + answer() + } + AddResult.NOT_HTTPS -> answer("Only https pages can be web apps") + AddResult.FULL -> answer("Up to $MAX_APPS web apps") + } + ACTION_DELETE -> if (remove(key)) { push(); answer() } else answer("No such web app") + ACTION_LIST -> answer(apps = _apps.value) + ACTION_UPDATE -> if (rename(key, name)) { push(); answer() } else answer("No such web app") + // The glasses open pages through the phone, not the other way round. + ACTION_OPEN -> answer("Web apps open from the phone") + // Visor keeps the list, so developer mode is always on for it. + ACTION_DEV_MODE_STATUS -> answer() + else -> answer("Unknown action") + } + } + + /** Get cookies: the phone's own jar for the page's address, as cookies for its host. */ + private fun cookies(f: Wire.Fields): ByteArray { + val url = f.bytes(2).decodeToString() + val host = Uri.parse(url).host.orEmpty() + val jar = runCatching { CookieManager.getInstance().getCookie(url) }.getOrNull().orEmpty() + val cookies = jar.split(";").map { it.trim().split("=", limit = 2) }.filter { it.size == 2 }.fold(ByteArray(0)) { out, (name, value) -> + out + Wire.fieldBytes( + 3, + Wire.fieldBytes(1, name.encodeToByteArray()) + Wire.fieldBytes(2, value.encodeToByteArray()) + + Wire.fieldBytes(3, host.encodeToByteArray()) + Wire.fieldBytes(4, PATH_ROOT), + ) + } + return Wire.fieldBytes(1, f.bytes(1)) + Wire.fieldBytes(2, url.encodeToByteArray()) + cookies + } + + /** WebappAppInfo {1 partnerKey, 2 appName, 3 appUrl, 4 iconUrl}. */ + private fun info(app: WebApp): ByteArray = + Wire.fieldBytes(1, app.key.encodeToByteArray()) + Wire.fieldBytes(2, app.name.encodeToByteArray()) + + Wire.fieldBytes(3, app.url.encodeToByteArray()) + Wire.fieldBytes(4, ByteArray(0)) + + /** + * The stored PartnerAppSettings with its dev entries replaced by ours. PartnerAppSettings {3 displayConfigs: map + * entry {1 key, 2 DisplayConfig {1 Item {1 title}, 2 url}}}. Everything else in the setting is kept as it was. + */ + private fun partnerSettings(current: String, apps: List): ByteArray { + val old = runCatching { parseProto(Base64.decode(current, Base64.NO_WRAP)) }.getOrNull().orEmpty() + val kept = old.filterNot { it.number == DISPLAY_CONFIGS && entryKey(it)?.startsWith(PREFIX) == true } + val ours = apps.map { app -> + message(DISPLAY_CONFIGS, listOf(text(1, app.key), message(2, listOf(message(1, listOf(text(1, app.name))), text(2, app.url))))) + } + return encodeProto(kept + ours) + } + + private fun entryKey(entry: ProtoField): String? = + parseProto(entry.bytes)?.firstOrNull { it.number == 1 }?.bytes?.decodeToString() + + private fun text(number: Int, value: String) = ProtoField(number, 2, bytes = value.encodeToByteArray()) + + private fun message(number: Int, fields: List) = ProtoField(number, 2, bytes = encodeProto(fields)) + + private fun save(list: List) { + _apps.value = list + runCatching { + file.writeText(JSONArray(list.map { JSONObject().put("key", it.key).put("name", it.name).put("url", it.url) }).toString()) + } + } + + private fun load(): List = runCatching { + val a = JSONArray(file.readText()) + List(a.length()) { i -> a.getJSONObject(i).let { WebApp(it.getString("key"), it.getString("name"), it.getString("url")) } } + }.getOrDefault(emptyList()) + + companion object { + const val SVC = 55074 + private const val SVC_LEASE = 56 + private const val MT_LEASE = 1 + private const val MT_LIFECYCLE = 3 + private const val STATE_BOUND = 1L + private const val FEATURE_WEB_APP = 19L + private const val MT_MANAGE = 15 + private const val MT_MANAGE_RESPONSE = 16 + private const val MT_GET_COOKIES = 11 + private const val MT_SET_COOKIES = 9 + private const val MT_BLE_SCAN_START = 17 + private const val MT_BLE_SCAN_RESPONSE = 20 + private const val ACTION_CREATE = 1L + private const val ACTION_DELETE = 2L + private const val ACTION_LIST = 3L + private const val ACTION_UPDATE = 4L + private const val ACTION_OPEN = 5L + private const val ACTION_DEV_MODE_STATUS = 6L + private const val PREFIX = "dev_mode_web_app_" + private const val SETTING = "partner_app_settings" + private const val DEVELOPER_MODE = "developer_mode" + private const val DISPLAY_CONFIGS = 3 + private const val MAX_APPS = 20 + private const val BOUND_MS = 3_000L + private const val REGISTER_MS = 1_500L + private val PATH_ROOT = "/".encodeToByteArray() + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 16030a83..f6445525 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -158,6 +158,7 @@ object ConnectionModule { placeSearch: com.macedonga.visor.connection.PlaceSearch, mapsSupport: com.macedonga.visor.connection.MapsSupport, mapsQuiet: com.macedonga.visor.connection.MapsQuiet, + webApps: com.macedonga.visor.connection.WebApps, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -187,6 +188,7 @@ object ConnectionModule { com.macedonga.visor.connection.MapsSupport.SVC to mapsSupport, com.macedonga.visor.connection.MapsQuiet.SVC_PERSISTENT_STORE to mapsQuiet.responder(com.macedonga.visor.connection.MapsQuiet.SVC_PERSISTENT_STORE), com.macedonga.visor.connection.MapsQuiet.SVC_CONSTELLATION to mapsQuiet.responder(com.macedonga.visor.connection.MapsQuiet.SVC_CONSTELLATION), + com.macedonga.visor.connection.WebApps.SVC to webApps, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt index aaa81387..11bb83ea 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt @@ -29,6 +29,7 @@ import androidx.lifecycle.viewModelScope import androidx.navigation3.runtime.NavKey import com.macedonga.visor.AccountsKey import com.macedonga.visor.OnboardingKey +import com.macedonga.visor.WebAppsKey import com.macedonga.visor.WifiNetworksKey import com.macedonga.visor.R import com.macedonga.visor.connection.GlassesService @@ -230,6 +231,7 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( } }.toTypedArray(), if (hasDisplay(info?.build)) row { ValueRow(stringResource(R.string.wifi_networks), null, onClick = { open(WifiNetworksKey) }, locked = !ready) } else null, + if (hasDisplay(info?.build)) row { ValueRow(stringResource(R.string.webapps_row), null, onClick = { open(WebAppsKey) }, locked = !ready) } else null, ) } } diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt new file mode 100644 index 00000000..4711af14 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt @@ -0,0 +1,157 @@ +package com.macedonga.visor.settings + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.lifecycle.viewModelScope +import com.macedonga.visor.R +import com.macedonga.visor.connection.AddResult +import com.macedonga.visor.connection.OPEN_GATES +import com.macedonga.visor.connection.OpenGate +import com.macedonga.visor.connection.WebApp +import com.macedonga.visor.connection.WebApps +import com.macedonga.visor.connection.gateOf +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.media.GlassesConfig +import com.macedonga.visor.core.session.LinkState +import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.launch + +enum class WebAppNote { SAVED_ON_PHONE, OPENED, NOT_OPENED, RESTART, SWITCH_NOT_SENT, NOT_HTTPS, FULL } + +@HiltViewModel +class WebAppsModel @Inject constructor( + private val webApps: WebApps, + val connection: GlassesConnection, + private val config: GlassesConfig, +) : ViewModel() { + val apps: StateFlow> = webApps.apps + val gate: StateFlow = config.state.map { gateOf(it) }.stateIn(viewModelScope, SharingStarted.Eagerly, gateOf(config.state.value)) + + private val _note = MutableStateFlow(null) + val note: StateFlow = _note.asStateFlow() + + fun sync() { + webApps.push() + } + + fun add(name: String, url: String) { + _note.value = when (webApps.add(name, url)) { + AddResult.ADDED, AddResult.EXISTS -> if (webApps.push()) null else WebAppNote.SAVED_ON_PHONE + AddResult.NOT_HTTPS -> WebAppNote.NOT_HTTPS + AddResult.FULL -> WebAppNote.FULL + } + } + + fun remove(key: String) { + webApps.remove(key) + _note.value = if (webApps.push()) null else WebAppNote.SAVED_ON_PHONE + } + + fun open(app: WebApp) { + viewModelScope.launch { + _note.value = if (webApps.open(app)) WebAppNote.OPENED else WebAppNote.NOT_OPENED + } + } + + /** Switch the two gates on by MobileConfig: the glasses need a restart to use them. */ + fun turnOn() { + viewModelScope.launch { + _note.value = if (config.send(OPEN_GATES.associateWith { true }) == null) WebAppNote.RESTART else WebAppNote.SWITCH_NOT_SENT + } + } +} + +/** The web apps Visor lists for the Display, and opening one on the glasses. */ +@Composable +fun WebAppsScreen(model: WebAppsModel = hiltViewModel()) { + val glasses by model.connection.state.collectAsStateWithLifecycle() + val ready = glasses.link == LinkState.READY + val apps by model.apps.collectAsStateWithLifecycle() + val gate by model.gate.collectAsStateWithLifecycle() + val note by model.note.collectAsStateWithLifecycle() + var adding by remember { mutableStateOf(false) } + LaunchedEffect(ready) { if (ready) model.sync() } + + SettingsPage(stringResource(R.string.webapps_title), light = 0f) { + item(key = "apps") { + Column { + Rows( + *apps.map { app -> + @Composable { + RowFrame(app.name, onClick = null, sub = app.url) { + ActionPill( + stringResource(R.string.webapps_open), { model.open(app) }, quiet = true, + modifier = Modifier.padding(start = 12.dp), enabled = ready && gate == OpenGate.ON, + ) + ActionPill( + stringResource(R.string.webapps_remove), { model.remove(app.key) }, quiet = true, + modifier = Modifier.padding(start = 12.dp), + ) + } + } + }.toTypedArray(), + { ValueRow(stringResource(R.string.webapps_add), null, onClick = { adding = true }) }, + if (gate == OpenGate.OFF) row { ValueRow(stringResource(R.string.webapps_turn_on), null, onClick = { model.turnOn() }, locked = !ready) } else null, + ) + when (gate) { + OpenGate.UNAVAILABLE -> Footnote(stringResource(R.string.webapps_unavailable)) + OpenGate.UNREAD -> Footnote(stringResource(R.string.webapps_unread)) + OpenGate.OFF, OpenGate.ON -> Unit + } + note?.let { Footnote(stringResource(noteText(it))) } + if (apps.isEmpty()) Footnote(stringResource(R.string.webapps_none)) + } + } + } + + if (adding) { + var name by remember { mutableStateOf("") } + var url by remember { mutableStateOf("https://") } + VisorSheet({ adding = false }) { + SheetTitle(stringResource(R.string.webapps_add)) + SheetField(name, { name = it }, hint = stringResource(R.string.webapps_name), focus = true) + Spacer(Modifier.height(10.dp)) + SheetField(url, { url = it }, hint = stringResource(R.string.webapps_url)) + Row(Modifier.fillMaxWidth().padding(horizontal = SIDE).padding(top = 16.dp), horizontalArrangement = Arrangement.End) { + ActionPill(stringResource(R.string.settings_save), { + model.add(name.trim(), url.trim()) + adding = false + }, enabled = url.isNotBlank()) + } + } + } +} + +private fun noteText(note: WebAppNote): Int = when (note) { + WebAppNote.SAVED_ON_PHONE -> R.string.webapps_saved_on_phone + WebAppNote.OPENED -> R.string.webapps_opened + WebAppNote.NOT_OPENED -> R.string.webapps_not_opened + WebAppNote.RESTART -> R.string.webapps_restart + WebAppNote.SWITCH_NOT_SENT -> R.string.webapps_switch_not_sent + WebAppNote.NOT_HTTPS -> R.string.webapps_not_https + WebAppNote.FULL -> R.string.webapps_full +} diff --git a/app/src/main/res/values-it/strings_webapps.xml b/app/src/main/res/values-it/strings_webapps.xml new file mode 100644 index 00000000..3d5d4476 --- /dev/null +++ b/app/src/main/res/values-it/strings_webapps.xml @@ -0,0 +1,21 @@ + + + App web + App web + Aggiungi + Apri + Rimuovi + Nome + Indirizzo (https) + Nessuna app web + Attiva le app web + Gli occhiali non hanno questo interruttore + Collega gli occhiali per leggerlo + Salvata sul telefono. Gli occhiali la ricevono quando sono collegati. + Aperta sugli occhiali + Gli occhiali non l\'hanno aperta + Riavvia gli occhiali per usare le app web + L\'interruttore non è stato inviato + Usa un indirizzo https + Massimo 20 app web + diff --git a/app/src/main/res/values/strings_webapps.xml b/app/src/main/res/values/strings_webapps.xml new file mode 100644 index 00000000..6d90a956 --- /dev/null +++ b/app/src/main/res/values/strings_webapps.xml @@ -0,0 +1,21 @@ + + + Web apps + Web apps + Add + Open + Remove + Name + Address (https) + No web apps + Turn on web apps + The glasses do not have this switch + Connect the glasses to read this + Saved on the phone. The glasses get it when connected. + Opened on the glasses + The glasses did not open it + Restart the glasses to use web apps + The switch was not sent + Use an https address + Up to 20 web apps + diff --git a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json index ae88c9b0..ffd08bd3 100644 --- a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json +++ b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json @@ -13018,10 +13018,36 @@ ], "Display":[ { +"n":1, +"label":"Item", +"type":"message", +"ref":"Item" +}, +{ "n":2, "label":"Link", "type":"string" } +], +"Item":[ +{ +"n":1, +"label":"Title", +"type":"string" +}, +{ +"n":2, +"label":"Icon", +"type":"message", +"ref":"Image" +} +], +"Image":[ +{ +"n":1, +"label":"Uri", +"type":"string" +} ] } } From af0b76c2a95d0255962210c993ec895334f46cfe Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:52:07 +0200 Subject: [PATCH 65/90] Display: a Wi-Fi link when the Display asks for one. A svc41107 link lease that needs Wi-Fi is now granted when the phone can take it (Wi-Fi on, Wi-Fi Direct allowed, the Live view not using Wi-Fi): Visor answers OK, then moves to a Wi-Fi Direct session with the glasses the way media and the Mirror do, pushes the internet switch's state (svc41108) on a channel it opens there so the Display's apps take the tunnel over Wi-Fi, and comes back to Bluetooth once no lease is held and the tunnel has been idle for two minutes, or when the Live view starts or the Wi-Fi link drops; the state is pushed again on Bluetooth. Otherwise the lease is refused as before, with the reason logged. Not tried on the Display yet. --- .../visor/connection/InternetSwitch.kt | 11 ++ .../visor/connection/LeaseService.kt | 34 ++++-- .../visor/connection/PhoneInternet.kt | 9 ++ .../macedonga/visor/connection/WifiLease.kt | 104 ++++++++++++++++++ 4 files changed, 148 insertions(+), 10 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/WifiLease.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt index a3ee79e5..332a166a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt @@ -17,6 +17,8 @@ import java.util.concurrent.atomic.AtomicBoolean import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.withTimeoutOrNull /** * svc41108, internet_switch: how the phone's internet is (type 3): connected, metered, and its DNS servers. The Display @@ -73,6 +75,15 @@ class InternetSwitch @Inject constructor( glasses.submit { it.sendOnService(SVC, MT_COMPANION_STATE, state) } } + /** The state on [mux] itself, for a session other than Bluetooth's (Wi-Fi during a link lease). */ + suspend fun pushOn(mux: DataXConnection) = mux.sendOnService(SVC, MT_COMPANION_STATE, encode(phoneInternet())) + + /** After a move back to Bluetooth: its session is new, so the state goes on it once the link is up again. */ + suspend fun pushWhenReady() { + withTimeoutOrNull(60_000) { connection.get().state.first { it.link == LinkState.READY } } + push() + } + private data class PhoneInternet(val connected: Boolean, val metered: Boolean, val dns: List) private fun phoneInternet(): PhoneInternet { diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt index 93d37d3b..d2bd7938 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt @@ -13,11 +13,13 @@ import javax.inject.Singleton /** * svc41107, lease_service: the glasses ask for internet (type 1, InternetLeaseRequest) and for a link (type 4, * LinkLeaseRequest), and end a lease (type 3, no answer). Internet leases are answered as the Meta app answers them - * (2, use the glasses' own Wi-Fi); a link lease (5) on Bluetooth is granted, one that needs Wi-Fi is refused for now. + * (2, use the glasses' own Wi-Fi); a link lease (5) on Bluetooth is granted, one that needs Wi-Fi moves the phone to + * Wi-Fi Direct ([WifiLease]) when it can, and is refused otherwise. */ @Singleton class LeaseService @Inject constructor( private val log: Provider, + private val wifiLease: Provider, ) : Responder { // Leases the glasses hold, by lease id, so an expiry can be named in the log. private val held = ConcurrentHashMap() @@ -47,22 +49,34 @@ class LeaseService @Inject constructor( request.has(6) -> request.varint(6) >= 1 else -> null } - val reply = when (wifi) { - null -> Wire.fieldBytes(1, id) + refusal(ERROR_BAD_REQUEST, "Invalid Request") - true -> Wire.fieldBytes(1, id) + refusal(ERROR_WIFI_DIRECT_NOT_AVAILABLE, "Wi-Fi is not available for this link.") - false -> Wire.fieldBytes(1, id) + // Checked before anything is answered or moved: a Wi-Fi link the phone cannot take is refused. + val unavailable = if (wifi == true) wifiLease.get().refusal() else null + val granted = wifi == false || (wifi == true && unavailable == null) + val reply = when { + wifi == null -> Wire.fieldBytes(1, id) + refusal(ERROR_BAD_REQUEST, "Invalid Request") + granted -> Wire.fieldBytes(1, id) + else -> Wire.fieldBytes(1, id) + refusal(ERROR_WIFI_DIRECT_NOT_AVAILABLE, "Wi-Fi is not available for this link.") } log.get().add( - when (wifi) { - null -> "Lease: the glasses asked for a link with no requirement; refused" - true -> "Lease: the glasses asked for Wi-Fi; not available yet, refused" - false -> "Lease: the glasses asked for a link on Bluetooth; answered OK" + when { + wifi == null -> "Lease: the glasses asked for a link with no requirement; refused" + wifi == false -> "Lease: the glasses asked for a link on Bluetooth; answered OK" + unavailable != null -> "Lease: the glasses asked for Wi-Fi; refused, $unavailable" + else -> "Lease: the glasses asked for Wi-Fi; answered OK" }, ) if (wifi == false) held[key] = "link" + if (wifi == true && granted) { + held[key] = "Wi-Fi link" + wifiLease.get().add(key) + } connection.send(channel, reply, messageType = MT_LINK_RESPONSE) + if (wifi == true && granted) wifiLease.get().hold() + } + MT_LEASE_EXPIRATION -> { + held.remove(key)?.let { log.get().add("Lease: a $it lease ended") } + wifiLease.get().remove(key) } - MT_LEASE_EXPIRATION -> held.remove(key)?.let { log.get().add("Lease: a $it lease ended") } else -> log.get().add("Lease: message type ${message.info.messageType} is not handled") } } diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt index 3975c90a..ec438d1b 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt @@ -15,6 +15,7 @@ import java.nio.ByteBuffer import java.nio.ByteOrder import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @@ -51,10 +52,17 @@ class PhoneInternet @Inject constructor( // A connection that fails ends alone; nothing in here may take the app down. private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Internet: a connection failed: $e") }) private val sessions = ConcurrentHashMap() + // When the tunnel last carried anything either way; a move to Wi-Fi for a link ends once it has been idle a while. + private val lastTraffic = AtomicLong(System.currentTimeMillis()) + + fun idleMillis(): Long = System.currentTimeMillis() - lastTraffic.get() + + fun touch() = lastTraffic.set(System.currentTimeMillis()) override suspend fun respond(message: DataXMessage, connection: DataXConnection) { val info = message.info val channel = info.channelId + touch() if (info.flags.isClosed && message.payload.isEmpty()) { sessions.remove(channel)?.close() relay.forget(channel) @@ -111,6 +119,7 @@ class PhoneInternet @Inject constructor( /** An IP packet to the Display, framed for its tunnel. */ suspend fun deliver(version: Int, src: ByteArray, dst: ByteArray, protocol: Int, transport: ByteArray) { + touch() val ip = IpPacket.build(version, src, dst, protocol, transport, ids.incrementAndGet()) val frame = ByteBuffer.allocate(ICEE_HEADER + ip.size).order(ByteOrder.LITTLE_ENDIAN) .putShort(ICEE_MAGIC.toShort()).putShort(ip.size.toShort()) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/WifiLease.kt b/app/src/main/kotlin/com/macedonga/visor/connection/WifiLease.kt new file mode 100644 index 00000000..2660e6eb --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/WifiLease.kt @@ -0,0 +1,104 @@ +package com.macedonga.visor.connection + +import android.content.Context +import com.macedonga.visor.core.media.LiveStream +import com.macedonga.visor.core.media.wifiDirectAllowed +import com.macedonga.visor.core.media.wifiIsOn +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch + +/** + * The Wi-Fi link the Display asks for on svc41107 (a link lease that needs Wi-Fi). While one is held the phone moves + * to a Wi-Fi session with the glasses, as [LiveStream.overWifi] does for media, and the internet tunnel (svc41111) + * runs there with svc41108's state. The move ends when no lease is left and the tunnel has been idle for two minutes. + */ +@Singleton +class WifiLease @Inject constructor( + @ApplicationContext private val context: Context, + private val live: Provider, + private val internet: Provider, + private val tunnel: Provider, + private val log: Provider, +) { + private val leases = ConcurrentHashMap.newKeySet() + private val moving = AtomicBoolean(false) + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + + fun add(id: String) { + leases.add(id) + } + + fun remove(id: String) { + leases.remove(id) + } + + /** Why the phone cannot take a Wi-Fi link now, or null. While a move runs, another lease simply joins it. */ + fun refusal(): String? { + if (moving.get()) return null + val glasses = live.get() + return when { + glasses.running.value || glasses.wifiSession.value != null -> "the live view is using Wi-Fi" + !wifiIsOn(context) -> "Wi-Fi is off" + !wifiDirectAllowed(context) -> "Wi-Fi Direct is not allowed" + else -> null + } + } + + /** Moves to Wi-Fi for the leases held, unless a move is running already. */ + fun hold() { + if (!moving.compareAndSet(false, true)) return + scope.launch { + try { + log.get().add("Wi-Fi: moving to Wi-Fi Direct, because the Display asked for a link that needs Wi-Fi") + val failed = live.get().overWifi { mux -> ride(mux) } + if (failed == null) { + log.get().add("Wi-Fi: back on Bluetooth") + internet.get().pushWhenReady() + } else { + leases.clear() + log.get().add("Wi-Fi: the move did not start: $failed") + } + } finally { + moving.set(false) + } + } + } + + /** Keeps the Wi-Fi session up until no lease is left and the tunnel is idle, or the Live view takes Wi-Fi. */ + private suspend fun ride(mux: DataXConnection): String? { + internet.get().pushOn(mux) + tunnel.get().touch() + while (true) { + delay(1_000) + val glasses = live.get() + if (glasses.running.value) { + log.get().add("Wi-Fi: leaving Wi-Fi, because the live view started") + break + } + if (glasses.wifiSession.value == null) { + log.get().add("Wi-Fi: leaving Wi-Fi, because the Wi-Fi link dropped") + break + } + if (leases.isEmpty() && tunnel.get().idleMillis() >= IDLE_MS) { + log.get().add("Wi-Fi: leaving Wi-Fi, because no link lease is held and the tunnel has been idle for 2 min") + break + } + } + return null + } + + companion object { + private const val IDLE_MS = 120_000L + } +} From 6f2cebf7e3fcd8926690ccaa9d0ad181208538f0 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:56:13 +0200 Subject: [PATCH 66/90] Display: link Facebook and Instagram to the glasses (svc89 type 6). The Accounts page gets Facebook and Instagram rows. Linking one mints the accounts app's token from the user's Meta login, finds the account linked in the user's Accounts Center (MetaLinkedAccounts) and its tokens for the glasses (MetaLinkedAccountAccessAndFirstPartyTokensQuery, as com.facebook.stella), and sends MetaLinkedAccountSyncRequest (type 6: session, account type, obfuscated id, login, the access and first-party tokens), answered by 14. The account metadata the Meta app adds from per-provider queries is not sent. Messenger and Instagram DM media are fetched by those apps on the phone through their own IPC in the Meta app, so the data fetcher still turns them down; WhatsApp's encrypted media was already served. Not tried on the Display yet. --- .../visor/connection/AccountLinking.kt | 51 ++++++++- .../com/macedonga/visor/login/MetaLogin.kt | 2 +- .../visor/settings/AccountsScreen.kt | 38 ++++++- .../main/res/values-it/strings_accounts.xml | 2 + app/src/main/res/values/strings_accounts.xml | 2 + .../core/identity/token/LinkedAccounts.kt | 102 ++++++++++++++++++ 6 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/LinkedAccounts.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt b/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt index 62369a1c..45d5b719 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt @@ -1,8 +1,11 @@ package com.macedonga.visor.connection import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.token.LinkedAccountType import com.macedonga.visor.core.identity.token.TokenError import com.macedonga.visor.core.identity.token.exchangeAccountToken +import com.macedonga.visor.core.identity.token.fetchLinkedAccounts +import com.macedonga.visor.core.identity.token.fetchLinkedTokens import com.macedonga.visor.core.protocol.Wire import com.macedonga.visor.core.session.ServiceData import com.macedonga.visor.log.EventLog @@ -21,13 +24,14 @@ import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeoutOrNull /** What a link attempt came to, for the Accounts page to say. */ -enum class LinkOutcome { LINKED, LOGGED_OUT, UNREACHABLE, NO_ANSWER, REFUSED } +enum class LinkOutcome { LINKED, LOGGED_OUT, UNREACHABLE, NO_ANSWER, REFUSED, NOT_LINKED } /** * svc89, wearables_whatsapp_linking: the Meta account and WhatsApp linking the Meta app runs with the Display. Visor * opens the channel. 25 asks whether a Meta account is present (answered by 26). 5 is the Meta account sync, with the - * user's token minted for the glasses' accounts app (answered by 13, after an ack 17 Visor does not wait for). 22 is - * the WhatsApp QR pairing: the glasses show a code for WhatsApp's Linked devices and answer 23 when it is scanned. + * user's token minted for the glasses' accounts app (answered by 13, after an ack 17 Visor does not wait for). 6 is a + * linked provider account, Facebook or Instagram, with its tokens (answered by 14, after an ack 18). 22 is the WhatsApp + * QR pairing: the glasses show a code for WhatsApp's Linked devices and answer 23 when it is scanned. */ @Singleton class AccountLinking @Inject constructor( @@ -78,6 +82,45 @@ class AccountLinking @Inject constructor( return LinkOutcome.LINKED } + /** Sends a provider account of the Meta account, with its tokens, to the glasses. */ + suspend fun syncLinked(type: LinkedAccountType): LinkOutcome { + if (!login.source.hasCachedLogin) return LinkOutcome.LOGGED_OUT + val found = try { + withContext(Dispatchers.IO) { + val accountsToken = exchangeAccountToken(login.source.get()).accessToken + val account = fetchLinkedAccounts(accountsToken).firstOrNull { it.type == type } ?: return@withContext null + account to (fetchLinkedTokens(accountsToken, account, login.deviceId) ?: return@withContext null) + } + } catch (error: TokenError) { + log.add("Accounts: ${error.message}") + return if (error.unreachable) LinkOutcome.UNREACHABLE else LinkOutcome.REFUSED + } + if (found == null) { + log.add("Accounts: ${type.name.lowercase()} is not linked to this Meta account") + return LinkOutcome.NOT_LINKED + } + val (account, tokens) = found + val credentials = Wire.fieldBytes(1, str(tokens.accessToken)) + + (tokens.firstPartyToken?.let { Wire.fieldBytes(2, str(it)) } ?: ByteArray(0)) + val request = Wire.fieldBytes(1, str(uuid())) + + Wire.fieldVarint(2, type.wire.toLong()) + + Wire.fieldBytes(4, str(account.obfuscatedId)) + + Wire.fieldVarint(5, flag(true)) + + Wire.fieldBytes(6, str(uuid())) + + Wire.fieldBytes(7, credentials) + val f = ask(MT_LINKED_SYNC_REQUEST, MT_LINKED_SYNC_RESPONSE, request, ANSWER_MS) + if (f == null) { + log.add("Accounts: the glasses did not answer the ${type.name.lowercase()} account") + return LinkOutcome.NO_ANSWER + } + if (f.varint(2) != 1L) { + log.add("Accounts: the glasses refused the ${type.name.lowercase()} account, code ${f.varint(3)}") + return LinkOutcome.REFUSED + } + log.add("Accounts: ${type.name.lowercase()} account sent to the glasses") + return LinkOutcome.LINKED + } + /** Has the glasses show their WhatsApp code; the answer comes once it is scanned from WhatsApp's Linked devices. */ suspend fun linkWhatsApp(): LinkOutcome { val f = ask(MT_QR_PAIRING_REQUEST, MT_QR_PAIRING_RESPONSE, Wire.fieldBytes(1, str(uuid())), SCAN_MS) @@ -135,5 +178,7 @@ class AccountLinking @Inject constructor( private const val MT_PRESENT_RESPONSE = 26 private const val MT_QR_PAIRING_REQUEST = 22 private const val MT_QR_PAIRING_RESPONSE = 23 + private const val MT_LINKED_SYNC_REQUEST = 6 + private const val MT_LINKED_SYNC_RESPONSE = 14 } } diff --git a/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt b/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt index 25c24c09..89188509 100644 --- a/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt +++ b/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt @@ -201,7 +201,7 @@ class MetaLogin @Inject constructor( } /** One id per install, as the Meta app keeps its `device_id`: the login's server state is tied to it. */ - private val deviceId: String by lazy { + val deviceId: String by lazy { val prefs = context.getSharedPreferences("meta_login", Context.MODE_PRIVATE) prefs.getString("device_id", null) ?: UUID.randomUUID().toString().also { prefs.edit().putString("device_id", it).apply() } } diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt index 8e9e278d..99f8a9b3 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt @@ -15,6 +15,7 @@ import com.macedonga.visor.R import com.macedonga.visor.connection.AccountLinking import com.macedonga.visor.connection.LinkOutcome import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.token.LinkedAccountType import com.macedonga.visor.core.session.LinkState import dagger.hilt.android.lifecycle.HiltViewModel import javax.inject.Inject @@ -23,7 +24,7 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch -enum class AccountRow { META, WHATSAPP } +enum class AccountRow { META, WHATSAPP, FACEBOOK, INSTAGRAM } @HiltViewModel class AccountsModel @Inject constructor( @@ -44,6 +45,9 @@ class AccountsModel @Inject constructor( private val _whatsappResult = MutableStateFlow(null) val whatsappResult: StateFlow = _whatsappResult.asStateFlow() + private val _linkedResults = MutableStateFlow>(emptyMap()) + val linkedResults: StateFlow> = _linkedResults.asStateFlow() + fun refresh() { viewModelScope.launch { linking.metaPresent()?.let { _meta.value = it } } } @@ -57,6 +61,10 @@ class AccountsModel @Inject constructor( _whatsappResult.value = linking.linkWhatsApp() } + fun linkProvider(row: AccountRow, type: LinkedAccountType) = task(row) { + _linkedResults.value = _linkedResults.value + (type to linking.syncLinked(type)) + } + private fun task(row: AccountRow, work: suspend () -> Unit) { if (_busy.value != null) return _busy.value = row @@ -79,6 +87,7 @@ fun AccountsScreen(model: AccountsModel = hiltViewModel()) { val busy by model.busy.collectAsStateWithLifecycle() val metaResult by model.metaResult.collectAsStateWithLifecycle() val whatsappResult by model.whatsappResult.collectAsStateWithLifecycle() + val linked by model.linkedResults.collectAsStateWithLifecycle() LaunchedEffect(ready) { if (ready) model.refresh() } SettingsPage(stringResource(R.string.accounts_title), light = 0f) { @@ -121,11 +130,37 @@ fun AccountsScreen(model: AccountsModel = hiltViewModel()) { ) } }, + { LinkedRow(model, AccountRow.FACEBOOK, LinkedAccountType.FACEBOOK, R.string.accounts_facebook, ready, busy, linked) }, + { LinkedRow(model, AccountRow.INSTAGRAM, LinkedAccountType.INSTAGRAM, R.string.accounts_instagram, ready, busy, linked) }, ) } } } +@Composable +private fun LinkedRow( + model: AccountsModel, + row: AccountRow, + type: LinkedAccountType, + label: Int, + ready: Boolean, + busy: AccountRow?, + results: Map, +) { + RowFrame( + stringResource(label), + onClick = null, + sub = results[type]?.let { outcomeWords(it) }, + locked = !ready, + busy = busy == row, + ) { + ActionPill( + stringResource(R.string.accounts_link), { model.linkProvider(row, type) }, + quiet = true, modifier = Modifier.padding(start = 12.dp), enabled = ready && busy == null, + ) + } +} + @Composable private fun outcomeWords(outcome: LinkOutcome): String = stringResource( when (outcome) { @@ -134,5 +169,6 @@ private fun outcomeWords(outcome: LinkOutcome): String = stringResource( LinkOutcome.UNREACHABLE -> R.string.accounts_unreachable LinkOutcome.NO_ANSWER -> R.string.accounts_no_answer LinkOutcome.REFUSED -> R.string.accounts_refused + LinkOutcome.NOT_LINKED -> R.string.accounts_not_linked }, ) diff --git a/app/src/main/res/values-it/strings_accounts.xml b/app/src/main/res/values-it/strings_accounts.xml index 96458b2a..bc8b3632 100644 --- a/app/src/main/res/values-it/strings_accounts.xml +++ b/app/src/main/res/values-it/strings_accounts.xml @@ -3,6 +3,8 @@ Account Account Meta WhatsApp + Facebook + Instagram Collega Collegato Non collegato diff --git a/app/src/main/res/values/strings_accounts.xml b/app/src/main/res/values/strings_accounts.xml index 830ec608..ce280a30 100644 --- a/app/src/main/res/values/strings_accounts.xml +++ b/app/src/main/res/values/strings_accounts.xml @@ -3,6 +3,8 @@ Accounts Meta account WhatsApp + Facebook + Instagram Link Linked Not linked diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/LinkedAccounts.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/LinkedAccounts.kt new file mode 100644 index 00000000..8cf2dbe4 --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/LinkedAccounts.kt @@ -0,0 +1,102 @@ +package com.macedonga.visor.core.identity.token + +import com.macedonga.visor.core.identity.HttpPost +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.pyUrlencode +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.jsonObject + +/** A provider account of the Meta account: its svc89 account type, the Graph name and the app id its token is minted for. */ +enum class LinkedAccountType(val wire: Int, val graphName: String, val appId: String) { + FACEBOOK(1, "FACEBOOK", "1053491986377452"), + INSTAGRAM(2, "INSTAGRAM", "1313693057438013"), + THREADS(3, "THREADS", "1590413528774085"), +} + +/** A linked account as the Meta account lists it: the id the Graph queries take. */ +class LinkedAccountRef(val type: LinkedAccountType, val obfuscatedId: String) + +/** A linked account's access token and, when Meta gives one, its first-party token. Secrets: never log them. */ +class LinkedTokens(val accessToken: String, val firstPartyToken: String?) + +private const val GRAPH_URL = "https://ar-genai.graph.meta.com/graphql" +private const val LINKED_LIST_DOC = "3214186391731893797068997674" +private const val LINKED_TOKENS_DOC = "254156320718283261599350148705" +private const val LINKED_TOKENS_ROOT = "fx_ar_linked_access_and_first_party_token_from_obid" +private const val TOKEN_PACKAGE = "com.facebook.stella" + +/** + * The provider accounts linked to the Meta account, from the accounts app's token minted by [exchangeAccountToken]. Each + * node that carries an obfuscated id and an account type counts. + */ +fun fetchLinkedAccounts(accountsToken: String, http: HttpPost = UrlConnectionPost): List { + val found = mutableListOf() + fun walk(element: JsonElement) { + when (element) { + is JsonObject -> { + val id = element.text("obfuscated_account_id") + val type = element.text("account_type")?.let { name -> LinkedAccountType.entries.firstOrNull { it.graphName == name.uppercase() } } + if (id != null && type != null) found += LinkedAccountRef(type, id) + element.values.forEach(::walk) + } + is JsonArray -> element.forEach(::walk) + else -> Unit + } + } + walk(graphql(accountsToken, "MetaLinkedAccounts", LINKED_LIST_DOC, "{}", http)) + return found.distinctBy { it.type } +} + +/** The tokens of one linked account, from the accounts app's token; null when Meta gives no access token for it. */ +fun fetchLinkedTokens( + accountsToken: String, + account: LinkedAccountRef, + deviceId: String, + http: HttpPost = UrlConnectionPost, +): LinkedTokens? { + val variables = JsonObject( + mapOf( + "obfuscated_account_id" to JsonPrimitive(account.obfuscatedId), + "account_type" to JsonPrimitive(account.type.graphName), + "app_id" to JsonPrimitive(account.type.appId), + "package_name" to JsonPrimitive(TOKEN_PACKAGE), + "device_id" to JsonPrimitive(deviceId), + "service" to JsonPrimitive("WEARABLES_LINKED_ACCOUNTS"), + ), + ).toString() + val json = graphql(accountsToken, "MetaLinkedAccountAccessAndFirstPartyTokensQuery", LINKED_TOKENS_DOC, variables, http) + val node = (json["data"] as? JsonObject)?.get(LINKED_TOKENS_ROOT) as? JsonObject ?: return null + val token = node.text("token")?.takeIf { it.isNotEmpty() } ?: return null + return LinkedTokens(token, node.text("first_party_token")?.takeIf { it.isNotEmpty() }) +} + +/** One persisted query on Meta's ar GraphQL, as the Meta app's own calls send it. */ +private fun graphql(accessToken: String, name: String, doc: String, variables: String, http: HttpPost): JsonObject { + val body = pyUrlencode( + linkedMapOf( + "fb_api_req_friendly_name" to name, + "client_doc_id" to doc, + "variables" to variables, + "method" to "post", + "pretty" to "false", + "format" to "json", + ), + ).toByteArray(Charsets.US_ASCII) + val reply = try { + http.post( + GRAPH_URL, + mapOf("Authorization" to "OAuth $accessToken", "Content-Type" to "application/x-www-form-urlencoded"), + body, + ) + } catch (error: Exception) { + throw TokenError("Meta could not be reached: ${error.message}", unreachable = true) + } + val json = runCatching { Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject } + .getOrElse { throw TokenError("Meta answered ${reply.status} with no JSON") } + if (json["errors"] != null) throw TokenError("Meta refused the query $name") + return json +} From 2cc0d821ece9b4c5ec3cd280dcd093f6b7456bbf Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 10:58:07 +0200 Subject: [PATCH 67/90] Display: keep the internet tunnel from crowding the link. Tunnel frames now wait for one of eight send slots, so the link's own replies (the delivery check, the canary, the device state) queue behind at most eight of them; a TCP retransmit resends only the oldest unacknowledged segment instead of the whole window every 1.5 s while holding the flow's lock; and tunnels built on an earlier connection are closed when the glasses come back, where they kept writing under the old link's keys and retransmitting for up to ten minutes. These are code faults that fit "the link dropped every few seconds with internet sharing on, right after a Display reboot"; that they were its cause is not proven. --- .../visor/connection/PhoneInternet.kt | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt index ec438d1b..0f99b087 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt @@ -33,7 +33,9 @@ import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import kotlinx.coroutines.runInterruptible import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.withTimeoutOrNull /** @@ -52,6 +54,11 @@ class PhoneInternet @Inject constructor( // A connection that fails ends alone; nothing in here may take the app down. private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Internet: a connection failed: $e") }) private val sessions = ConcurrentHashMap() + // Tunnel frames waiting for the link. Control frames of the session share its one send lock, so they wait + // behind at most this many of these. + private val tunnelSlots = Semaphore(TUNNEL_FRAMES) + // The connection the tunnels run on; a new one ends the old tunnels. + private var owner: DataXConnection? = null // When the tunnel last carried anything either way; a move to Wi-Fi for a link ends once it has been idle a while. private val lastTraffic = AtomicLong(System.currentTimeMillis()) @@ -63,6 +70,12 @@ class PhoneInternet @Inject constructor( val info = message.info val channel = info.channelId touch() + if (connection !== owner) { + // A tunnel of an earlier connection would write on this one under the old keys, and keep retransmitting. + owner = connection + sessions.values.forEach { it.close() } + sessions.clear() + } if (info.flags.isClosed && message.payload.isEmpty()) { sessions.remove(channel)?.close() relay.forget(channel) @@ -125,7 +138,7 @@ class PhoneInternet @Inject constructor( .putShort(ICEE_MAGIC.toShort()).putShort(ip.size.toShort()) .order(ByteOrder.BIG_ENDIAN).putShort(if (version == 4) ETHER_IPV4 else ETHER_IPV6).putShort(0) .put(ip).array() - mux.send(reply, frame, messageType = MT_FRAME) + tunnelSlots.withPermit { mux.send(reply, frame, messageType = MT_FRAME) } } private fun resetFor(p: IpPacket, seg: TcpSegment) { @@ -289,17 +302,13 @@ class PhoneInternet @Inject constructor( } } + /** + * Resend the oldest unacked segment only. Resending the whole window each tick queued up to 48 frames per flow + * every 1.5 s behind a link that was already not keeping up, and held this flow's lock while they waited. + */ private suspend fun retransmit() { - lock.withLock { - var at = 0 - var seq = sndUna - while (at < pending.size) { - val end = minOf(pending.size, at + mss) - send(TcpSegment.ACK or TcpSegment.PSH, seq, pending.copyOfRange(at, end)) - seq = (seq + (end - at)) and MASK - at = end - } - } + val (seq, data) = lock.withLock { sndUna to pending.copyOf(minOf(pending.size, mss)) } + if (data.isNotEmpty()) send(TcpSegment.ACK or TcpSegment.PSH, seq, data) } private suspend fun send(flags: Int, seq: Long, data: ByteArray, mssOption: Int = 0) { @@ -390,6 +399,7 @@ class PhoneInternet @Inject constructor( // The Display's tunnel has an MTU of 1400. private const val MSS_V4 = 1360 private const val MSS_V6 = 1340 + private const val TUNNEL_FRAMES = 8 private const val WINDOW = 65535 private const val CONNECT_MS = 10_000 private const val IDLE_MS = 600_000L From e4e8ca7b563da5c646110171fc40a58321d0345b Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 11:03:59 +0200 Subject: [PATCH 68/90] Display: the Neural Band sets up on a clean install. Band setup only took a band already bonded as "Meta Band", and nothing in Visor bonded one, so on a new phone it always ended in "pair the band with this phone first". It now listens 15 s for a Meta band's advertisement (company 0x01AB, product 1537 or 1538), prefers one in pairing mode, and bonds it before the change of owner. A failure while linking an account on the Accounts page is logged there instead of taking the app down. The rest of the clean-install path for the Display and the band was read through without other faults found. Not tried on the band yet. --- .../com/macedonga/visor/band/BandSetup.kt | 8 +++++- .../visor/settings/AccountsScreen.kt | 6 +++++ .../macedonga/visor/setup/NearbyGlasses.kt | 25 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandSetup.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandSetup.kt index c0599bc0..64f7020e 100644 --- a/app/src/main/kotlin/com/macedonga/visor/band/BandSetup.kt +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandSetup.kt @@ -17,6 +17,8 @@ import com.macedonga.visor.core.protocol.Band import com.macedonga.visor.core.transport.BluetoothLeTransport import com.macedonga.visor.log.EventLog import com.macedonga.visor.login.MetaLogin +import com.macedonga.visor.setup.BondFailed +import com.macedonga.visor.setup.NearbyGlasses import com.macedonga.visor.setup.words import dagger.hilt.android.qualifiers.ApplicationContext import javax.inject.Inject @@ -40,6 +42,7 @@ class BandSetup @Inject constructor( private val store: IdentityStore, private val bands: Bands, private val link: BandLink, + private val nearby: NearbyGlasses, private val login: MetaLogin, private val log: EventLog, ) { @@ -65,7 +68,8 @@ class BandSetup @Inject constructor( if (glasses == null || !owned) return@launch fail(Reason.NO_GLASSES) link.release() try { - val candidates = bondedBands() + // Not bonded yet: the band heard nearby is bonded here, as the glasses are. + val candidates = bondedBands().ifEmpty { listOfNotNull(nearby.bandNearby()?.let { nearby.bond(it) }) } if (candidates.isEmpty()) return@launch fail(Reason.NOT_PAIRED) // A band paired more than once leaves a bond per pairing; the one that answers is the band. val address = candidates.firstOrNull { reachable(it) } ?: return@launch fail(Reason.NO_ANSWER) @@ -84,6 +88,8 @@ class BandSetup @Inject constructor( val from = if (result.deviceEcPublicKey != null) "receipt" else if (publicKey != null) "certificate" else "none" log.add("Band setup: owner changed. Serial ${result.serial}, device key from $from") _status.value = Status.Done(result.serial) + } catch (failed: BondFailed) { + fail(if (failed.why == BondFailed.Why.NO_ANSWER) Reason.NO_ANSWER else Reason.FAILED, failed.why.name) } catch (error: Exception) { val wrong = error is WrongOwnerError fail(if (wrong) Reason.WRONG_OWNER else Reason.FAILED, error.message ?: error.javaClass.simpleName) diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt index 99f8a9b3..c582b404 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt @@ -30,6 +30,7 @@ enum class AccountRow { META, WHATSAPP, FACEBOOK, INSTAGRAM } class AccountsModel @Inject constructor( val connection: GlassesConnection, private val linking: AccountLinking, + private val log: com.macedonga.visor.log.EventLog, ) : ViewModel() { // Null until the glasses answer. @@ -71,6 +72,11 @@ class AccountsModel @Inject constructor( viewModelScope.launch { try { work() + } catch (error: kotlinx.coroutines.CancellationException) { + throw error + } catch (error: Exception) { + // A failure the linking did not foresee stays on this page; it must not take the app down. + log.add("Accounts: ${error.javaClass.simpleName}: ${error.message}") } finally { _busy.value = null } diff --git a/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt b/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt index f4080424..f0af7538 100644 --- a/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt +++ b/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt @@ -135,6 +135,30 @@ class NearbyGlasses @Inject constructor(@ApplicationContext private val context: throw BondFailed(BondFailed.Why.NO_ANSWER) } + /** + * The Neural Band heard nearby, one in pairing mode before the strongest: its address, or null when none answers + * within [BAND_SCAN_MS]. Bonded by [bond], as the glasses are. + */ + @SuppressLint("MissingPermission") + suspend fun bandNearby(): String? { + val scanner = adapter?.takeIf { it.isEnabled }?.bluetoothLeScanner ?: throw BondFailed(BondFailed.Why.BLUETOOTH_OFF) + val heard = HashMap>() + val callback = object : ScanCallback() { + override fun onScanResult(callbackType: Int, result: ScanResult) { + val advert = result.scanRecord?.getManufacturerSpecificData(META_COMPANY)?.let(::advert) ?: return + if (advert.product !in PRODUCT_BANDS) return + synchronized(heard) { heard[result.device.address] = advert.pairing to result.rssi } + } + } + scanner.startScan(null, ScanSettings.Builder().setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY).build(), callback) + try { + delay(BAND_SCAN_MS) + } finally { + runCatching { scanner.stopScan(callback) } + } + return synchronized(heard) { heard.maxByOrNull { (_, v) -> (if (v.first) 1_000 else 0) + v.second }?.key } + } + private class Advert(val product: Int, val pairing: Boolean) /** Manufacturer data after the company id: legacy [product:2 LE][flags], or V2 [version][product:2 LE][reserved][flags]. */ @@ -159,6 +183,7 @@ class NearbyGlasses @Inject constructor(@ApplicationContext private val context: const val FORGET_MS = 12_000L const val SWEEP_MS = 2_000L const val BOND_MS = 90_000L + const val BAND_SCAN_MS = 15_000L const val POLL_MS = 400L val GLASSES_NAME = Regex("ray-?ban|meta|oakley|stories", RegexOption.IGNORE_CASE) From d2c8d01bf088660505172857318668f15e52e90b Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 11:04:54 +0200 Subject: [PATCH 69/90] Display: browse Meta's web app catalogue. The Web apps page gets Browse: Visor asks Meta's app store (WearablesAppStoreCatalogQuery, with the user's token, for the Display's serial and its wearables release version, which the glasses report in svc100 field 9) and lists the entries that are web apps (those with a web_app_url), each with Add, or "On the glasses" when it is there already. A catalogue app goes to the glasses like a developer one, as a dev_mode_web_app_ entry of partner_app_settings, with Meta's signed manifest in DisplayConfig field 3. An empty catalogue and a failed fetch each have their own state. Icons are not shown. Not tried on the Display yet. --- .../com/macedonga/visor/connection/WebApps.kt | 21 ++++-- .../macedonga/visor/settings/WebAppsScreen.kt | 71 +++++++++++++++++++ .../main/res/values-it/strings_webapps.xml | 4 ++ app/src/main/res/values/strings_webapps.xml | 4 ++ .../visor/core/connection/DeviceInfo.kt | 3 + .../core/identity/appstore/AppCatalogue.kt | 54 ++++++++++++++ 6 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 core/identity/src/main/kotlin/com/macedonga/visor/core/identity/appstore/AppCatalogue.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt b/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt index 824ad1ac..3f700886 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt @@ -35,7 +35,8 @@ import kotlinx.coroutines.withTimeoutOrNull import org.json.JSONArray import org.json.JSONObject -data class WebApp(val key: String, val name: String, val url: String) +/** [manifest] is Meta's signed manifest, base64, for a catalogue app; a developer app has none. */ +data class WebApp(val key: String, val name: String, val url: String, val manifest: String? = null) enum class AddResult { ADDED, EXISTS, NOT_HTTPS, FULL } @@ -83,7 +84,7 @@ class WebApps @Inject constructor( } /** Add a page, from the Display's CREATE or the Add sheet. The glasses take it with [push]. */ - fun add(name: String, url: String): AddResult { + fun add(name: String, url: String, manifest: String? = null): AddResult { val uri = Uri.parse(url) if (uri.scheme != "https" || uri.host.isNullOrEmpty()) return AddResult.NOT_HTTPS synchronized(lock) { @@ -91,7 +92,7 @@ class WebApps @Inject constructor( if (list.any { it.url == url }) return AddResult.EXISTS if (list.size >= MAX_APPS) return AddResult.FULL val key = generateSequence(0) { it + 1 }.map { "$PREFIX$it" }.first { k -> list.none { it.key == k } } - save(list + WebApp(key, name.ifEmpty { uri.host.orEmpty() }, url)) + save(list + WebApp(key, name.ifEmpty { uri.host.orEmpty() }, url, manifest)) } log.get().add("Web apps: added ${uri.host}") return AddResult.ADDED @@ -208,7 +209,13 @@ class WebApps @Inject constructor( val old = runCatching { parseProto(Base64.decode(current, Base64.NO_WRAP)) }.getOrNull().orEmpty() val kept = old.filterNot { it.number == DISPLAY_CONFIGS && entryKey(it)?.startsWith(PREFIX) == true } val ours = apps.map { app -> - message(DISPLAY_CONFIGS, listOf(text(1, app.key), message(2, listOf(message(1, listOf(text(1, app.name))), text(2, app.url))))) + // DisplayConfig {1 item, 2 url, 3 signedWebAppManifest}: the manifest goes in as the catalogue sent it. + val config = listOfNotNull( + message(1, listOf(text(1, app.name))), + text(2, app.url), + app.manifest?.let { runCatching { Base64.decode(it, Base64.DEFAULT) }.getOrNull() }?.let { ProtoField(3, 2, bytes = it) }, + ) + message(DISPLAY_CONFIGS, listOf(text(1, app.key), message(2, config))) } return encodeProto(kept + ours) } @@ -223,13 +230,15 @@ class WebApps @Inject constructor( private fun save(list: List) { _apps.value = list runCatching { - file.writeText(JSONArray(list.map { JSONObject().put("key", it.key).put("name", it.name).put("url", it.url) }).toString()) + file.writeText(JSONArray(list.map { app -> + JSONObject().put("key", app.key).put("name", app.name).put("url", app.url).also { o -> app.manifest?.let { o.put("manifest", it) } } + }).toString()) } } private fun load(): List = runCatching { val a = JSONArray(file.readText()) - List(a.length()) { i -> a.getJSONObject(i).let { WebApp(it.getString("key"), it.getString("name"), it.getString("url")) } } + List(a.length()) { i -> a.getJSONObject(i).let { WebApp(it.getString("key"), it.getString("name"), it.getString("url"), it.optString("manifest", "").ifEmpty { null }) } } }.getOrDefault(emptyList()) companion object { diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt index 4711af14..af73db48 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt @@ -28,8 +28,11 @@ import com.macedonga.visor.connection.WebApp import com.macedonga.visor.connection.WebApps import com.macedonga.visor.connection.gateOf import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.appstore.CatalogueApp +import com.macedonga.visor.core.identity.appstore.fetchAppCatalogue import com.macedonga.visor.core.media.GlassesConfig import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.login.MetaLogin import dagger.hilt.android.lifecycle.HiltViewModel import javax.inject.Inject import kotlinx.coroutines.flow.MutableStateFlow @@ -37,16 +40,26 @@ import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.map +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext enum class WebAppNote { SAVED_ON_PHONE, OPENED, NOT_OPENED, RESTART, SWITCH_NOT_SENT, NOT_HTTPS, FULL } +sealed interface CatalogueState { + data object Idle : CatalogueState + data object Loading : CatalogueState + data class Loaded(val apps: List) : CatalogueState + data object Failed : CatalogueState +} + @HiltViewModel class WebAppsModel @Inject constructor( private val webApps: WebApps, val connection: GlassesConnection, private val config: GlassesConfig, + private val login: MetaLogin, ) : ViewModel() { val apps: StateFlow> = webApps.apps val gate: StateFlow = config.state.map { gateOf(it) }.stateIn(viewModelScope, SharingStarted.Eagerly, gateOf(config.state.value)) @@ -54,6 +67,9 @@ class WebAppsModel @Inject constructor( private val _note = MutableStateFlow(null) val note: StateFlow = _note.asStateFlow() + private val _catalogue = MutableStateFlow(CatalogueState.Idle) + val catalogue: StateFlow = _catalogue.asStateFlow() + fun sync() { webApps.push() } @@ -66,6 +82,29 @@ class WebAppsModel @Inject constructor( } } + /** Fetch Meta's list for these glasses, with the user's Meta login. */ + fun browse() { + val info = connection.state.value.info + if (info == null || info.wearablesRelease.isEmpty()) { + _catalogue.value = CatalogueState.Failed + return + } + _catalogue.value = CatalogueState.Loading + viewModelScope.launch { + _catalogue.value = runCatching { + withContext(Dispatchers.IO) { fetchAppCatalogue(login.source.get(), info.serial, info.wearablesRelease) } + }.fold({ CatalogueState.Loaded(it) }, { CatalogueState.Failed }) + } + } + + fun addFromCatalogue(app: CatalogueApp) { + _note.value = when (webApps.add(app.name, app.url, app.manifest)) { + AddResult.ADDED, AddResult.EXISTS -> if (webApps.push()) null else WebAppNote.SAVED_ON_PHONE + AddResult.NOT_HTTPS -> WebAppNote.NOT_HTTPS + AddResult.FULL -> WebAppNote.FULL + } + } + fun remove(key: String) { webApps.remove(key) _note.value = if (webApps.push()) null else WebAppNote.SAVED_ON_PHONE @@ -93,7 +132,9 @@ fun WebAppsScreen(model: WebAppsModel = hiltViewModel()) { val apps by model.apps.collectAsStateWithLifecycle() val gate by model.gate.collectAsStateWithLifecycle() val note by model.note.collectAsStateWithLifecycle() + val catalogue by model.catalogue.collectAsStateWithLifecycle() var adding by remember { mutableStateOf(false) } + var browsing by remember { mutableStateOf(false) } LaunchedEffect(ready) { if (ready) model.sync() } SettingsPage(stringResource(R.string.webapps_title), light = 0f) { @@ -115,6 +156,7 @@ fun WebAppsScreen(model: WebAppsModel = hiltViewModel()) { } }.toTypedArray(), { ValueRow(stringResource(R.string.webapps_add), null, onClick = { adding = true }) }, + { ValueRow(stringResource(R.string.webapps_browse), null, onClick = { browsing = true; model.browse() }, locked = !ready) }, if (gate == OpenGate.OFF) row { ValueRow(stringResource(R.string.webapps_turn_on), null, onClick = { model.turnOn() }, locked = !ready) } else null, ) when (gate) { @@ -128,6 +170,35 @@ fun WebAppsScreen(model: WebAppsModel = hiltViewModel()) { } } + if (browsing) { + VisorSheet({ browsing = false }, full = true) { + SheetTitle(stringResource(R.string.webapps_browse)) + when (val c = catalogue) { + CatalogueState.Idle, CatalogueState.Loading -> Spinner(Modifier.padding(SIDE).padding(top = 16.dp)) + CatalogueState.Failed -> Footnote(stringResource(R.string.webapps_catalogue_failed)) + is CatalogueState.Loaded -> if (c.apps.isEmpty()) { + Footnote(stringResource(R.string.webapps_catalogue_none)) + } else { + Rows( + *c.apps.map { app -> + @Composable { + val on = apps.any { it.url == app.url } + RowFrame(app.name, onClick = null, sub = app.url) { + if (on) { + ActionPill(stringResource(R.string.webapps_on_glasses), {}, quiet = true, modifier = Modifier.padding(start = 12.dp), enabled = false) + } else { + ActionPill(stringResource(R.string.webapps_add), { model.addFromCatalogue(app) }, quiet = true, modifier = Modifier.padding(start = 12.dp)) + } + } + } + }.toTypedArray(), + ) + } + } + note?.let { Footnote(stringResource(noteText(it))) } + } + } + if (adding) { var name by remember { mutableStateOf("") } var url by remember { mutableStateOf("https://") } diff --git a/app/src/main/res/values-it/strings_webapps.xml b/app/src/main/res/values-it/strings_webapps.xml index 3d5d4476..0c7107c1 100644 --- a/app/src/main/res/values-it/strings_webapps.xml +++ b/app/src/main/res/values-it/strings_webapps.xml @@ -8,6 +8,10 @@ Nome Indirizzo (https) Nessuna app web + Sfoglia + Sugli occhiali + Impossibile caricare il catalogo + Nessuna app web per questi occhiali Attiva le app web Gli occhiali non hanno questo interruttore Collega gli occhiali per leggerlo diff --git a/app/src/main/res/values/strings_webapps.xml b/app/src/main/res/values/strings_webapps.xml index 6d90a956..0f1d814a 100644 --- a/app/src/main/res/values/strings_webapps.xml +++ b/app/src/main/res/values/strings_webapps.xml @@ -8,6 +8,10 @@ Name Address (https) No web apps + Browse + On the glasses + Could not load the catalogue + No web apps for these glasses Turn on web apps The glasses do not have this switch Connect the glasses to read this diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/DeviceInfo.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/DeviceInfo.kt index 6443841c..6e014b2b 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/DeviceInfo.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/DeviceInfo.kt @@ -8,6 +8,8 @@ data class DeviceInfo( val serial: String, val deviceUuid: String = "", val model: String = "", + /** The wearables release version (field 9), e.g. 129.0.0.218.272: the store asks by it. */ + val wearablesRelease: String = "", ) private fun uuidShaped(text: String): Boolean = @@ -22,5 +24,6 @@ fun decodePeerDeviceInfo(payload: ByteArray): DeviceInfo { serial = String(fields.bytes(2), Charsets.ISO_8859_1), deviceUuid = if (uuidShaped(deviceUuid)) deviceUuid else "", model = String(fields.bytes(8), Charsets.ISO_8859_1), + wearablesRelease = String(fields.bytes(9), Charsets.ISO_8859_1), ) } diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/appstore/AppCatalogue.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/appstore/AppCatalogue.kt new file mode 100644 index 00000000..ff16f2f7 --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/appstore/AppCatalogue.kt @@ -0,0 +1,54 @@ +package com.macedonga.visor.core.identity.appstore + +import com.macedonga.visor.core.identity.HttpPost +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.language.graphPost +import com.macedonga.visor.core.identity.token.Token +import com.macedonga.visor.core.identity.token.text +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put + +/** A web app in Meta's catalogue for the glasses. [id] is Meta's; the glasses key it when it is written. */ +data class CatalogueApp( + val id: String, + val name: String, + val iconUrl: String, + val url: String, + /** Base64 of Meta's SignedWebAppManifest. Null when the entry has none. */ + val manifest: String?, +) + +private const val CATALOGUE_DOC = "162073133818421355301019414873" + +/** + * WearablesAppStoreCatalogQuery, the store's list for [serial] running [osVersion], the glasses' wearables release + * version. Only web apps come back: an app with a web address; the others name a package and are not listed here. + */ +fun fetchAppCatalogue( + token: Token, + serial: String, + osVersion: String, + http: HttpPost = UrlConnectionPost, +): List { + val variables = buildJsonObject { + put("osVersion", osVersion) + put("serial", serial) + } + val (_, json) = graphPost(token, "WearablesAppStoreCatalogQuery", CATALOGUE_DOC, variables, http) + val edges = json.obj("data")?.obj("viewer")?.obj("user")?.obj("discovery_applications")?.get("edges") as? JsonArray + ?: return emptyList() + return edges.mapNotNull { edge -> + val node = (edge as? JsonObject)?.obj("node") ?: return@mapNotNull null + CatalogueApp( + id = node.text("id") ?: return@mapNotNull null, + name = node.text("display_name") ?: return@mapNotNull null, + iconUrl = node.text("pdp_icon_url").orEmpty(), + url = node.text("web_app_url")?.takeIf { it.isNotEmpty() } ?: return@mapNotNull null, + manifest = node.text("signed_web_app_manifest")?.takeIf { it.isNotEmpty() }, + ) + } +} + +private fun JsonObject.obj(key: String): JsonObject? = this[key] as? JsonObject From c7409f2fbf1f2fd3eb12841d71f66e1e59b039e4 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 11:05:20 +0200 Subject: [PATCH 70/90] Live: decode H.264 as well as HEVC. The Display's camera service names only H.264, where the Ray-Ban Meta streams HEVC, and Visor decoded HEVC only. The stream's codec is now read from its parameter sets (or its first pictures) and every user follows it: the Live view's decoder, the still frame Live AI hands the model, the recording and the replay ring, and the broadcast, which sends H.264 as it is. The event log names the codec once per stream. An HEVC stream takes the same path as before. Not tried on the Display yet. --- .../macedonga/visor/core/media/Broadcast.kt | 17 ++-- .../macedonga/visor/core/media/FrameGrab.kt | 9 +- .../com/macedonga/visor/core/media/Hevc.kt | 4 +- .../macedonga/visor/core/media/LiveDecoder.kt | 10 +- .../macedonga/visor/core/media/LiveOutputs.kt | 2 +- .../visor/core/media/LiveRecorder.kt | 25 ++--- .../macedonga/visor/core/media/LiveStream.kt | 34 +++++-- .../macedonga/visor/core/media/VideoCodec.kt | 93 +++++++++++++++++++ 8 files changed, 157 insertions(+), 37 deletions(-) create mode 100644 core/media/src/main/kotlin/com/macedonga/visor/core/media/VideoCodec.kt diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Broadcast.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Broadcast.kt index 9b8e7e53..5f9d7a1f 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Broadcast.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Broadcast.kt @@ -83,16 +83,19 @@ class Broadcast(private val scope: CoroutineScope) { var transcoder: AvcTranscoder? = null try { rtmp.connect(scope) - val sps = hevcParameterSets(config)[33]?.firstOrNull()?.let(::parseHevcSps) - val width = sps?.width ?: 1080 - val height = sps?.height ?: 1920 + // H.264 from the glasses goes out as it is; HEVC goes out as HEVC, or is re-encoded to H.264. + val input = configCodec(config) + val out = if (input == VideoCodec.AVC) BroadcastCodec.H264 else target.codec + val (width, height) = videoSize(config) ?: (1080 to 1920) val audio = audioFormat() - rtmp.metadata(width, height, frameRate, target.codec, audio) + rtmp.metadata(width, height, frameRate, out, audio) if (audio != null) rtmp.audioHeader(audio) var base = -1L fun ms(pts: Long) = ((pts - base) / 1000).coerceAtLeast(0) - if (target.codec == BroadcastCodec.HEVC) { + if (out == BroadcastCodec.HEVC) { rtmp.videoHeader(BroadcastCodec.HEVC, checkNotNull(hvcC(config)) { "the stream's parameter sets do not parse" }) + } else if (input == VideoCodec.AVC) { + rtmp.videoHeader(BroadcastCodec.H264, checkNotNull(avcC(config)) { "the stream's parameter sets do not parse" }) } else { var headerSent = false transcoder = AvcTranscoder(width, height, config, bitrate, frameRate, @@ -119,8 +122,8 @@ class Broadcast(private val scope: CoroutineScope) { check(!transcoder.failed) { "the phone's H.264 encoder stopped" } transcoder.feed(item.frame) } else { - val units = annexBToLengthPrefixed(item.frame.data, dropParameterSets = true) - rtmp.video(BroadcastCodec.HEVC, ms(item.frame.ptsUs), item.frame.isKey, units) + val units = annexBToLengthPrefixed(item.frame.data, dropParameterSets = out == BroadcastCodec.HEVC) + rtmp.video(out, ms(item.frame.ptsUs), item.frame.isKey, units) } sent += item.frame.data.size } diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/FrameGrab.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/FrameGrab.kt index b87bb6b0..5c0de51a 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/FrameGrab.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/FrameGrab.kt @@ -17,12 +17,13 @@ import kotlinx.coroutines.withContext * it works with no screen showing the stream. Null if the decoder gave nothing. */ suspend fun grabJpeg(picture: Replay, quality: Int = 92): ByteArray? = withContext(Dispatchers.Default) { - val sps = hevcParameterSets(picture.config)[33]?.firstOrNull()?.let(::parseHevcSps) ?: return@withContext null - val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_HEVC, sps.width, sps.height).apply { + val (width, height) = videoSize(picture.config) ?: return@withContext null + val videoCodec = picture.video.firstOrNull()?.codec ?: configCodec(picture.config) + val format = MediaFormat.createVideoFormat(videoCodec.mime, width, height).apply { setByteBuffer("csd-0", ByteBuffer.wrap(picture.config)) setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible) } - val codec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_HEVC) + val codec = MediaCodec.createDecoderByType(videoCodec.mime) try { codec.configure(format, null, null, 0) codec.start() @@ -49,7 +50,7 @@ suspend fun grabJpeg(picture: Replay, quality: Int = 92): ByteArray? = withConte } val o = codec.dequeueOutputBuffer(info, 5_000) if (o >= 0) { - if (info.size > 0) codec.getOutputImage(o)?.use { last = jpegOf(it, sps.width, sps.height, quality) } + if (info.size > 0) codec.getOutputImage(o)?.use { last = jpegOf(it, width, height, quality) } codec.releaseOutputBuffer(o, false) if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) ended = true } diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Hevc.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Hevc.kt index d8df0378..906af747 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Hevc.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Hevc.kt @@ -122,7 +122,7 @@ fun hvcC(config: ByteArray): ByteArray? { } /** Drops the emulation-prevention bytes (00 00 03) from a NAL unit's payload. */ -private fun unescape(unit: ByteArray): ByteArray { +internal fun unescape(unit: ByteArray): ByteArray { val out = ByteArrayOutputStream(unit.size) var zeros = 0 for (b in unit) { @@ -136,7 +136,7 @@ private fun unescape(unit: ByteArray): ByteArray { return out.toByteArray() } -private class BitReader(private val data: ByteArray, startByte: Int) { +internal class BitReader(private val data: ByteArray, startByte: Int) { private var pos = startByte * 8 fun bits(n: Int): Int { diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveDecoder.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveDecoder.kt index b38b2874..fac08553 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveDecoder.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveDecoder.kt @@ -12,7 +12,7 @@ import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.TimeUnit /** - * Hardware HEVC decoding onto a Surface, one decoder per stream, asynchronous so a decoded frame is + * Hardware HEVC or H.264 decoding onto a Surface, one decoder per stream, asynchronous so a decoded frame is * shown the moment it exists rather than when the next one arrives. It is configured for the size * asked for with room up to 1920 either way, because a decoder fed a stream larger than its * configured size draws garbage on many phones. @@ -50,7 +50,7 @@ class LiveDecoder( fun feed(frame: VideoFrame, config: ByteArray?, show: Boolean = true): Boolean { val c = codec ?: run { if (!frame.isKey) return false - start(config) + start(config, frame.codec) } if (frame.data.size > MAX_INPUT) return false return queue(c, frame.data, frame.ptsUs, 0).also { if (it && !show) hidden.add(frame.ptsUs) } @@ -83,8 +83,8 @@ class LiveDecoder( false } - private fun start(config: ByteArray?): MediaCodec { - val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_HEVC, width, height) + private fun start(config: ByteArray?, videoCodec: VideoCodec): MediaCodec { + val format = MediaFormat.createVideoFormat(videoCodec.mime, width, height) format.setInteger(MediaFormat.KEY_MAX_WIDTH, MAX_SIDE) format.setInteger(MediaFormat.KEY_MAX_HEIGHT, MAX_SIDE) format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, MAX_INPUT) @@ -93,7 +93,7 @@ class LiveDecoder( if (config != null) format.setByteBuffer("csd-0", ByteBuffer.wrap(config)) val t = HandlerThread("live-decoder").also { it.start(); thread = it } - return MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_HEVC).also { + return MediaCodec.createDecoderByType(videoCodec.mime).also { it.setCallback(object : MediaCodec.Callback() { override fun onInputBufferAvailable(codec: MediaCodec, index: Int) { freeInputs.offer(index) diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveOutputs.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveOutputs.kt index 467a9a42..41848858 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveOutputs.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveOutputs.kt @@ -70,7 +70,7 @@ class LiveOutputs(private val context: Context) { } fun onVideo(input: VideoFrame) { - val frame = VideoFrame(input.seq, rebase(input.ptsUs), input.data, input.arrivedNs) + val frame = VideoFrame(input.seq, rebase(input.ptsUs), input.data, input.arrivedNs, input.codec) replay.add(frame) mp4?.let { r -> runCatching { synchronized(r) { r.video(frame) } } } broadcast.video(frame) diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveRecorder.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveRecorder.kt index 51a1bd62..48ff254a 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveRecorder.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveRecorder.kt @@ -17,7 +17,7 @@ class AudioPacket(val ptsUs: Long, val data: ByteArray) class AacFormat(val sampleRate: Int, val channels: Int, val asc: ByteArray) /** - * One MP4 in Movies/Visor, written as the frames come: the glasses' HEVC as it is, the audio as AAC. The file + * One MP4 in Movies/Visor, written as the frames come: the glasses' video as it is (HEVC or H.264), the audio as AAC. The file * stays pending, so the gallery does not list it, until [finish]. Audio before the first video frame is dropped, * and everything is timed from that frame. */ @@ -33,7 +33,6 @@ class Mp4Recording(private val context: Context, config: ByteArray, audio: AacFo private val info = MediaCodec.BufferInfo() init { - val sps = hevcParameterSets(config)[33]?.firstOrNull()?.let(::parseHevcSps) val values = ContentValues().apply { put(MediaStore.MediaColumns.DISPLAY_NAME, name) put(MediaStore.MediaColumns.MIME_TYPE, "video/mp4") @@ -44,8 +43,10 @@ class Mp4Recording(private val context: Context, config: ByteArray, audio: AacFo uri = checkNotNull(context.contentResolver.insert(MediaStore.Video.Media.EXTERNAL_CONTENT_URI, values)) { "MediaStore refused the file" } fd = checkNotNull(context.contentResolver.openFileDescriptor(uri, "rw")) { "no file to write" } muxer = MediaMuxer(fd.fileDescriptor, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) - val video = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_HEVC, sps?.width ?: 1080, sps?.height ?: 1920) - video.setByteBuffer("csd-0", ByteBuffer.wrap(onlyParameterSets(config))) + val codec = configCodec(config) + val (width, height) = videoSize(config) ?: (1080 to 1920) + val video = MediaFormat.createVideoFormat(codec.mime, width, height) + video.setByteBuffer("csd-0", ByteBuffer.wrap(onlyParameterSets(codec, config))) videoTrack = muxer.addTrack(video) audioTrack = if (audio != null) { val a = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, audio.sampleRate, audio.channels) @@ -64,7 +65,7 @@ class Mp4Recording(private val context: Context, config: ByteArray, audio: AacFo firstPts = frame.ptsUs } if (frame.ptsUs < lastPts) return - val data = withoutParameterSets(frame.data) + val data = withoutParameterSets(frame.codec, frame.data) info.set(0, data.size, frame.ptsUs - firstPts, if (frame.isKey) MediaCodec.BUFFER_FLAG_KEY_FRAME else 0) muxer.writeSampleData(videoTrack, ByteBuffer.wrap(data), info) lastPts = frame.ptsUs @@ -103,7 +104,7 @@ class ReplayRing(@Volatile var windowUs: Long) { @Synchronized fun config(next: ByteArray) { - val size = { c: ByteArray? -> c?.let { hevcParameterSets(it)[33]?.firstOrNull()?.let(::parseHevcSps) }?.let { it.width to it.height } } + val size = { c: ByteArray? -> c?.let(::videoSize) } if (size(config) != size(next)) { video.clear() audio.clear() @@ -155,14 +156,14 @@ class ReplayRing(@Volatile var windowUs: Long) { class Replay(val config: ByteArray, val video: List, val audio: List) -/** Only the VPS, SPS and PPS of [config], Annex-B, as csd-0 wants them. */ -private fun onlyParameterSets(config: ByteArray): ByteArray = annexB(annexBUnits(config).filter { hevcNalType(it) in 32..34 }) +/** Only the parameter sets of [config], Annex-B, as csd-0 wants them. */ +private fun onlyParameterSets(codec: VideoCodec, config: ByteArray): ByteArray = annexB(annexBUnits(config).filter { codec.isParameterSet(it) }) -/** A frame with any parameter sets in it taken out: MP4's hvc1 keeps them in the sample entry only. */ -private fun withoutParameterSets(data: ByteArray): ByteArray { +/** A frame with any parameter sets in it taken out: MP4 keeps them in the sample entry only. */ +private fun withoutParameterSets(codec: VideoCodec, data: ByteArray): ByteArray { val units = annexBUnits(data) - if (units.none { hevcNalType(it) in 32..34 }) return data - return annexB(units.filter { hevcNalType(it) !in 32..34 }) + if (units.none { codec.isParameterSet(it) }) return data + return annexB(units.filter { !codec.isParameterSet(it) }) } private fun annexB(units: List): ByteArray { diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt index bb60a9a9..7a6840da 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt @@ -50,6 +50,7 @@ import kotlinx.coroutines.flow.SharedFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asSharedFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch @@ -93,8 +94,14 @@ enum class StreamQuality(val width: Int, val height: Int, val label: String) { * One reassembled frame: the glasses' presentation time in microseconds, then Annex-B video. [seq] * counts frames, so a reader can tell one was lost and wait for the next keyframe. */ -class VideoFrame(val seq: Long, val ptsUs: Long, val data: ByteArray, val arrivedNs: Long = System.nanoTime()) { - val isKey: Boolean by lazy { nalTypes(data).any { it in HEVC_KEY_NALS } } +class VideoFrame( + val seq: Long, + val ptsUs: Long, + val data: ByteArray, + val arrivedNs: Long = System.nanoTime(), + val codec: VideoCodec = VideoCodec.HEVC, +) { + val isKey: Boolean by lazy { codec.isKeyFrame(data) } } /** @@ -134,6 +141,11 @@ class StreamReceiver : Responder { private val _codecConfig = MutableStateFlow(null) val codecConfig: StateFlow = _codecConfig.asStateFlow() + // HEVC until the parameter sets or a picture say otherwise. + private var codec = VideoCodec.HEVC + private val _videoCodec = MutableStateFlow(null) + val videoCodec: StateFlow = _videoCodec.asStateFlow() + private val audioChunks = ByteArrayOutputStream() private val _audio = MutableSharedFlow(extraBufferCapacity = 256, onBufferOverflow = BufferOverflow.DROP_OLDEST) val audio: SharedFlow = _audio.asSharedFlow() @@ -183,9 +195,19 @@ class StreamReceiver : Responder { fpsCount = 0 gapMax = 0 _codecConfig.value = null + codec = VideoCodec.HEVC + _videoCodec.value = null _stats.value = StreamStats() } + private fun noteCodec(data: ByteArray) { + if (_videoCodec.value != null) return + detectVideoCodec(data)?.let { + codec = it + _videoCodec.value = it + } + } + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { val payload = message.payload when (message.info.messageType) { @@ -201,6 +223,7 @@ class StreamReceiver : Responder { // Field 2 says which; failing that, video parameter sets are Annex-B and an AudioSpecificConfig is not. if (type == CodecType.VIDEO || (type != CodecType.AUDIO && nalTypes(config).isNotEmpty())) { _codecConfig.value = config + noteCodec(config) recording?.write(config) } else { aacFormatOf(config)?.let { _aac.value = it } @@ -275,7 +298,8 @@ class StreamReceiver : Responder { maxGapMs = if (gap >= 0) gap else it.maxGapMs, ) } - _frames.tryEmit(VideoFrame(seq++, pts, data)) + noteCodec(data) + _frames.tryEmit(VideoFrame(seq++, pts, data, codec = codec)) } } @@ -449,6 +473,7 @@ class LiveStream( scope.launch { receiver.control.collect(::onControl) } scope.launch { receiver.frames.collect(outputs::onVideo) } scope.launch { receiver.codecConfig.collect { if (it != null) outputs.onConfig(it) } } + scope.launch { receiver.videoCodec.filterNotNull().collect { say("Video codec: ${it.label}") } } scope.launch { receiver.audio.collect(outputs::onAudio) } scope.launch { receiver.aac.collect { if (it != null) outputs.onAudioFormat(it) } } scope.launch { @@ -1095,9 +1120,6 @@ private fun snappWords(code: Long): String = when (code) { else -> "error $code" } -// HEVC NAL unit types: 16-21 are IRAP (keyframes), 32-34 VPS/SPS/PPS. -private val HEVC_KEY_NALS = setOf(16, 17, 18, 19, 20, 21) - /** The HEVC NAL unit types in an Annex-B buffer. */ fun nalTypes(data: ByteArray): List { val out = ArrayList() diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/VideoCodec.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/VideoCodec.kt new file mode 100644 index 00000000..0169eaa4 --- /dev/null +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/VideoCodec.kt @@ -0,0 +1,93 @@ +package com.macedonga.visor.core.media + +import android.media.MediaFormat + +/** The video codecs a glasses stream may carry: HEVC from the Gen 2, H.264 (AVC) possibly from the Display. */ +enum class VideoCodec(val mime: String, val label: String) { + HEVC(MediaFormat.MIMETYPE_VIDEO_HEVC, "HEVC"), + AVC(MediaFormat.MIMETYPE_VIDEO_AVC, "H.264"), +} + +/** + * The codec of Annex-B bytes, from an HEVC VPS, SPS or PPS (0x40, 0x42, 0x44 then 0x01) or an H.264 SPS + * (nal_ref_idc set, type 7). HEVC base-layer units never start with a byte that reads as the H.264 SPS, so the + * two cannot be confused. Null when the bytes show neither, such as slices with no parameter sets. + */ +fun detectVideoCodec(data: ByteArray): VideoCodec? { + val units = annexBUnits(data) + if (units.any { it.size >= 2 && (it[0].toInt() and 0xFF) in setOf(0x40, 0x42, 0x44) && it[1].toInt() == 1 }) return VideoCodec.HEVC + if (units.any { isAvcSpsHeader(it[0].toInt() and 0xFF) }) return VideoCodec.AVC + return null +} + +private fun isAvcSpsHeader(b: Int): Boolean = (b and 0x80) == 0 && (b and 0x1F) == 7 && (b and 0x60) != 0 + +/** The codec of a config, HEVC unless it shows H.264. */ +fun configCodec(config: ByteArray): VideoCodec = detectVideoCodec(config) ?: VideoCodec.HEVC + +/** Width and height from the first SPS in [config], whichever codec it is in. */ +fun videoSize(config: ByteArray): Pair? = when (configCodec(config)) { + VideoCodec.HEVC -> hevcParameterSets(config)[33]?.firstOrNull()?.let(::parseHevcSps)?.let { it.width to it.height } + VideoCodec.AVC -> annexBUnits(config).firstOrNull { (it[0].toInt() and 0x1F) == 7 }?.let(::parseAvcSps) +} + +/** Parameter sets: HEVC VPS, SPS, PPS (32 to 34); H.264 SPS and PPS (7, 8). */ +fun VideoCodec.isParameterSet(unit: ByteArray): Boolean = when (this) { + VideoCodec.HEVC -> hevcNalType(unit) in 32..34 + VideoCodec.AVC -> (unit[0].toInt() and 0x1F) in 7..8 +} + +/** A keyframe: HEVC IRAP units (16 to 21), H.264 IDR slices (5). */ +fun VideoCodec.isKeyFrame(data: ByteArray): Boolean = when (this) { + VideoCodec.HEVC -> nalTypes(data).any { it in 16..21 } + VideoCodec.AVC -> annexBUnits(data).any { (it[0].toInt() and 0x1F) == 5 } +} + +private val HIGH_PROFILES = setOf(100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135) + +/** Width and height of an H.264 SPS, cropping applied. Streams with scaling matrices are not read. */ +fun parseAvcSps(unit: ByteArray): Pair? = runCatching { + val r = BitReader(unescape(unit), 1) + val profile = r.bits(8) + r.bits(16) // constraint flags and level + r.ue() // seq_parameter_set_id + var chroma = 1 + if (profile in HIGH_PROFILES) { + chroma = r.ue() + if (chroma == 3) r.bits(1) + r.ue() + r.ue() // bit depths + r.bits(1) // qpprime_y_zero_transform_bypass + check(r.bits(1) == 0) { "scaling matrices" } + } + r.ue() // log2_max_frame_num_minus4 + when (r.ue()) { + 0 -> r.ue() + 1 -> { + r.bits(1) + r.ue() + r.ue() + repeat(r.ue()) { r.ue() } + } + } + r.ue() // max_num_ref_frames + r.bits(1) // gaps_in_frame_num_value_allowed_flag + val widthMbs = r.ue() + 1 + val mapUnits = r.ue() + 1 + val frameMbsOnly = r.bits(1) + if (frameMbsOnly == 0) r.bits(1) // mb_adaptive_frame_field_flag + r.bits(1) // direct_8x8_inference_flag + var left = 0 + var right = 0 + var top = 0 + var bottom = 0 + if (r.bits(1) == 1) { + left = r.ue() + right = r.ue() + top = r.ue() + bottom = r.ue() + } + val cropX = if (chroma == 1 || chroma == 2) 2 else 1 + val cropY = (if (chroma == 1) 2 else 1) * (2 - frameMbsOnly) + (widthMbs * 16 - cropX * (left + right)) to ((2 - frameMbsOnly) * mapUnits * 16 - cropY * (top + bottom)) +}.getOrNull() From 35786d351e5811081c21cdb141cbbdd0c3f3ae07 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 11:07:10 +0200 Subject: [PATCH 71/90] Fix four faults found in review. The Display's location request picks a provider that exists and is enabled (fused, GPS when precise, network, passive) and answers ERROR_OTHER when there is none or the request fails, where it threw and left the glasses waiting. A caption pack is downloaded only when the glasses are connected. The L2CAP open's watchdog and a connect that has just succeeded no longer both win: before, the watchdog could close a socket that had opened and Visor kept the closed one. A bands file that cannot be read is logged and kept aside as bands.json.bad instead of being overwritten. --- .../kotlin/com/macedonga/visor/band/Bands.kt | 41 +++++++++++++------ .../visor/connection/LocationShare.kt | 21 +++++++--- .../macedonga/visor/display/CaptionPacks.kt | 1 + .../core/transport/BluetoothLeTransport.kt | 11 +++-- 4 files changed, 53 insertions(+), 21 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/band/Bands.kt b/app/src/main/kotlin/com/macedonga/visor/band/Bands.kt index ff6f1fb9..7c3bf079 100644 --- a/app/src/main/kotlin/com/macedonga/visor/band/Bands.kt +++ b/app/src/main/kotlin/com/macedonga/visor/band/Bands.kt @@ -2,6 +2,7 @@ package com.macedonga.visor.band import android.content.Context import android.util.Base64 +import android.util.Log import com.macedonga.visor.core.identity.KeystoreSealer import dagger.hilt.android.qualifiers.ApplicationContext import java.io.File @@ -30,6 +31,8 @@ data class BandRecord(val serial: String, val address: String, val publicKey: By class Bands @Inject constructor(@ApplicationContext context: Context) { private val file = File(context.filesDir, "bands.json") private val sealer = KeystoreSealer(alias = "visor.band") + // Set when bands.json exists but cannot be read; the next write keeps it aside as bands.json.bad first. + private var unreadable = false private val _all = MutableStateFlow(read()) val all: StateFlow> = _all.asStateFlow() @@ -48,21 +51,35 @@ class Bands @Inject constructor(@ApplicationContext context: Context) { write() } - private fun read(): List = runCatching { - val list = JSONArray(file.readText()) - (0 until list.length()).map { i -> - val o = list.getJSONObject(i) - BandRecord( - serial = o.getString("serial"), - address = o.getString("address"), - publicKey = o.optString("public_key").takeIf { it.isNotEmpty() }?.let { Base64.decode(it, Base64.NO_WRAP) }, - glasses = o.optString("glasses").takeIf { it.isNotEmpty() }, - key = o.optString("key").takeIf { it.isNotEmpty() }?.let { sealer.unseal(Base64.decode(it, Base64.NO_WRAP)) }, - ) + private fun read(): List { + if (!file.exists()) return emptyList() + return runCatching { + val list = JSONArray(file.readText()) + (0 until list.length()).map { i -> + val o = list.getJSONObject(i) + BandRecord( + serial = o.getString("serial"), + address = o.getString("address"), + publicKey = o.optString("public_key").takeIf { it.isNotEmpty() }?.let { Base64.decode(it, Base64.NO_WRAP) }, + glasses = o.optString("glasses").takeIf { it.isNotEmpty() }, + key = o.optString("key").takeIf { it.isNotEmpty() }?.let { sealer.unseal(Base64.decode(it, Base64.NO_WRAP)) }, + ) + } + }.getOrElse { + Log.w("Visor", "bands.json cannot be read", it) + unreadable = true + emptyList() } - }.getOrDefault(emptyList()) + } private fun write() { + if (unreadable) { + runCatching { file.copyTo(File(file.parentFile, "${file.name}.bad"), overwrite = true) }.onFailure { + Log.w("Visor", "bands.json could not be kept aside", it) + return + } + unreadable = false + } val list = JSONArray() for (b in _all.value) { list.put( diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/LocationShare.kt b/app/src/main/kotlin/com/macedonga/visor/connection/LocationShare.kt index 15a08e7e..8e8bf5d1 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/LocationShare.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/LocationShare.kt @@ -63,13 +63,20 @@ class LocationShare @Inject constructor( requests.finish(logId, "no location permission", failed = true) return } - val provider = when { - manager.allProviders.contains(LocationManager.FUSED_PROVIDER) -> LocationManager.FUSED_PROVIDER - precise && manager.isProviderEnabled(LocationManager.GPS_PROVIDER) -> LocationManager.GPS_PROVIDER - else -> LocationManager.NETWORK_PROVIDER + val provider = listOf(LocationManager.FUSED_PROVIDER, LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER, LocationManager.PASSIVE_PROVIDER) + .firstOrNull { p -> (p != LocationManager.GPS_PROVIDER || precise) && manager.allProviders.contains(p) && manager.isProviderEnabled(p) } + if (provider == null) { + send(Wire.fieldVarint(1, STATUS_OTHER_ERROR)) + requests.finish(logId, "no location provider enabled", failed = true) + return + } + try { + manager.getLastKnownLocation(provider)?.let { onLocationChanged(it) } + manager.requestLocationUpdates(provider, INTERVAL_MS, 0f, this, Looper.getMainLooper()) + } catch (error: RuntimeException) { + send(Wire.fieldVarint(1, STATUS_OTHER_ERROR)) + requests.finish(logId, "location updates refused: ${error.message}", failed = true) } - manager.getLastKnownLocation(provider)?.let { onLocationChanged(it) } - manager.requestLocationUpdates(provider, INTERVAL_MS, 0f, this, Looper.getMainLooper()) } fun stop() { @@ -107,6 +114,8 @@ class LocationShare @Inject constructor( private const val ACCURACY_BEST = 0L private const val STATUS_OK = 0L private const val STATUS_NO_PERMISSION = 1L + // ERROR_OTHER in LocationResult: no provider can answer (notes/213, location-to-glasses.md:191). + private const val STATUS_OTHER_ERROR = 2L private const val INTERVAL_MS = 2_000L private const val SOURCE_LOCATION = -3 } diff --git a/app/src/main/kotlin/com/macedonga/visor/display/CaptionPacks.kt b/app/src/main/kotlin/com/macedonga/visor/display/CaptionPacks.kt index fac0ed4f..bf907daf 100644 --- a/app/src/main/kotlin/com/macedonga/visor/display/CaptionPacks.kt +++ b/app/src/main/kotlin/com/macedonga/visor/display/CaptionPacks.kt @@ -96,6 +96,7 @@ class CaptionPacks @Inject constructor( if (job?.isActive == true) return job = scope.launch { try { + if (connection.state.value.link != LinkState.READY) error("the glasses are not connected") _work.value = PackWork.Working(feature, PackStep.DOWNLOADING) val offer = _offers.value[feature] ?: languages.offersFor(listOf(FEATURE + feature)).firstOrNull() ?: error("Meta has no pack for these glasses") diff --git a/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt b/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt index 3c807041..ea7fba41 100644 --- a/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt +++ b/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt @@ -12,6 +12,7 @@ import android.content.Context import android.content.pm.PackageManager import java.io.IOException import java.util.UUID +import java.util.concurrent.atomic.AtomicBoolean import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.Dispatchers @@ -78,20 +79,24 @@ class BluetoothLeTransport( } // A healthy open takes a fraction of a second; after Visor died with a channel open, Android's own wait is // 20 to 30 s for nothing. Closing the socket ends the wait. - var timedOut = false + // Whoever flips [settled] first wins: the timer closes the socket only if it gets there before the connect ends. + val settled = AtomicBoolean(false) + val timedOut = AtomicBoolean(false) val watchdog = java.util.Timer(true).apply { schedule(object : java.util.TimerTask() { override fun run() { - timedOut = true + if (!settled.compareAndSet(false, true)) return + timedOut.set(true) runCatching { opening.close() } } }, L2CAP_OPEN_MS) } try { opening.connect() + if (!settled.compareAndSet(false, true)) throw IOException("the watchdog closed the socket") } catch (error: IOException) { runCatching { opening.close() } - val why = if (timedOut) "no answer in ${L2CAP_OPEN_MS / 1000} s" else error.message + val why = if (timedOut.get()) "no answer in ${L2CAP_OPEN_MS / 1000} s" else error.message throw TransportConnectException(ConnectFailure.NO_ANSWER, "the glasses did not open L2CAP psm $psm: $why", error) } finally { watchdog.cancel() From 27bc3c2ff1218fdde23bd3b3c92e6390dcf3da22 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 11:09:07 +0200 Subject: [PATCH 72/90] Keep the Ray-Ban Meta as on main. A review of the branch against main found Display behaviour reaching other glasses; each is now gated to the Display: the Live AI keep-alive's interactionId (ev392 field 4), mirroring the SMS app's notifications (other glasses get texts over Bluetooth, so they came twice) and keeping their conversations, the notification removal (svc46 type 1022), and opening svc26 with the sharing service first among airship's takers. AirShield's Multikey stays on when the peer leaves EnableEncryption field 5 out, as it always was. A Meta call answered with OAuth error 190 now forgets the token and tries once more with a fresh one everywhere (firmware, band and pack offers, the frame look, account linking), not only in the login's own GraphQL. Device state held while svc100 is unanswered is applied after 5 s, as before the hold. --- .../com/macedonga/visor/band/BandUpdate.kt | 6 ++--- .../visor/connection/AccountLinking.kt | 5 ++-- .../visor/connection/PhoneSharing.kt | 7 +++-- .../visor/firmware/FirmwareUpdate.kt | 3 ++- .../visor/language/LanguageInstaller.kt | 4 +-- .../com/macedonga/visor/login/MetaLogin.kt | 14 +++++----- .../macedonga/visor/notify/MirrorListener.kt | 6 +++++ .../com/macedonga/visor/stage/Frames.kt | 5 ++-- .../visor/core/assistant/Assistant.kt | 2 +- .../macedonga/visor/core/assistant/LiveAi.kt | 15 ++++++++--- .../core/connection/GlassesConnection.kt | 19 ++++++++++++-- .../core/identity/language/GlassesLook.kt | 2 ++ .../core/identity/language/LanguagePacks.kt | 2 ++ .../core/identity/language/PairPicture.kt | 2 ++ .../visor/core/identity/token/AccountToken.kt | 1 + .../visor/core/identity/token/Token.kt | 26 ++++++++++++++++++- .../visor/core/identity/token/WebLogin.kt | 2 +- .../macedonga/visor/core/protocol/Messages.kt | 3 ++- 18 files changed, 97 insertions(+), 27 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandUpdate.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandUpdate.kt index e5cdb24a..41b5fb7c 100644 --- a/app/src/main/kotlin/com/macedonga/visor/band/BandUpdate.kt +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandUpdate.kt @@ -1,5 +1,6 @@ package com.macedonga.visor.band +import com.macedonga.visor.core.identity.token.withToken import android.content.Context import android.util.Base64 import com.macedonga.visor.core.connection.GlassesConnection @@ -95,7 +96,7 @@ class BandUpdate @Inject constructor( val serial = serial() val firmware = link.holdFor { withSession { deviceInfo().firmware } } // The Meta app never sends a version below 1. - val found = fetchBandOffer(login.source.get(), serial, firmware.coerceAtLeast(1).toString()) + val found = login.source.withToken { fetchBandOffer(it, serial, firmware.coerceAtLeast(1).toString()) } _offer.value = found _checked.value = true _work.value = BandWork.Idle @@ -117,12 +118,11 @@ class BandUpdate @Inject constructor( val zip = File(File(context.cacheDir, "band-update").apply { mkdirs() }, "band.zip") try { val serial = serial() - val token = login.source.get() download(offer, zip) val blobs = unpack(zip) _work.value = BandWork.Installing() log.add("Band firmware: installing ${offer.targetVersion}") - link.holdFor { installOn(offer, serial, token, blobs) } + login.source.withToken { token -> link.holdFor { installOn(offer, serial, token, blobs) } } _offer.value = null _work.value = BandWork.Installed log.add("Band firmware: ${offer.targetVersion} is installed") diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt b/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt index 45d5b719..fb25e59a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt @@ -1,5 +1,6 @@ package com.macedonga.visor.connection +import com.macedonga.visor.core.identity.token.withToken import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.identity.token.LinkedAccountType import com.macedonga.visor.core.identity.token.TokenError @@ -52,7 +53,7 @@ class AccountLinking @Inject constructor( suspend fun syncMeta(): LinkOutcome { if (!login.source.hasCachedLogin) return LinkOutcome.LOGGED_OUT val identity = try { - withContext(Dispatchers.IO) { exchangeAccountToken(login.source.get()) } + withContext(Dispatchers.IO) { login.source.withToken { exchangeAccountToken(it) } } } catch (error: TokenError) { log.add("Accounts: ${error.message}") return if (error.unreachable) LinkOutcome.UNREACHABLE else LinkOutcome.REFUSED @@ -87,7 +88,7 @@ class AccountLinking @Inject constructor( if (!login.source.hasCachedLogin) return LinkOutcome.LOGGED_OUT val found = try { withContext(Dispatchers.IO) { - val accountsToken = exchangeAccountToken(login.source.get()).accessToken + val accountsToken = login.source.withToken { exchangeAccountToken(it) }.accessToken val account = fetchLinkedAccounts(accountsToken).firstOrNull { it.type == type } ?: return@withContext null account to (fetchLinkedTokens(accountsToken, account, login.deviceId) ?: return@withContext null) } diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneSharing.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneSharing.kt index e7cba4a7..14b4daab 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneSharing.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneSharing.kt @@ -15,6 +15,7 @@ import com.macedonga.visor.core.session.DataXMessage import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.core.session.Responder import com.macedonga.visor.log.EventLog +import com.macedonga.visor.settings.hasDisplay import dagger.hilt.android.qualifiers.ApplicationContext import java.io.File import java.util.concurrent.ConcurrentHashMap @@ -53,10 +54,12 @@ class PhoneSharing @Inject constructor( private val files = ConcurrentHashMap>() fun start(scope: CoroutineScope) { - airship.extras.add(0, this) scope.launch { connection.get().state.map { it.link == LinkState.READY }.distinctUntilChanged().collect { ready -> - if (ready) connection.get().submit { it.sendOnService(SVC, MT_ENUMERATE_REQUEST, ByteArray(0)) } + // Only the Display's sharing is used; other glasses keep their own. + if (!ready || !hasDisplay(connection.get().state.value.info?.build)) return@collect + if (this@PhoneSharing !in airship.extras) airship.extras.add(0, this@PhoneSharing) + connection.get().submit { it.sendOnService(SVC, MT_ENUMERATE_REQUEST, ByteArray(0)) } } } } diff --git a/app/src/main/kotlin/com/macedonga/visor/firmware/FirmwareUpdate.kt b/app/src/main/kotlin/com/macedonga/visor/firmware/FirmwareUpdate.kt index 51a6fdaf..d3496509 100644 --- a/app/src/main/kotlin/com/macedonga/visor/firmware/FirmwareUpdate.kt +++ b/app/src/main/kotlin/com/macedonga/visor/firmware/FirmwareUpdate.kt @@ -1,5 +1,6 @@ package com.macedonga.visor.firmware +import com.macedonga.visor.core.identity.token.withToken import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.identity.firmware.FirmwareOffer import com.macedonga.visor.core.identity.firmware.fetchFirmwareOffer @@ -84,7 +85,7 @@ class FirmwareUpdate @Inject constructor( val type = parts.getOrNull(4)?.substringAfter(':', "user").orEmpty() val product = parts.getOrNull(1).orEmpty() if (firmware.isEmpty() || product.isEmpty()) error("the glasses' build is unknown") - val found = fetchFirmwareOffer(login.source.get(), info.serial, "ota.$product.$type", firmware) + val found = login.source.withToken { fetchFirmwareOffer(it, info.serial, "ota.$product.$type", firmware) } _offer.value = found _problem.value = "" _checked.value = true diff --git a/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt b/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt index 73b094d7..b0e89360 100644 --- a/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt +++ b/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt @@ -1,5 +1,6 @@ package com.macedonga.visor.language +import com.macedonga.visor.core.identity.token.withToken import android.content.Context import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.identity.language.LANGUAGE_FEATURE @@ -172,8 +173,7 @@ class LanguageInstaller @Inject constructor( val firmware = parts.getOrNull(4)?.substringBefore(':').orEmpty() val type = parts.getOrNull(4)?.substringAfter(':', "user").orEmpty() if (firmware.isEmpty()) error("the glasses' build is unknown") - val token = login.source.get() - return fetchPackOffers(token, info.serial, firmware, features, "ota.${parts.getOrElse(1) { "hammerhead" }}.$type") + return login.source.withToken { token -> fetchPackOffers(token, info.serial, firmware, features, "ota.${parts.getOrElse(1) { "hammerhead" }}.$type") } } private companion object { diff --git a/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt b/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt index 89188509..492d22ea 100644 --- a/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt +++ b/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt @@ -13,7 +13,9 @@ import com.macedonga.visor.core.identity.bloks.BloksHost import com.macedonga.visor.core.identity.bloks.BloksSession import com.macedonga.visor.core.identity.pyUrlencode import com.macedonga.visor.core.identity.token.AuthorizeRequest +import com.macedonga.visor.core.identity.token.SessionEnded import com.macedonga.visor.core.identity.token.Token +import com.macedonga.visor.core.identity.token.withToken import com.macedonga.visor.core.identity.token.WebLoginSource import com.macedonga.visor.log.EventLog import dagger.hilt.android.qualifiers.ApplicationContext @@ -80,12 +82,12 @@ class MetaLogin @Inject constructor( "method" to "post", "locale" to "en_US", "pretty" to "false", "format" to "json", ), ).toByteArray() - val token = source.get() - val answer = post(token, body) - // OAuthException 190: the session behind the token has ended. - if (runCatching { JSONObject(answer).optJSONObject("error")?.optInt("code") }.getOrNull() != 190) return answer - source.forget(token) - return post(source.get(), body) + return source.withToken { token -> + post(token, body).also { answer -> + // OAuthException 190: the session behind the token has ended. + if (runCatching { JSONObject(answer).optJSONObject("error")?.optInt("code") }.getOrNull() == 190) throw SessionEnded() + } + } } private fun post(token: Token, body: ByteArray): String = diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt b/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt index cfd932fa..61ba64d6 100644 --- a/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt +++ b/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt @@ -30,6 +30,7 @@ import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.protocol.Wire import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.core.settings.GlassesSettings +import com.macedonga.visor.settings.hasDisplay import dagger.hilt.android.AndroidEntryPoint import java.io.ByteArrayOutputStream import javax.inject.Inject @@ -186,6 +187,7 @@ class MirrorListener : NotificationListenerService(), Messaging { synchronized(held) { held.remove(sbn.key) } textReplies.remove(sbn.key) val (provider, uuid) = mirrored.remove(sbn.key) ?: return + if (!display()) return val request = Wire.fieldBytes(1, sbn.key.toByteArray()) + Wire.fieldVarint(2, provider.toLong()) + Wire.fieldBytes(6, uuid.toByteArray()) connection.submit { it.sendOnService(GlassesNotification.SVC, GlassesNotification.MT_REMOVE, request) } } @@ -329,6 +331,9 @@ class MirrorListener : NotificationListenerService(), Messaging { private fun ready() = connection.state.value.link == LinkState.READY + // The Display mirrors the phone's texts and removals too; other glasses get them over Bluetooth already. + private fun display() = hasDisplay(connection.state.value.info?.build) + // The Meta app's filters: nothing ongoing, no group summaries, nothing empty or below default // importance, and not the dialer, whose calls reach the glasses by Bluetooth already. private fun wanted(sbn: StatusBarNotification, ranking: RankingMap): Boolean { @@ -340,6 +345,7 @@ class MirrorListener : NotificationListenerService(), Messaging { val r = Ranking() if (ranking.getRanking(sbn.key, r) && r.importance < NotificationManager.IMPORTANCE_DEFAULT) return false if (sbn.packageName == getSystemService(TelecomManager::class.java)?.defaultDialerPackage) return false + if (sbn.packageName == Telephony.Sms.getDefaultSmsPackage(this) && !display()) return false return true } diff --git a/app/src/main/kotlin/com/macedonga/visor/stage/Frames.kt b/app/src/main/kotlin/com/macedonga/visor/stage/Frames.kt index 96406734..8522bf16 100644 --- a/app/src/main/kotlin/com/macedonga/visor/stage/Frames.kt +++ b/app/src/main/kotlin/com/macedonga/visor/stage/Frames.kt @@ -1,5 +1,6 @@ package com.macedonga.visor.stage +import com.macedonga.visor.core.identity.token.withToken import android.content.Context import android.graphics.BitmapFactory import androidx.compose.ui.graphics.ImageBitmap @@ -132,7 +133,7 @@ class Looks @Inject constructor( picturing.withLock { val missing = serials.filter { it !in _pictures.value } if (missing.isEmpty() || !login.source.hasCachedLogin) return@withLock - val where = runCatching { fetchPairPictures(login.source.get(), missing) } + val where = runCatching { login.source.withToken { fetchPairPictures(it, missing) } } .onFailure { log.add("Pictures: ${it.message}") }.getOrNull() ?: return@withLock for ((serial, picture) in where) { // The app is dark: the picture made for a dark background. @@ -197,7 +198,7 @@ class Looks @Inject constructor( if (!force && now - prefs.getLong(NO_RECORD + serial, 0) < RETRY_MS) return null // Only language and country: Android adds the regional preferences to the default locale's name. val locale = Locale.getDefault().let { if (it.country.isEmpty()) "en_US" else "${it.language}_${it.country}" } - return runCatching { fetchGlassesLook(login.source.get(), serial, locale) } + return runCatching { login.source.withToken { fetchGlassesLook(it, serial, locale) } } .onSuccess { look -> if (look == null) { log.add("Look: Meta has no record of $serial") diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt index 9ab22952..bc88e9fc 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt @@ -182,7 +182,7 @@ class Assistant( } override fun note(text: String) = say(text) - }) + }, display) // The glasses' last language, kept so the speech model can be fetched at launch, before they connect. private val speechPrefs = context.getSharedPreferences("speech", Context.MODE_PRIVATE) diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt index 0e7eb4c1..f90e08bf 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt @@ -37,7 +37,12 @@ enum class LiveAiPhase { OFF, STARTING, LISTENING, THINKING } * the microphone on s0 with their own speech recognition unloaded. The server acknowledges the interaction, * listens, and answers each utterance as a sub-interaction of it, with the camera's picture. */ -internal class LiveAi(private val context: Context, private val scope: CoroutineScope, private val host: LiveAiHost) { +internal class LiveAi( + private val context: Context, + private val scope: CoroutineScope, + private val host: LiveAiHost, + private val display: () -> Boolean, +) { private val _phase = MutableStateFlow(LiveAiPhase.OFF) val phase: StateFlow = _phase.asStateFlow() @@ -83,7 +88,9 @@ internal class LiveAi(private val context: Context, private val scope: Coroutine delay(20) host.event(EV_SERVER_CONNECTION_STATUS, ThriftWriter().str(1, id).bool(2, true).bool(3, false).i32(4, 2).done()) delay(20) - host.event(EV_KEEP_AIR_SESSION_ALIVE, ThriftWriter().str(1, "AsrStartListening").i32(2, 20_000).i32(3, nonce.getAndIncrement()).str(4, id).done()) + val keep = ThriftWriter().str(1, "AsrStartListening").i32(2, 20_000).i32(3, nonce.getAndIncrement()) + if (display()) keep.str(4, id) + host.event(EV_KEEP_AIR_SESSION_ALIVE, keep.done()) ears?.stop() ears = GlassesEars(context, locale, continuous = true, onUtterance = ::heard, onProblem = { host.note("Live AI: $it") }).also { it.start() } keepalive?.cancel() @@ -144,7 +151,9 @@ internal class LiveAi(private val context: Context, private val scope: Coroutine while (scope.isActive && active) { delay(KEEPALIVE_MS) host.event(EV_FRAME_KEEP_ALIVE, ThriftWriter().i64(1, System.currentTimeMillis()).i32(2, nonce.getAndIncrement()).done()) - host.event(EV_KEEP_AIR_SESSION_ALIVE, ThriftWriter().str(1, "LiveAiSession").i32(2, 20_000).i32(3, nonce.getAndIncrement()).str(4, interaction).done()) + val keep = ThriftWriter().str(1, "LiveAiSession").i32(2, 20_000).i32(3, nonce.getAndIncrement()) + if (display()) keep.str(4, interaction) + host.event(EV_KEEP_AIR_SESSION_ALIVE, keep.done()) } } diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt index 306cfa49..5c2f4019 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt @@ -88,8 +88,10 @@ class GlassesConnection( private val judged = AtomicBoolean(false) private var deadline: Job? = null - // svc28 state that came before svc100 said which model this is. + // svc28 state that came before svc100 said which model this is. Let through after a while if svc100 never decodes. private val heldStatus = mutableListOf() + private var holdingStatus = true + private var statusTimer: Job? = null /** Choose a target, open a session on it, and keep the state current until [stop]. */ fun start(scope: CoroutineScope, override: GlassesTarget? = null) { @@ -212,7 +214,7 @@ class GlassesConnection( } if (event.serviceId == SVC_STATUS && isDeviceStateType(event.messageType)) { // How it is framed depends on the model, which svc100 names: what comes first waits for it. - if (_state.value.info == null && heldStatus.size < HELD_STATUS) heldStatus += event else applyStatus(event) + if (holdingStatus && _state.value.info == null && heldStatus.size < HELD_STATUS) heldStatus += event else applyStatus(event) } // Airship's file chunks and the live stream's frames are their responders' business, // and thousands of them would push everything else out of this buffer. @@ -241,6 +243,9 @@ class GlassesConnection( private fun newAttempt() { heldStatus.clear() + holdingStatus = true + statusTimer?.cancel() + statusTimer = null canaryAnswered = false judged.set(false) deadline?.cancel() @@ -262,6 +267,13 @@ class GlassesConnection( statusChannel = subscribeOn(connection, null) syncClock(connection) } + statusTimer = scope.launch { + delay(STATUS_HOLD_MILLIS) + if (canaryAnswered) return@launch + holdingStatus = false + heldStatus.forEach(::applyStatus) + heldStatus.clear() + } if (canaryAnswered) { judge(host, bundle) return @@ -312,6 +324,9 @@ class GlassesConnection( /** 20 seconds, from `__main__.py`'s `_DEFAULT_TIMEOUT_SECONDS`. */ const val CANARY_TIMEOUT_MILLIS: Long = 20_000 + /** How long svc28 waits for svc100 before it is applied anyway. */ + private const val STATUS_HOLD_MILLIS: Long = 5_000 + private const val SVC_AIRSHIP = 20 private const val SVC_MEDIA_STREAMING_CONTROL = 51350 private const val HELD_STATUS = 64 diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/GlassesLook.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/GlassesLook.kt index 9cd87670..06603e44 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/GlassesLook.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/GlassesLook.kt @@ -1,5 +1,6 @@ package com.macedonga.visor.core.identity.language +import com.macedonga.visor.core.identity.token.checkSession import com.macedonga.visor.core.identity.HttpPost import com.macedonga.visor.core.identity.UrlConnectionPost import com.macedonga.visor.core.identity.pyUrlencode @@ -58,6 +59,7 @@ fun fetchGlassesLook(token: Token, serial: String, locale: String = "en_US", htt } val json = runCatching { Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject } .getOrElse { throw TokenError("Meta answered ${reply.status} with no JSON") } + checkSession(json) json["errors"]?.let { throw TokenError("Meta refused: ${it.toString().take(200)}") } val edges = (json["data"] as? JsonObject)?.get("xar_ar_ent_wearable_device_sku_query_by_serial_number") ?.let { it as? JsonObject }?.get("edges") as? JsonArray ?: return null diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt index 0cdd9542..92a19876 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt @@ -1,5 +1,6 @@ package com.macedonga.visor.core.identity.language +import com.macedonga.visor.core.identity.token.checkSession import com.macedonga.visor.core.identity.HttpPost import com.macedonga.visor.core.identity.UrlConnectionPost import com.macedonga.visor.core.identity.pyUrlencode @@ -126,6 +127,7 @@ internal fun graphPost(token: Token, friendlyName: String, docId: String, variab } val json = runCatching { Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject } .getOrElse { throw TokenError("Meta answered ${reply.status} with no JSON") } + checkSession(json) json["errors"]?.let { throw TokenError("Meta refused: ${it.toString().take(200)}") } return reply.status to json } diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/PairPicture.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/PairPicture.kt index ddeac9f2..1ad6dbbc 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/PairPicture.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/PairPicture.kt @@ -1,5 +1,6 @@ package com.macedonga.visor.core.identity.language +import com.macedonga.visor.core.identity.token.checkSession import com.macedonga.visor.core.identity.HttpPost import com.macedonga.visor.core.identity.UrlConnectionPost import com.macedonga.visor.core.identity.pyUrlencode @@ -49,6 +50,7 @@ fun fetchPairPictures(token: Token, serials: List, http: HttpPost = UrlC } val json = runCatching { Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject } .getOrElse { throw TokenError("Meta answered ${reply.status} with no JSON") } + checkSession(json) json["errors"]?.let { throw TokenError("Meta refused: ${it.toString().take(200)}") } val categories = ((json["data"] as? JsonObject)?.get("xar_wearable_device_assets_by_serial_numbers") as? JsonObject)?.get("data") as? JsonArray ?: return emptyMap() diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/AccountToken.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/AccountToken.kt index 8093513e..a3c3e268 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/AccountToken.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/AccountToken.kt @@ -61,6 +61,7 @@ fun exchangeAccountToken( } catch (error: Exception) { throw TokenError("app_token_exchange: response was not JSON") } + checkSession(payload) payload["error"]?.let { throw TokenError("token exchange refused: ${(it as? JsonPrimitive)?.content ?: it}") } val token = payload.text("access_token") val userId = payload.text("user_id") diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Token.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Token.kt index f09373cb..69110452 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Token.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Token.kt @@ -3,10 +3,11 @@ package com.macedonga.visor.core.identity.token import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.intOrNull /** Where the provisioning token comes from. */ /** [unreachable]: Meta was not reached at all, so the login itself may well be fine. */ -class TokenError(message: String, val unreachable: Boolean = false) : Exception(message) +open class TokenError(message: String, val unreachable: Boolean = false) : Exception(message) class Token( val value: String, @@ -19,6 +20,29 @@ class Token( fun interface TokenSource { fun get(): Token + + /** Meta ended the session behind [dropped]: the next [get] logs in for a new token. */ + fun forget(dropped: Token) {} +} + +/** Meta answered OAuthException 190 to the token: the session behind it has ended. */ +class SessionEnded : TokenError("Meta ended the login session; log in again if this repeats") + +/** Throws [SessionEnded] when a Meta answer carries error 190. */ +fun checkSession(json: JsonObject) { + val code = (json["error"] as? JsonObject)?.get("code") as? JsonPrimitive + if (code?.intOrNull == 190) throw SessionEnded() +} + +/** Runs [block] with the token. When Meta ended its session, forgets the token and runs [block] once more with a fresh one. */ +inline fun TokenSource.withToken(block: (Token) -> T): T { + val token = get() + return try { + block(token) + } catch (error: SessionEnded) { + forget(token) + block(get()) + } } /** The user agent both Meta auth calls send. */ diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt index ff1fe388..76fd6e04 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt @@ -118,7 +118,7 @@ class WebLoginSource( } /** Meta no longer takes [dropped]: the next [get] exchanges the login for a new token. */ - fun forget(dropped: Token) { + override fun forget(dropped: Token) { synchronized(this) { if (token == dropped) token = null } } diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Messages.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Messages.kt index 14a2497e..fbbb4bde 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Messages.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Messages.kt @@ -151,7 +151,8 @@ class EnableEncryption( seed = f.bytes(2), iv = f.bytes(3), base = f.varint(4), - parameters = f.varint(5).toInt(), + // Left out, the glasses mean 31: Multikey on. + parameters = if (f.has(5)) f.varint(5).toInt() else 31, quirks = f.varint(6).toInt(), phasedLinkSetupSupported = f.varint(7).toInt(), supportedLinkSetupServices = f.varint(8), From a8e5cf88a8d060fd171af965ae68ff5835620c20 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 19:45:47 +0200 Subject: [PATCH 73/90] Display: stay on Bluetooth when the Display asks for a Wi-Fi link. Granting the lease moved the whole session to Wi-Fi Direct, and the app then took the glasses for disconnected: "out of reach" while Meta AI still answered, no notifications, Mirror and the Display's pages unavailable, and the session stayed on Wi-Fi after the lease ended. A Wi-Fi link lease is refused again (WIFI_DIRECT_NOT_AVAILABLE). This reverts af0b76c. --- .../visor/connection/InternetSwitch.kt | 11 -- .../visor/connection/LeaseService.kt | 34 ++---- .../visor/connection/PhoneInternet.kt | 9 -- .../macedonga/visor/connection/WifiLease.kt | 104 ------------------ 4 files changed, 10 insertions(+), 148 deletions(-) delete mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/WifiLease.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt index 332a166a..a3ee79e5 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt @@ -17,8 +17,6 @@ import java.util.concurrent.atomic.AtomicBoolean import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.withTimeoutOrNull /** * svc41108, internet_switch: how the phone's internet is (type 3): connected, metered, and its DNS servers. The Display @@ -75,15 +73,6 @@ class InternetSwitch @Inject constructor( glasses.submit { it.sendOnService(SVC, MT_COMPANION_STATE, state) } } - /** The state on [mux] itself, for a session other than Bluetooth's (Wi-Fi during a link lease). */ - suspend fun pushOn(mux: DataXConnection) = mux.sendOnService(SVC, MT_COMPANION_STATE, encode(phoneInternet())) - - /** After a move back to Bluetooth: its session is new, so the state goes on it once the link is up again. */ - suspend fun pushWhenReady() { - withTimeoutOrNull(60_000) { connection.get().state.first { it.link == LinkState.READY } } - push() - } - private data class PhoneInternet(val connected: Boolean, val metered: Boolean, val dns: List) private fun phoneInternet(): PhoneInternet { diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt index d2bd7938..93d37d3b 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt @@ -13,13 +13,11 @@ import javax.inject.Singleton /** * svc41107, lease_service: the glasses ask for internet (type 1, InternetLeaseRequest) and for a link (type 4, * LinkLeaseRequest), and end a lease (type 3, no answer). Internet leases are answered as the Meta app answers them - * (2, use the glasses' own Wi-Fi); a link lease (5) on Bluetooth is granted, one that needs Wi-Fi moves the phone to - * Wi-Fi Direct ([WifiLease]) when it can, and is refused otherwise. + * (2, use the glasses' own Wi-Fi); a link lease (5) on Bluetooth is granted, one that needs Wi-Fi is refused for now. */ @Singleton class LeaseService @Inject constructor( private val log: Provider, - private val wifiLease: Provider, ) : Responder { // Leases the glasses hold, by lease id, so an expiry can be named in the log. private val held = ConcurrentHashMap() @@ -49,34 +47,22 @@ class LeaseService @Inject constructor( request.has(6) -> request.varint(6) >= 1 else -> null } - // Checked before anything is answered or moved: a Wi-Fi link the phone cannot take is refused. - val unavailable = if (wifi == true) wifiLease.get().refusal() else null - val granted = wifi == false || (wifi == true && unavailable == null) - val reply = when { - wifi == null -> Wire.fieldBytes(1, id) + refusal(ERROR_BAD_REQUEST, "Invalid Request") - granted -> Wire.fieldBytes(1, id) - else -> Wire.fieldBytes(1, id) + refusal(ERROR_WIFI_DIRECT_NOT_AVAILABLE, "Wi-Fi is not available for this link.") + val reply = when (wifi) { + null -> Wire.fieldBytes(1, id) + refusal(ERROR_BAD_REQUEST, "Invalid Request") + true -> Wire.fieldBytes(1, id) + refusal(ERROR_WIFI_DIRECT_NOT_AVAILABLE, "Wi-Fi is not available for this link.") + false -> Wire.fieldBytes(1, id) } log.get().add( - when { - wifi == null -> "Lease: the glasses asked for a link with no requirement; refused" - wifi == false -> "Lease: the glasses asked for a link on Bluetooth; answered OK" - unavailable != null -> "Lease: the glasses asked for Wi-Fi; refused, $unavailable" - else -> "Lease: the glasses asked for Wi-Fi; answered OK" + when (wifi) { + null -> "Lease: the glasses asked for a link with no requirement; refused" + true -> "Lease: the glasses asked for Wi-Fi; not available yet, refused" + false -> "Lease: the glasses asked for a link on Bluetooth; answered OK" }, ) if (wifi == false) held[key] = "link" - if (wifi == true && granted) { - held[key] = "Wi-Fi link" - wifiLease.get().add(key) - } connection.send(channel, reply, messageType = MT_LINK_RESPONSE) - if (wifi == true && granted) wifiLease.get().hold() - } - MT_LEASE_EXPIRATION -> { - held.remove(key)?.let { log.get().add("Lease: a $it lease ended") } - wifiLease.get().remove(key) } + MT_LEASE_EXPIRATION -> held.remove(key)?.let { log.get().add("Lease: a $it lease ended") } else -> log.get().add("Lease: message type ${message.info.messageType} is not handled") } } diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt index 0f99b087..2065f15a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt @@ -15,7 +15,6 @@ import java.nio.ByteBuffer import java.nio.ByteOrder import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.atomic.AtomicLong import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @@ -59,17 +58,10 @@ class PhoneInternet @Inject constructor( private val tunnelSlots = Semaphore(TUNNEL_FRAMES) // The connection the tunnels run on; a new one ends the old tunnels. private var owner: DataXConnection? = null - // When the tunnel last carried anything either way; a move to Wi-Fi for a link ends once it has been idle a while. - private val lastTraffic = AtomicLong(System.currentTimeMillis()) - - fun idleMillis(): Long = System.currentTimeMillis() - lastTraffic.get() - - fun touch() = lastTraffic.set(System.currentTimeMillis()) override suspend fun respond(message: DataXMessage, connection: DataXConnection) { val info = message.info val channel = info.channelId - touch() if (connection !== owner) { // A tunnel of an earlier connection would write on this one under the old keys, and keep retransmitting. owner = connection @@ -132,7 +124,6 @@ class PhoneInternet @Inject constructor( /** An IP packet to the Display, framed for its tunnel. */ suspend fun deliver(version: Int, src: ByteArray, dst: ByteArray, protocol: Int, transport: ByteArray) { - touch() val ip = IpPacket.build(version, src, dst, protocol, transport, ids.incrementAndGet()) val frame = ByteBuffer.allocate(ICEE_HEADER + ip.size).order(ByteOrder.LITTLE_ENDIAN) .putShort(ICEE_MAGIC.toShort()).putShort(ip.size.toShort()) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/WifiLease.kt b/app/src/main/kotlin/com/macedonga/visor/connection/WifiLease.kt deleted file mode 100644 index 2660e6eb..00000000 --- a/app/src/main/kotlin/com/macedonga/visor/connection/WifiLease.kt +++ /dev/null @@ -1,104 +0,0 @@ -package com.macedonga.visor.connection - -import android.content.Context -import com.macedonga.visor.core.media.LiveStream -import com.macedonga.visor.core.media.wifiDirectAllowed -import com.macedonga.visor.core.media.wifiIsOn -import com.macedonga.visor.core.session.DataXConnection -import com.macedonga.visor.log.EventLog -import dagger.hilt.android.qualifiers.ApplicationContext -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicBoolean -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.delay -import kotlinx.coroutines.launch - -/** - * The Wi-Fi link the Display asks for on svc41107 (a link lease that needs Wi-Fi). While one is held the phone moves - * to a Wi-Fi session with the glasses, as [LiveStream.overWifi] does for media, and the internet tunnel (svc41111) - * runs there with svc41108's state. The move ends when no lease is left and the tunnel has been idle for two minutes. - */ -@Singleton -class WifiLease @Inject constructor( - @ApplicationContext private val context: Context, - private val live: Provider, - private val internet: Provider, - private val tunnel: Provider, - private val log: Provider, -) { - private val leases = ConcurrentHashMap.newKeySet() - private val moving = AtomicBoolean(false) - private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - - fun add(id: String) { - leases.add(id) - } - - fun remove(id: String) { - leases.remove(id) - } - - /** Why the phone cannot take a Wi-Fi link now, or null. While a move runs, another lease simply joins it. */ - fun refusal(): String? { - if (moving.get()) return null - val glasses = live.get() - return when { - glasses.running.value || glasses.wifiSession.value != null -> "the live view is using Wi-Fi" - !wifiIsOn(context) -> "Wi-Fi is off" - !wifiDirectAllowed(context) -> "Wi-Fi Direct is not allowed" - else -> null - } - } - - /** Moves to Wi-Fi for the leases held, unless a move is running already. */ - fun hold() { - if (!moving.compareAndSet(false, true)) return - scope.launch { - try { - log.get().add("Wi-Fi: moving to Wi-Fi Direct, because the Display asked for a link that needs Wi-Fi") - val failed = live.get().overWifi { mux -> ride(mux) } - if (failed == null) { - log.get().add("Wi-Fi: back on Bluetooth") - internet.get().pushWhenReady() - } else { - leases.clear() - log.get().add("Wi-Fi: the move did not start: $failed") - } - } finally { - moving.set(false) - } - } - } - - /** Keeps the Wi-Fi session up until no lease is left and the tunnel is idle, or the Live view takes Wi-Fi. */ - private suspend fun ride(mux: DataXConnection): String? { - internet.get().pushOn(mux) - tunnel.get().touch() - while (true) { - delay(1_000) - val glasses = live.get() - if (glasses.running.value) { - log.get().add("Wi-Fi: leaving Wi-Fi, because the live view started") - break - } - if (glasses.wifiSession.value == null) { - log.get().add("Wi-Fi: leaving Wi-Fi, because the Wi-Fi link dropped") - break - } - if (leases.isEmpty() && tunnel.get().idleMillis() >= IDLE_MS) { - log.get().add("Wi-Fi: leaving Wi-Fi, because no link lease is held and the tunnel has been idle for 2 min") - break - } - } - return null - } - - companion object { - private const val IDLE_MS = 120_000L - } -} From ca32c40cf286e057ab42db8769be61dc40697b5a Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 19:45:47 +0200 Subject: [PATCH 74/90] Neural Band: show the band the Display reports. A band set up on the Display but not in Visor showed "Not set up" while its battery was known. The band's summary and page now show the battery and fit the Display reports; Set up stays for Visor's own link to the band (wrist, buzz, firmware). --- .../main/kotlin/com/macedonga/visor/settings/BandPart.kt | 7 ++++++- .../kotlin/com/macedonga/visor/settings/SettingsScreen.kt | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt index 155af1b8..0d5d91bd 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt @@ -47,7 +47,12 @@ internal fun BandPart(model: SettingsViewModel) { problem = problem, ) if (all.none { it.glasses == pair }) { - Rows({ SetUpRow(R.string.band_set_up) }) + // A band the Display already holds reports through it; Visor's own link to the band needs the setup. + Rows( + battery?.let { b -> row { ValueRow(stringResource(R.string.band_battery), batteryWords(b, charging), onClick = null) } }, + fitWords(device?.bandFit)?.let { fit -> row { ValueRow(stringResource(R.string.band_fit), fit, onClick = null) } }, + { SetUpRow(R.string.band_set_up) }, + ) return } Rows( diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt index 6eb7356a..103a713a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt @@ -569,8 +569,12 @@ private fun summary(topic: Topic, model: SettingsViewModel, state: SettingsState Topic.BAND -> { val band by model.band.state.collectAsStateWithLifecycle() val pair by model.connection.pair.collectAsStateWithLifecycle() - if (model.bands.forGlasses(pair) == null) stringResource(R.string.band_none) - else listOfNotNull(band.battery?.let { batteryWords(it, band.charging) }, band.leftHanded?.let { wristWords(it) }).joinToString(" · ") + val glasses by model.connection.state.collectAsStateWithLifecycle() + // The Display reports the band it holds, set up in Visor or not. + val battery = glasses.device?.bandBattery ?: band.battery + val charging = glasses.device?.bandCharging ?: band.charging + if (model.bands.forGlasses(pair) == null && battery == null) stringResource(R.string.band_none) + else listOfNotNull(battery?.let { batteryWords(it, charging) }, band.leftHanded?.let { wristWords(it) }).joinToString(" · ") } Topic.SOUND -> listOfNotNull( shown("notification_volume")?.let { stringResource(R.string.sum_sound_volume, it) }, From 7b2b385e907e8091abb2b1c67c1c52909dcb58bd Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 20:01:06 +0200 Subject: [PATCH 75/90] Meta AI: answer typed questions in the app again. With a Display connected they went to the glasses as a remote turn (ev337 then ev52), but the Display skips StartRemoteInteraction when no assistant session runs, which is whenever the wearer has not just spoken to it (seen in its log), so the turn never started there. This reverts bf30bb0. --- .../com/macedonga/visor/metaai/MetaAiModel.kt | 4 +- .../visor/core/assistant/Assistant.kt | 37 +------------------ .../macedonga/visor/core/assistant/Oxide.kt | 1 - 3 files changed, 3 insertions(+), 39 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/metaai/MetaAiModel.kt b/app/src/main/kotlin/com/macedonga/visor/metaai/MetaAiModel.kt index 3b38e2fe..3aeab54d 100644 --- a/app/src/main/kotlin/com/macedonga/visor/metaai/MetaAiModel.kt +++ b/app/src/main/kotlin/com/macedonga/visor/metaai/MetaAiModel.kt @@ -59,9 +59,7 @@ class MetaAiModel @Inject constructor( val forgotten: StateFlow = _forgotten.asStateFlow() private var forgetting: Job? = null - /** On a Display the question runs as a turn on the glasses; otherwise, or when they cannot take it, it is answered here. */ - fun ask(question: String, conversation: String?): String = - assistant.askOnGlasses(question, conversation) ?: assistant.askFromPhone(question, conversation) + fun ask(question: String, conversation: String?) = assistant.askFromPhone(question, conversation) fun askAgain(id: String) = assistant.askAgain(id) diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt index bc88e9fc..1dc3e89b 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt @@ -838,8 +838,7 @@ class Assistant( private suspend fun answer(l: Link, interaction: String, transcript: String, visual: Visual, glasses: GlassesToolbox? = null) { val started = System.currentTimeMillis() - // A typed turn is in the diary already, with its source and conversation. - if (diary.turn(interaction) == null) record(TurnRecord(interaction, started, transcript)) + record(TurnRecord(interaction, started, transcript)) _phase.value = AssistantPhase.THINKING var problem = "" // During the Live every question comes with the picture: the wearer is showing it. @@ -1010,15 +1009,11 @@ class Assistant( return packets.sumOf { it.durationUs } / 1000 } - /** ev52 FinalTranscription: the words of a turn. */ - private fun finalTranscription(interaction: String, words: String): ByteArray = - ThriftWriter().str(1, words).i32(2, 0).str(4, "").bool(7, false).str(8, interaction).done() - /** ev71, ev52 and ev342, which open every answer. ev52 and ev342 go at a tagged type 0, as the app sends them. */ private suspend fun preamble(l: Link, interaction: String, transcript: String, text: String) { val m0 = checkNotNull(l.m0) val e71 = ThriftWriter().str(1, interaction).done() - val e52 = finalTranscription(interaction, transcript.ifEmpty { text }) + val e52 = ThriftWriter().str(1, transcript.ifEmpty { text }).i32(2, 0).str(4, "").bool(7, false).str(8, interaction).done() val e342 = ThriftWriter().str(1, interaction).bool(2, false).str(3, REPLY_INTENT).bool(4, false).done() for ((type, event, body) in listOf( Triple(MT_NET_EVENT, EV_START_OF_USER_SPEECH, e71), @@ -1141,34 +1136,6 @@ class Assistant( return id } - /** - * A question typed in the app, run as a turn on the glasses when they are a Display: 337 starts the turn, 52 gives - * its words, and the answer goes as for any turn. Null when the glasses cannot take it, so the app answers it. - */ - fun askOnGlasses(question: String, conversation: String?): String? { - val text = question.trim() - val l = link?.takeIf { display() && it.m0 != null && it.s0 != null } ?: return null - if (text.isEmpty()) return null - val id = UUID.randomUUID().toString() - // Taken before the glasses could open a microphone for the turn, so the phone does not answer its own recognition. - l.taken.add(id) - record(TurnRecord(id, System.currentTimeMillis(), text, source = TurnSource.PHONE, conversation = conversation.orEmpty())) - scope.launch { - try { - val m0 = checkNotNull(l.m0) - l.mux.send(m0, netEvent(EV_START_REMOTE_INTERACTION, ThriftWriter().str(1, id).done()), messageType = MT_NET_EVENT) - delay(20) - l.mux.send(m0, netEvent(EV_FINAL_TRANSCRIPTION, finalTranscription(id, text)), messageType = 0) - } catch (error: Exception) { - say("The typed question did not reach the glasses: ${error.message}") - update(id) { it.copy(problem = error.message.orEmpty()) } - return@launch - } - answer(l, id, text, Visual(null, "", "", false, "")) - } - return id - } - private fun liveAiNote(): String = if (liveAiDetailed()) LIVE_AI_NOTE_DETAILED else LIVE_AI_NOTE /** [id]'s question again, with the model and settings as they are now; the new answer replaces the old one. */ diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt index 7885c8bf..dead841d 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt @@ -35,7 +35,6 @@ internal const val MT_ENVELOPE_ACK = 9001 // table beside the names; all 206 are in events.tsv). Several are not what lensbreak called them, // but every send below is byte for byte the one the glasses accepted from it. internal const val EV_FINAL_TRANSCRIPTION = 52 -internal const val EV_START_REMOTE_INTERACTION = 337 internal const val EV_VOICE_ACTION_RESPONSE = 53 internal const val EV_FULFILLMENT_REQUEST = 57 internal const val EV_ACTION_FULFILLMENT_REQUEST = 55 From af076982d347a3ef95c9c23cceab0752fb1eb094 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 20:01:06 +0200 Subject: [PATCH 76/90] Display: place search from OpenStreetMap's Nominatim. The phone's geocoder found nothing for the Maps app's searches ("cafe", "park", "attraction" all came back with 0 places). Searches now go to Nominatim within the asked radius around the phone, nearest first, and farther when nothing is that close; it reads a kind of place as one and also finds names and addresses. Results carry the address, the kind of place, the phone number when known, and the walking distance. As the Meta app does, a prewarm is answered at once with no places and its search fills a five-minute cache; requests are spaced a second apart, as Nominatim asks. A search without a known position takes a fresh fix first. --- .../visor/connection/PhonePosition.kt | 21 ++ .../macedonga/visor/connection/PlaceSearch.kt | 208 ++++++++++++++---- 2 files changed, 184 insertions(+), 45 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhonePosition.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhonePosition.kt index d9f5d526..714ad492 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhonePosition.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhonePosition.kt @@ -7,10 +7,15 @@ import android.content.pm.PackageManager import android.location.Location import android.location.LocationListener import android.location.LocationManager +import android.os.Build +import android.os.CancellationSignal import android.os.Looper import dagger.hilt.android.qualifiers.ApplicationContext import javax.inject.Inject import javax.inject.Singleton +import kotlin.coroutines.resume +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withTimeoutOrNull /** * The phone's position for navigation and place search, from the provider [LocationShare] uses. Updates run only @@ -47,6 +52,21 @@ class PhonePosition @Inject constructor(@ApplicationContext private val context: return (fix ?: manager.getLastKnownLocation(provider(manager)))?.let { LatLng(it.latitude, it.longitude) } } + /** A fix taken now, for when none is known yet; null without one in [timeoutMs]. */ + @SuppressLint("MissingPermission") + suspend fun fresh(timeoutMs: Long = FRESH_MS): LatLng? { + if (manager == null || !granted() || Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return null + return withTimeoutOrNull(timeoutMs) { + suspendCancellableCoroutine { cont -> + val cancel = CancellationSignal() + cont.invokeOnCancellation { cancel.cancel() } + manager.getCurrentLocation(provider(manager), cancel, context.mainExecutor) { fix -> + cont.resume(fix?.let { LatLng(it.latitude, it.longitude) }) + } + } + } + } + private fun granted(): Boolean = listOf(Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_COARSE_LOCATION) .any { context.checkSelfPermission(it) == PackageManager.PERMISSION_GRANTED } @@ -59,5 +79,6 @@ class PhonePosition @Inject constructor(@ApplicationContext private val context: private companion object { const val INTERVAL_MS = 1_000L + const val FRESH_MS = 5_000L } } diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PlaceSearch.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PlaceSearch.kt index a7fbe50b..ec8cfea1 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PlaceSearch.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PlaceSearch.kt @@ -1,15 +1,15 @@ package com.macedonga.visor.connection -import android.content.Context -import android.location.Address -import android.location.Geocoder import com.macedonga.visor.core.protocol.Wire import com.macedonga.visor.core.session.DataXConnection import com.macedonga.visor.core.session.DataXMessage import com.macedonga.visor.core.session.Responder import com.macedonga.visor.log.EventLog -import dagger.hilt.android.qualifiers.ApplicationContext import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import java.net.URLEncoder +import java.util.Locale import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @@ -19,22 +19,49 @@ import kotlinx.coroutines.CoroutineExceptionHandler import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext +import org.json.JSONArray +import org.json.JSONObject /** * svc55072, smartglasses_place_search: the glasses search for a place (PlaceSearchRequest, types 0 and 1 {1 query, - * 2 radius in metres, 4 requestId}) and get PlaceSearchResponse (type 2) with the places nearest the phone, from the - * phone's own geocoder. Media requests (3) get an empty PlaceMediaResponse (4). + * 2 radius in metres, 3 search type, 4 requestId, 5 prewarm}) and get PlaceSearchResponse (type 2) with places from + * OpenStreetMap's Nominatim within the radius around the phone, nearest first. Nominatim reads a kind of place + * ("cafe", "park") as one, and also finds names and addresses. As the Meta app does, a prewarm is answered at once + * with no places, and its search fills a five-minute cache for the real request. Media requests (3) get an empty + * PlaceMediaResponse (4). */ @Singleton class PlaceSearch @Inject constructor( - @ApplicationContext private val context: Context, private val position: PhonePosition, private val requests: RequestLog, private val log: Provider, ) : Responder { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Place search: failed: $e") }) + // Nominatim allows one request a second, and the glasses send their prewarms together. + private val nominatim = Mutex() + private var lastAsked = 0L + private val cache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry) = size > CACHE_SIZE + } + + private class Cached(val at: Long, val places: List) + + private class Place( + val id: String, + val name: String, + val at: LatLng, + val street: String, + val city: String, + val postcode: String, + val country: String, + val category: String, + val phone: String, + ) override suspend fun respond(message: DataXMessage, connection: DataXConnection) { if (message.info.flags.isClosed) return @@ -42,63 +69,138 @@ class PlaceSearch @Inject constructor( when (message.info.messageType) { MT_SEARCH, MT_SEARCH_REQUEST -> { val req = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return - scope.launch { connection.send(reply, search(req), messageType = MT_SEARCH_RESPONSE) } + val query = req.bytes(1).decodeToString().trim() + val radius = req.float(2).takeIf { it > 0f }?.toDouble() ?: DEFAULT_RADIUS_M + if (req.varint(5) != 0L) { + connection.send(reply, Wire.fieldVarint(1, RESULT_SUCCESS), messageType = MT_SEARCH_RESPONSE) + scope.launch { places(query, radius, here()) } + return + } + scope.launch { connection.send(reply, answer(query, radius), messageType = MT_SEARCH_RESPONSE) } } MT_MEDIA_REQUEST -> connection.send(reply, ByteArray(0), messageType = MT_MEDIA_RESPONSE) } } - private suspend fun search(req: Wire.Fields): ByteArray = withContext(Dispatchers.IO) { - val query = req.bytes(1).decodeToString() - val radius = req.float(2).takeIf { it > 0f }?.toDouble() ?: DEFAULT_RADIUS_M + private suspend fun here(): LatLng? = position.current() ?: position.fresh() + + private suspend fun answer(query: String, radius: Double): ByteArray { val logId = requests.start("place://$query", SOURCE_MAPS) - val here = position.current() - if (!Geocoder.isPresent()) { - requests.finish(logId, "no geocoder on the phone", failed = true) - return@withContext ERROR_NO_GEOCODER - } + val here = here() val found = try { - lookup(query, here, radius) + places(query, radius, here) } catch (e: IOException) { requests.finish(logId, e.message ?: "search failed", failed = true) - return@withContext ERROR_NETWORK - } catch (e: IllegalArgumentException) { + log.get().add("Place search: \"$query\" failed: ${e.message}") + return error(e.message ?: "Search failed", ERROR_NETWORK) + } catch (e: Exception) { requests.finish(logId, e.message ?: "search failed", failed = true) - return@withContext ERROR_NETWORK + log.get().add("Place search: \"$query\" failed: $e") + return error(e.message ?: "Search failed", ERROR_INTERNAL) } requests.finish(logId, "${found.size} places", failed = false) - log.get().add("Place search: \"$query\", ${found.size} places") + log.get().add("Place search: \"$query\", ${found.size} places" + if (here == null) ", without the phone's position" else "") var body = Wire.fieldVarint(1, RESULT_SUCCESS) - for (address in found) body += Wire.fieldBytes(2, place(address, here)) - body + for (p in found) body += Wire.fieldBytes(2, encode(p, here)) + return body } - /** Bounded to [radius] around the phone when it has a fix. */ - @Suppress("DEPRECATION") - private fun lookup(query: String, here: LatLng?, radius: Double): List
{ - val geocoder = Geocoder(context) - if (here == null) return geocoder.getFromLocationName(query, MAX_RESULTS).orEmpty() - val dLat = radius / METRES_PER_DEGREE - val dLng = radius / (METRES_PER_DEGREE * cos(Math.toRadians(here.lat)).coerceAtLeast(MIN_COS)) - return geocoder.getFromLocationName(query, MAX_RESULTS, here.lat - dLat, here.lng - dLng, here.lat + dLat, here.lng + dLng) - .orEmpty() - .sortedBy { haversine(here, LatLng(it.latitude, it.longitude)) } + /** From the cache when the same search near the same spot ran in the last five minutes. */ + private suspend fun places(query: String, radius: Double, here: LatLng?): List { + if (query.isEmpty()) return emptyList() + val key = query.lowercase(Locale.ROOT) + "@" + (here?.let { "%.3f,%.3f".format(Locale.ROOT, it.lat, it.lng) } ?: "") + val now = System.currentTimeMillis() + synchronized(cache) { cache[key]?.takeIf { now - it.at < CACHE_MS }?.let { return it.places } } + val found = if (here == null) { + search(query, null, bounded = false) + } else { + // A name farther than the radius is still found, the nearest first. + val box = box(here, radius) + search(query, box, bounded = true).ifEmpty { search(query, box, bounded = false) }.sortedBy { haversine(here, it.at) } + } + synchronized(cache) { cache[key] = Cached(now, found) } + return found + } + + /** The viewbox Nominatim takes: west, north, east, south. */ + private fun box(here: LatLng, radius: Double): String { + val r = radius.coerceIn(MIN_RADIUS_M, MAX_RADIUS_M) + val dLat = r / METRES_PER_DEGREE + val dLng = r / (METRES_PER_DEGREE * cos(Math.toRadians(here.lat)).coerceAtLeast(MIN_COS)) + return listOf(here.lng - dLng, here.lat + dLat, here.lng + dLng, here.lat - dLat).joinToString(",") { "%.5f".format(Locale.ROOT, it) } } - /** PlaceSearchResult {1 id, 2 title, 3 Address {1 street}, 4 GeoCoordinate {1 lat, 2 lng}, 13 walking {1 metres}}. */ - private fun place(a: Address, here: LatLng?): ByteArray { - val street = if (a.maxAddressLineIndex >= 0) a.getAddressLine(0).orEmpty() else "" - val title = a.featureName ?: a.thoroughfare ?: street - var b = Wire.fieldBytes(1, "${a.latitude},${a.longitude}".encodeToByteArray()) + - Wire.fieldBytes(2, title.encodeToByteArray()) + - Wire.fieldBytes(3, Wire.fieldBytes(1, street.encodeToByteArray())) + - Wire.fieldBytes(4, Wire.fieldDouble(1, a.latitude) + Wire.fieldDouble(2, a.longitude)) + private suspend fun search(query: String, viewbox: String?, bounded: Boolean): List = nominatim.withLock { + val wait = lastAsked + SPACING_MS - System.currentTimeMillis() + if (wait > 0) delay(wait) + val url = StringBuilder(NOMINATIM) + .append("?format=jsonv2&addressdetails=1&extratags=1&limit=").append(MAX_RESULTS) + .append("&accept-language=").append(Locale.getDefault().toLanguageTag()) + .append("&q=").append(URLEncoder.encode(query, "UTF-8")) + if (viewbox != null) url.append("&viewbox=").append(viewbox).append(if (bounded) "&bounded=1" else "") + try { + val results = JSONArray(fetch(url.toString())) + (0 until results.length()).mapNotNull { place(results.getJSONObject(it)) } + } finally { + lastAsked = System.currentTimeMillis() + } + } + + private fun place(o: JSONObject): Place? { + val a = o.optJSONObject("address") ?: JSONObject() + val extra = o.optJSONObject("extratags") ?: JSONObject() + val street = listOf(a.optString("road"), a.optString("house_number")).filter { it.isNotBlank() }.joinToString(" ") + val name = o.optString("name").ifBlank { street }.ifBlank { o.optString("display_name").substringBefore(',') }.ifBlank { return null } + val lat = o.optString("lat").toDoubleOrNull() ?: return null + val lng = o.optString("lon").toDoubleOrNull() ?: return null + return Place( + id = "osm:${o.optString("osm_type")}/${o.optLong("osm_id")}", + name = name, + at = LatLng(lat, lng), + street = street, + city = CITY_KEYS.firstNotNullOfOrNull { k -> a.optString(k).takeIf { it.isNotBlank() } }.orEmpty(), + postcode = a.optString("postcode"), + country = a.optString("country"), + category = o.optString("type").takeIf { o.optString("category") in KIND_KEYS }?.let(::label).orEmpty(), + phone = extra.optString("phone").ifBlank { extra.optString("contact:phone") }, + ) + } + + private suspend fun fetch(url: String): String = withContext(Dispatchers.IO) { + val conn = URL(url).openConnection() as HttpURLConnection + try { + // Nominatim's policy asks for a user agent that names the app. + conn.setRequestProperty("User-Agent", USER_AGENT) + conn.connectTimeout = TIMEOUT_MS + conn.readTimeout = TIMEOUT_MS + if (conn.responseCode != 200) throw IOException("${conn.url.host} answered ${conn.responseCode}") + conn.inputStream.bufferedReader().use { it.readText() } + } finally { + conn.disconnect() + } + } + + /** PlaceSearchResult {1 id, 2 title, 3 Address, 4 GeoCoordinate, 8 category, 13 walking, 14 phone}. */ + private fun encode(p: Place, here: LatLng?): ByteArray { + val address = listOf(1 to p.street, 2 to p.city, 4 to p.country, 5 to p.postcode) + .filter { it.second.isNotBlank() } + .fold(ByteArray(0)) { b, (field, value) -> b + Wire.fieldBytes(field, value.encodeToByteArray()) } + var b = Wire.fieldBytes(1, p.id.encodeToByteArray()) + + Wire.fieldBytes(2, p.name.encodeToByteArray()) + + Wire.fieldBytes(3, address) + + Wire.fieldBytes(4, Wire.fieldDouble(1, p.at.lat) + Wire.fieldDouble(2, p.at.lng)) + if (p.category.isNotEmpty()) b += Wire.fieldBytes(8, p.category.encodeToByteArray()) if (here != null) { - b += Wire.fieldBytes(13, Wire.fieldVarint(1, haversine(here, LatLng(a.latitude, a.longitude)).roundToLong())) + val metres = haversine(here, p.at) + b += Wire.fieldBytes(13, Wire.fieldVarint(1, metres.roundToLong()) + Wire.fieldVarint(2, (metres / WALK_MPS).roundToLong())) } + if (p.phone.isNotEmpty()) b += Wire.fieldBytes(14, p.phone.encodeToByteArray()) return b } + private fun error(text: String, code: Long): ByteArray = + Wire.fieldVarint(1, RESULT_ERROR) + Wire.fieldBytes(3, text.encodeToByteArray()) + Wire.fieldVarint(4, code) + companion object { const val SVC = 55072 private const val SOURCE_MAPS = -6 @@ -108,11 +210,27 @@ class PlaceSearch @Inject constructor( private const val MT_MEDIA_REQUEST = 3 private const val MT_MEDIA_RESPONSE = 4 private const val RESULT_SUCCESS = 0L - private val ERROR_NO_GEOCODER = Wire.fieldVarint(1, 1) + Wire.fieldBytes(3, "No geocoder on the phone".encodeToByteArray()) - private val ERROR_NETWORK = Wire.fieldVarint(1, 1) + Wire.fieldBytes(3, "Search failed".encodeToByteArray()) + Wire.fieldVarint(4, 1) - private const val DEFAULT_RADIUS_M = 5_000.0 + private const val RESULT_ERROR = 1L + private const val ERROR_INTERNAL = 0L + private const val ERROR_NETWORK = 1L + private const val NOMINATIM = "https://nominatim.openstreetmap.org/search" + private const val USER_AGENT = "Visor (https://github.com/macedonga/visor)" + private const val SPACING_MS = 1_100L + private const val DEFAULT_RADIUS_M = 1_500.0 + private const val MIN_RADIUS_M = 300.0 + private const val MAX_RADIUS_M = 20_000.0 private const val METRES_PER_DEGREE = 111_320.0 private const val MIN_COS = 0.01 private const val MAX_RESULTS = 10 + private const val CACHE_MS = 5 * 60_000L + private const val CACHE_SIZE = 10 + private const val TIMEOUT_MS = 12_000 + private const val WALK_MPS = 1.3 + + // The kinds of place whose type is worth showing as the place's category. + private val KIND_KEYS = setOf("amenity", "shop", "tourism", "leisure", "historic") + private val CITY_KEYS = listOf("city", "town", "village", "suburb") + + private fun label(value: String): String = value.replace('_', ' ').replaceFirstChar { it.uppercase() } } } From 622195a3d705569812c143c2a0744f8c6e210707 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 20:26:35 +0200 Subject: [PATCH 77/90] Display: web apps are the glasses' own list. Visor kept its own list and wrote it over every dev_mode_web_app entry of partner_app_settings, each time the page opened: a web app added with the Meta app was deleted, and none of those were ever listed. The page now lists the glasses' setting as it is (title, page and icon) and changes only the entry it adds, removes or renames, refusing to write a value it cannot read. The "Turn on web apps" row is gone: it read stale MobileConfig values, asked to enable switches that were on, and the glasses need no restart. Developer mode is written only when it is off. --- .../com/macedonga/visor/connection/WebApps.kt | 174 +++++++++--------- .../macedonga/visor/settings/WebAppsScreen.kt | 67 ++----- .../main/res/values-it/strings_webapps.xml | 8 +- app/src/main/res/values/strings_webapps.xml | 8 +- 4 files changed, 104 insertions(+), 153 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt b/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt index 3f700886..555e19ae 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt @@ -1,73 +1,59 @@ package com.macedonga.visor.connection -import android.content.Context import android.net.Uri import android.util.Base64 import android.webkit.CookieManager import com.macedonga.visor.core.connection.GlassesConnection -import com.macedonga.visor.core.media.ConfigState -import com.macedonga.visor.core.media.GlassesConfig import com.macedonga.visor.core.protocol.Wire import com.macedonga.visor.core.session.DataXConnection import com.macedonga.visor.core.session.DataXMessage import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.core.session.Responder import com.macedonga.visor.core.settings.GlassesSettings +import com.macedonga.visor.core.settings.SettingsState import com.macedonga.visor.core.settings.ProtoField import com.macedonga.visor.core.settings.encodeProto import com.macedonga.visor.core.settings.parseProto import com.macedonga.visor.log.EventLog -import dagger.hilt.android.qualifiers.ApplicationContext -import java.io.File import java.util.UUID import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.withTimeoutOrNull -import org.json.JSONArray -import org.json.JSONObject -/** [manifest] is Meta's signed manifest, base64, for a catalogue app; a developer app has none. */ -data class WebApp(val key: String, val name: String, val url: String, val manifest: String? = null) +/** A web app on the glasses: its partner_app_settings key, title, page and icon. */ +data class WebApp(val key: String, val name: String, val url: String, val icon: String = "") -enum class AddResult { ADDED, EXISTS, NOT_HTTPS, FULL } - -/** Whether the glasses open a page from the phone: the two MobileConfig switches the browser checks. */ -enum class OpenGate { UNREAD, UNAVAILABLE, OFF, ON } - -val OPEN_GATES = listOf("web_on_hn.web_app", "web_on_hn.web_app_connections") - -fun gateOf(cfg: ConfigState): OpenGate = when { - cfg.params.isEmpty() -> OpenGate.UNREAD - OPEN_GATES.any { it !in cfg.params } -> OpenGate.UNAVAILABLE - OPEN_GATES.all { cfg.values[it] == true } -> OpenGate.ON - else -> OpenGate.OFF -} +enum class AddResult { ADDED, EXISTS, NOT_HTTPS, FULL, NOT_CONNECTED, UNREADABLE } /** * svc55074, the browser's phone service: the Display's web-app manager (type 15: CREATE 1, DELETE 2, LIST 3, UPDATE 4, * OPEN 5, DEV MODE STATUS 6, answered with 16), cookies (11, answered with 9) and a BLE scan (17, answered with 20 and - * no devices). Visor keeps its own list in a file and writes it to the glasses' partner_app_settings, one - * dev_mode_web_app_ entry each, as the Meta app does. [open] sends OPEN to the glasses. + * no devices). The list is the glasses' own partner_app_settings, one dev_mode_web_app_ entry each, so the web apps + * added with the Meta app are in it; Visor changes only the entry it adds, removes or renames. [open] sends OPEN. */ @Singleton class WebApps @Inject constructor( - @ApplicationContext context: Context, private val connection: Provider, private val settings: Provider, private val log: Provider, ) : Responder { - private val file = File(context.filesDir, "web_apps.json") - private val _apps = MutableStateFlow(load()) - val apps: StateFlow> = _apps.asStateFlow() + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + val apps: StateFlow> by lazy { + settings.get().state.map { appsIn(inForce(it)) }.stateIn(scope, SharingStarted.Eagerly, appsIn(inForce(settings.get().state.value))) + } private val lock = Any() override suspend fun respond(message: DataXMessage, connection: DataXConnection) { @@ -83,45 +69,58 @@ class WebApps @Inject constructor( } } - /** Add a page, from the Display's CREATE or the Add sheet. The glasses take it with [push]. */ + /** Add a page, from the Display's CREATE or the Add sheet, with Meta's [manifest] (base64) for a catalogue app. */ fun add(name: String, url: String, manifest: String? = null): AddResult { val uri = Uri.parse(url) if (uri.scheme != "https" || uri.host.isNullOrEmpty()) return AddResult.NOT_HTTPS synchronized(lock) { - val list = _apps.value + val fields = current() ?: return if (ready()) AddResult.UNREADABLE else AddResult.NOT_CONNECTED + val list = apps(fields) if (list.any { it.url == url }) return AddResult.EXISTS if (list.size >= MAX_APPS) return AddResult.FULL - val key = generateSequence(0) { it + 1 }.map { "$PREFIX$it" }.first { k -> list.none { it.key == k } } - save(list + WebApp(key, name.ifEmpty { uri.host.orEmpty() }, url, manifest)) + val keys = fields.mapNotNull(::entryKey).toSet() + val key = generateSequence(0) { it + 1 }.map { "$PREFIX$it" }.first { it !in keys } + // DisplayConfig {1 item {1 title}, 2 url, 3 signedWebAppManifest}: the manifest goes in as the catalogue sent it. + val config = listOfNotNull( + message(1, listOf(text(1, name.ifEmpty { uri.host.orEmpty() }))), + text(2, url), + manifest?.let { runCatching { Base64.decode(it, Base64.DEFAULT) }.getOrNull() }?.let { ProtoField(3, 2, bytes = it) }, + ) + if (!write(fields + message(DISPLAY_CONFIGS, listOf(text(1, key), message(2, config))))) return AddResult.NOT_CONNECTED } log.get().add("Web apps: added ${uri.host}") return AddResult.ADDED } fun remove(key: String): Boolean = synchronized(lock) { - val list = _apps.value - val rest = list.filterNot { it.key == key } - if (rest.size == list.size) return false - save(rest) - true + val fields = current() ?: return false + val rest = fields.filterNot { it.number == DISPLAY_CONFIGS && entryKey(it) == key } + rest.size != fields.size && write(rest) } private fun rename(key: String, name: String): Boolean = synchronized(lock) { - if (name.isEmpty() || _apps.value.none { it.key == key }) return false - save(_apps.value.map { if (it.key == key) it.copy(name = name) else it }) - true + val fields = current() ?: return false + if (name.isEmpty() || fields.none { entryKey(it) == key }) return false + write(fields.map { f -> if (entryKey(f) == key) renamed(f, name) else f }) } - /** - * Write the list into the glasses' partner_app_settings, with developer mode on. False when the glasses cannot take - * it now (not connected, or their settings not read yet). - */ - fun push(): Boolean { - val glasses = connection.get() + private fun ready() = connection.get().state.value.link == LinkState.READY && settings.get().state.value.readAt != null + + /** The setting as it is now, read into fields; null when it cannot be changed safely (not connected, unreadable). */ + private fun current(): List? { + if (!ready()) return null + val value = inForce(settings.get().state.value) + if (value.isEmpty()) return emptyList() + return runCatching { Base64.decode(value, Base64.NO_WRAP) }.getOrNull()?.let(::parseProto) + } + + /** The value written last if the glasses have not confirmed it yet, else what they hold. */ + private fun inForce(state: SettingsState): String = state.pending[SETTING] ?: state.stored[SETTING].orEmpty() + + private fun write(fields: List): Boolean { val s = settings.get() - if (glasses.state.value.link != LinkState.READY || s.state.value.readAt == null) return false - val value = Base64.encodeToString(partnerSettings(s.value(SETTING).orEmpty(), _apps.value), Base64.NO_WRAP) - val sent = s.write(SETTING, value) && s.write(DEVELOPER_MODE, "true") + val sent = s.write(SETTING, Base64.encodeToString(encodeProto(fields), Base64.NO_WRAP)) && + (s.value(DEVELOPER_MODE) == "true" || s.write(DEVELOPER_MODE, "true")) if (!sent) log.get().add("Web apps: the list did not go to the glasses") return sent } @@ -163,19 +162,17 @@ class WebApps @Inject constructor( val name = f.bytes(3).decodeToString() return when (f.varint(2)) { ACTION_CREATE -> when (add(name, f.bytes(4).decodeToString())) { - AddResult.ADDED, AddResult.EXISTS -> { - push() - answer() - } + AddResult.ADDED, AddResult.EXISTS -> answer() AddResult.NOT_HTTPS -> answer("Only https pages can be web apps") AddResult.FULL -> answer("Up to $MAX_APPS web apps") + AddResult.NOT_CONNECTED, AddResult.UNREADABLE -> answer("The web apps could not be read") } - ACTION_DELETE -> if (remove(key)) { push(); answer() } else answer("No such web app") - ACTION_LIST -> answer(apps = _apps.value) - ACTION_UPDATE -> if (rename(key, name)) { push(); answer() } else answer("No such web app") + ACTION_DELETE -> if (remove(key)) answer() else answer("No such web app") + ACTION_LIST -> answer(apps = apps.value) + ACTION_UPDATE -> if (rename(key, name)) answer() else answer("No such web app") // The glasses open pages through the phone, not the other way round. ACTION_OPEN -> answer("Web apps open from the phone") - // Visor keeps the list, so developer mode is always on for it. + // Developer mode goes on with the first web app Visor adds. ACTION_DEV_MODE_STATUS -> answer() else -> answer("Unknown action") } @@ -199,48 +196,41 @@ class WebApps @Inject constructor( /** WebappAppInfo {1 partnerKey, 2 appName, 3 appUrl, 4 iconUrl}. */ private fun info(app: WebApp): ByteArray = Wire.fieldBytes(1, app.key.encodeToByteArray()) + Wire.fieldBytes(2, app.name.encodeToByteArray()) + - Wire.fieldBytes(3, app.url.encodeToByteArray()) + Wire.fieldBytes(4, ByteArray(0)) + Wire.fieldBytes(3, app.url.encodeToByteArray()) + Wire.fieldBytes(4, app.icon.encodeToByteArray()) - /** - * The stored PartnerAppSettings with its dev entries replaced by ours. PartnerAppSettings {3 displayConfigs: map - * entry {1 key, 2 DisplayConfig {1 Item {1 title}, 2 url}}}. Everything else in the setting is kept as it was. - */ - private fun partnerSettings(current: String, apps: List): ByteArray { - val old = runCatching { parseProto(Base64.decode(current, Base64.NO_WRAP)) }.getOrNull().orEmpty() - val kept = old.filterNot { it.number == DISPLAY_CONFIGS && entryKey(it)?.startsWith(PREFIX) == true } - val ours = apps.map { app -> - // DisplayConfig {1 item, 2 url, 3 signedWebAppManifest}: the manifest goes in as the catalogue sent it. - val config = listOfNotNull( - message(1, listOf(text(1, app.name))), - text(2, app.url), - app.manifest?.let { runCatching { Base64.decode(it, Base64.DEFAULT) }.getOrNull() }?.let { ProtoField(3, 2, bytes = it) }, - ) - message(DISPLAY_CONFIGS, listOf(text(1, app.key), message(2, config))) - } - return encodeProto(kept + ours) + /** PartnerAppSettings {3 displayConfigs: map entry {1 key, 2 DisplayConfig {1 Item {1 title, 2 icon}, 2 url}}}. */ + private fun appsIn(value: String): List = + value.takeIf { it.isNotEmpty() }?.let { runCatching { Base64.decode(it, Base64.NO_WRAP) }.getOrNull() }?.let(::parseProto)?.let(::apps).orEmpty() + + private fun apps(fields: List): List = fields.filter { it.number == DISPLAY_CONFIGS }.mapNotNull { entry -> + val e = parseProto(entry.bytes) ?: return@mapNotNull null + val key = e.firstOrNull { it.number == 1 }?.bytes?.decodeToString() ?: return@mapNotNull null + val config = e.firstOrNull { it.number == 2 }?.bytes?.let(::parseProto) ?: return@mapNotNull null + val url = config.firstOrNull { it.number == 2 }?.bytes?.decodeToString()?.takeIf { it.startsWith("http") } ?: return@mapNotNull null + val item = config.firstOrNull { it.number == 1 }?.bytes?.let(::parseProto).orEmpty() + val name = item.firstOrNull { it.number == 1 }?.bytes?.decodeToString().orEmpty() + WebApp(key, name.ifEmpty { Uri.parse(url).host.orEmpty() }, url, item.firstOrNull { it.number == 2 }?.bytes?.decodeToString().orEmpty()) + } + + /** [entry] with its item's title set to [name]; every other field kept as it was. */ + private fun renamed(entry: ProtoField, name: String): ProtoField { + val e = parseProto(entry.bytes) ?: return entry + return message(DISPLAY_CONFIGS, e.map { f -> + val config = if (f.number == 2) parseProto(f.bytes) else null + if (config == null) f else message(2, config.map { c -> + val item = if (c.number == 1) parseProto(c.bytes) else null + if (item == null) c else message(1, listOf(text(1, name)) + item.filterNot { it.number == 1 }) + }) + }) } private fun entryKey(entry: ProtoField): String? = - parseProto(entry.bytes)?.firstOrNull { it.number == 1 }?.bytes?.decodeToString() + entry.takeIf { it.number == DISPLAY_CONFIGS }?.let { parseProto(it.bytes) }?.firstOrNull { it.number == 1 }?.bytes?.decodeToString() private fun text(number: Int, value: String) = ProtoField(number, 2, bytes = value.encodeToByteArray()) private fun message(number: Int, fields: List) = ProtoField(number, 2, bytes = encodeProto(fields)) - private fun save(list: List) { - _apps.value = list - runCatching { - file.writeText(JSONArray(list.map { app -> - JSONObject().put("key", app.key).put("name", app.name).put("url", app.url).also { o -> app.manifest?.let { o.put("manifest", it) } } - }).toString()) - } - } - - private fun load(): List = runCatching { - val a = JSONArray(file.readText()) - List(a.length()) { i -> a.getJSONObject(i).let { WebApp(it.getString("key"), it.getString("name"), it.getString("url"), it.optString("manifest", "").ifEmpty { null }) } } - }.getOrDefault(emptyList()) - companion object { const val SVC = 55074 private const val SVC_LEASE = 56 diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt index af73db48..307b11b7 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt @@ -8,7 +8,6 @@ import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding import androidx.compose.runtime.Composable -import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember @@ -22,30 +21,23 @@ import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.lifecycle.viewModelScope import com.macedonga.visor.R import com.macedonga.visor.connection.AddResult -import com.macedonga.visor.connection.OPEN_GATES -import com.macedonga.visor.connection.OpenGate import com.macedonga.visor.connection.WebApp import com.macedonga.visor.connection.WebApps -import com.macedonga.visor.connection.gateOf import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.identity.appstore.CatalogueApp import com.macedonga.visor.core.identity.appstore.fetchAppCatalogue -import com.macedonga.visor.core.media.GlassesConfig import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.login.MetaLogin import dagger.hilt.android.lifecycle.HiltViewModel import javax.inject.Inject import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.flow.map import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch import kotlinx.coroutines.withContext -enum class WebAppNote { SAVED_ON_PHONE, OPENED, NOT_OPENED, RESTART, SWITCH_NOT_SENT, NOT_HTTPS, FULL } +enum class WebAppNote { NOT_CONNECTED, UNREADABLE, OPENED, NOT_OPENED, NOT_HTTPS, FULL } sealed interface CatalogueState { data object Idle : CatalogueState @@ -58,11 +50,9 @@ sealed interface CatalogueState { class WebAppsModel @Inject constructor( private val webApps: WebApps, val connection: GlassesConnection, - private val config: GlassesConfig, private val login: MetaLogin, ) : ViewModel() { val apps: StateFlow> = webApps.apps - val gate: StateFlow = config.state.map { gateOf(it) }.stateIn(viewModelScope, SharingStarted.Eagerly, gateOf(config.state.value)) private val _note = MutableStateFlow(null) val note: StateFlow = _note.asStateFlow() @@ -70,16 +60,8 @@ class WebAppsModel @Inject constructor( private val _catalogue = MutableStateFlow(CatalogueState.Idle) val catalogue: StateFlow = _catalogue.asStateFlow() - fun sync() { - webApps.push() - } - fun add(name: String, url: String) { - _note.value = when (webApps.add(name, url)) { - AddResult.ADDED, AddResult.EXISTS -> if (webApps.push()) null else WebAppNote.SAVED_ON_PHONE - AddResult.NOT_HTTPS -> WebAppNote.NOT_HTTPS - AddResult.FULL -> WebAppNote.FULL - } + _note.value = noteOf(webApps.add(name, url)) } /** Fetch Meta's list for these glasses, with the user's Meta login. */ @@ -98,44 +80,38 @@ class WebAppsModel @Inject constructor( } fun addFromCatalogue(app: CatalogueApp) { - _note.value = when (webApps.add(app.name, app.url, app.manifest)) { - AddResult.ADDED, AddResult.EXISTS -> if (webApps.push()) null else WebAppNote.SAVED_ON_PHONE - AddResult.NOT_HTTPS -> WebAppNote.NOT_HTTPS - AddResult.FULL -> WebAppNote.FULL - } + _note.value = noteOf(webApps.add(app.name, app.url, app.manifest)) } fun remove(key: String) { - webApps.remove(key) - _note.value = if (webApps.push()) null else WebAppNote.SAVED_ON_PHONE + _note.value = if (webApps.remove(key)) null else WebAppNote.NOT_CONNECTED } - fun open(app: WebApp) { - viewModelScope.launch { - _note.value = if (webApps.open(app)) WebAppNote.OPENED else WebAppNote.NOT_OPENED - } + private fun noteOf(result: AddResult): WebAppNote? = when (result) { + AddResult.ADDED, AddResult.EXISTS -> null + AddResult.NOT_HTTPS -> WebAppNote.NOT_HTTPS + AddResult.FULL -> WebAppNote.FULL + AddResult.NOT_CONNECTED -> WebAppNote.NOT_CONNECTED + AddResult.UNREADABLE -> WebAppNote.UNREADABLE } - /** Switch the two gates on by MobileConfig: the glasses need a restart to use them. */ - fun turnOn() { + fun open(app: WebApp) { viewModelScope.launch { - _note.value = if (config.send(OPEN_GATES.associateWith { true }) == null) WebAppNote.RESTART else WebAppNote.SWITCH_NOT_SENT + _note.value = if (webApps.open(app)) WebAppNote.OPENED else WebAppNote.NOT_OPENED } } } -/** The web apps Visor lists for the Display, and opening one on the glasses. */ +/** The Display's web apps, as the glasses list them, and opening one on the glasses. */ @Composable fun WebAppsScreen(model: WebAppsModel = hiltViewModel()) { val glasses by model.connection.state.collectAsStateWithLifecycle() val ready = glasses.link == LinkState.READY val apps by model.apps.collectAsStateWithLifecycle() - val gate by model.gate.collectAsStateWithLifecycle() val note by model.note.collectAsStateWithLifecycle() val catalogue by model.catalogue.collectAsStateWithLifecycle() var adding by remember { mutableStateOf(false) } var browsing by remember { mutableStateOf(false) } - LaunchedEffect(ready) { if (ready) model.sync() } SettingsPage(stringResource(R.string.webapps_title), light = 0f) { item(key = "apps") { @@ -146,24 +122,18 @@ fun WebAppsScreen(model: WebAppsModel = hiltViewModel()) { RowFrame(app.name, onClick = null, sub = app.url) { ActionPill( stringResource(R.string.webapps_open), { model.open(app) }, quiet = true, - modifier = Modifier.padding(start = 12.dp), enabled = ready && gate == OpenGate.ON, + modifier = Modifier.padding(start = 12.dp), enabled = ready, ) ActionPill( stringResource(R.string.webapps_remove), { model.remove(app.key) }, quiet = true, - modifier = Modifier.padding(start = 12.dp), + modifier = Modifier.padding(start = 12.dp), enabled = ready, ) } } }.toTypedArray(), - { ValueRow(stringResource(R.string.webapps_add), null, onClick = { adding = true }) }, + { ValueRow(stringResource(R.string.webapps_add), null, onClick = { adding = true }, locked = !ready) }, { ValueRow(stringResource(R.string.webapps_browse), null, onClick = { browsing = true; model.browse() }, locked = !ready) }, - if (gate == OpenGate.OFF) row { ValueRow(stringResource(R.string.webapps_turn_on), null, onClick = { model.turnOn() }, locked = !ready) } else null, ) - when (gate) { - OpenGate.UNAVAILABLE -> Footnote(stringResource(R.string.webapps_unavailable)) - OpenGate.UNREAD -> Footnote(stringResource(R.string.webapps_unread)) - OpenGate.OFF, OpenGate.ON -> Unit - } note?.let { Footnote(stringResource(noteText(it))) } if (apps.isEmpty()) Footnote(stringResource(R.string.webapps_none)) } @@ -218,11 +188,10 @@ fun WebAppsScreen(model: WebAppsModel = hiltViewModel()) { } private fun noteText(note: WebAppNote): Int = when (note) { - WebAppNote.SAVED_ON_PHONE -> R.string.webapps_saved_on_phone + WebAppNote.NOT_CONNECTED -> R.string.webapps_not_connected + WebAppNote.UNREADABLE -> R.string.webapps_unreadable WebAppNote.OPENED -> R.string.webapps_opened WebAppNote.NOT_OPENED -> R.string.webapps_not_opened - WebAppNote.RESTART -> R.string.webapps_restart - WebAppNote.SWITCH_NOT_SENT -> R.string.webapps_switch_not_sent WebAppNote.NOT_HTTPS -> R.string.webapps_not_https WebAppNote.FULL -> R.string.webapps_full } diff --git a/app/src/main/res/values-it/strings_webapps.xml b/app/src/main/res/values-it/strings_webapps.xml index 0c7107c1..4091e2f9 100644 --- a/app/src/main/res/values-it/strings_webapps.xml +++ b/app/src/main/res/values-it/strings_webapps.xml @@ -12,14 +12,10 @@ Sugli occhiali Impossibile caricare il catalogo Nessuna app web per questi occhiali - Attiva le app web - Gli occhiali non hanno questo interruttore - Collega gli occhiali per leggerlo - Salvata sul telefono. Gli occhiali la ricevono quando sono collegati. Aperta sugli occhiali Gli occhiali non l\'hanno aperta - Riavvia gli occhiali per usare le app web - L\'interruttore non è stato inviato Usa un indirizzo https Massimo 20 app web + Collega prima gli occhiali + Impossibile leggere le app web degli occhiali diff --git a/app/src/main/res/values/strings_webapps.xml b/app/src/main/res/values/strings_webapps.xml index 0f1d814a..20ffb6c2 100644 --- a/app/src/main/res/values/strings_webapps.xml +++ b/app/src/main/res/values/strings_webapps.xml @@ -12,14 +12,10 @@ On the glasses Could not load the catalogue No web apps for these glasses - Turn on web apps - The glasses do not have this switch - Connect the glasses to read this - Saved on the phone. The glasses get it when connected. Opened on the glasses The glasses did not open it - Restart the glasses to use web apps - The switch was not sent Use an https address Up to 20 web apps + Connect the glasses first + Could not read the glasses\' web apps From 46ba29c8b021783f865e197eae1aa05c0af14085 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 20:35:31 +0200 Subject: [PATCH 78/90] Display: read the Meta account's linked accounts as Meta lists them. MetaLinkedAccounts answers data.wearable_linked_accounts with obfuscated_id, account_type, display_name and is_enabled_for_wearable (seen today); Visor looked for obfuscated_account_id, so every account came back "not linked" and nothing was sent to the glasses. The Accounts page now reads the list when it opens and shows each provider as linked, with its name, or turned off for glasses, before anything is tapped. --- .../visor/connection/AccountLinking.kt | 12 +++++++ .../visor/settings/AccountsScreen.kt | 20 +++++++++-- .../main/res/values-it/strings_accounts.xml | 2 ++ app/src/main/res/values/strings_accounts.xml | 2 ++ .../core/identity/token/LinkedAccounts.kt | 35 ++++++++----------- 5 files changed, 47 insertions(+), 24 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt b/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt index fb25e59a..2a45b0bd 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt @@ -5,6 +5,7 @@ import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.identity.token.LinkedAccountType import com.macedonga.visor.core.identity.token.TokenError import com.macedonga.visor.core.identity.token.exchangeAccountToken +import com.macedonga.visor.core.identity.token.LinkedAccountRef import com.macedonga.visor.core.identity.token.fetchLinkedAccounts import com.macedonga.visor.core.identity.token.fetchLinkedTokens import com.macedonga.visor.core.protocol.Wire @@ -83,6 +84,17 @@ class AccountLinking @Inject constructor( return LinkOutcome.LINKED } + /** The provider accounts linked to the user's Meta account, as Meta lists them; null when they cannot be read. */ + suspend fun linkedAccounts(): List? { + if (!login.source.hasCachedLogin) return null + return try { + withContext(Dispatchers.IO) { fetchLinkedAccounts(login.source.withToken { exchangeAccountToken(it) }.accessToken) } + } catch (error: TokenError) { + log.add("Accounts: ${error.message}") + null + } + } + /** Sends a provider account of the Meta account, with its tokens, to the glasses. */ suspend fun syncLinked(type: LinkedAccountType): LinkOutcome { if (!login.source.hasCachedLogin) return LinkOutcome.LOGGED_OUT diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt index c582b404..3a69a7f8 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt @@ -15,6 +15,7 @@ import com.macedonga.visor.R import com.macedonga.visor.connection.AccountLinking import com.macedonga.visor.connection.LinkOutcome import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.token.LinkedAccountRef import com.macedonga.visor.core.identity.token.LinkedAccountType import com.macedonga.visor.core.session.LinkState import dagger.hilt.android.lifecycle.HiltViewModel @@ -49,8 +50,13 @@ class AccountsModel @Inject constructor( private val _linkedResults = MutableStateFlow>(emptyMap()) val linkedResults: StateFlow> = _linkedResults.asStateFlow() + // The provider accounts Meta lists for the user, null until read. + private val _accounts = MutableStateFlow?>(null) + val accounts: StateFlow?> = _accounts.asStateFlow() + fun refresh() { viewModelScope.launch { linking.metaPresent()?.let { _meta.value = it } } + viewModelScope.launch { linking.linkedAccounts()?.let { list -> _accounts.value = list.associateBy { it.type } } } } fun linkMeta() = task(AccountRow.META) { @@ -94,6 +100,7 @@ fun AccountsScreen(model: AccountsModel = hiltViewModel()) { val metaResult by model.metaResult.collectAsStateWithLifecycle() val whatsappResult by model.whatsappResult.collectAsStateWithLifecycle() val linked by model.linkedResults.collectAsStateWithLifecycle() + val accounts by model.accounts.collectAsStateWithLifecycle() LaunchedEffect(ready) { if (ready) model.refresh() } SettingsPage(stringResource(R.string.accounts_title), light = 0f) { @@ -136,8 +143,8 @@ fun AccountsScreen(model: AccountsModel = hiltViewModel()) { ) } }, - { LinkedRow(model, AccountRow.FACEBOOK, LinkedAccountType.FACEBOOK, R.string.accounts_facebook, ready, busy, linked) }, - { LinkedRow(model, AccountRow.INSTAGRAM, LinkedAccountType.INSTAGRAM, R.string.accounts_instagram, ready, busy, linked) }, + { LinkedRow(model, AccountRow.FACEBOOK, LinkedAccountType.FACEBOOK, R.string.accounts_facebook, ready, busy, linked, accounts) }, + { LinkedRow(model, AccountRow.INSTAGRAM, LinkedAccountType.INSTAGRAM, R.string.accounts_instagram, ready, busy, linked, accounts) }, ) } } @@ -152,11 +159,18 @@ private fun LinkedRow( ready: Boolean, busy: AccountRow?, results: Map, + accounts: Map?, ) { + val account = accounts?.get(type) RowFrame( stringResource(label), onClick = null, - sub = results[type]?.let { outcomeWords(it) }, + sub = results[type]?.let { outcomeWords(it) } ?: when { + accounts == null -> null + account == null -> stringResource(R.string.accounts_not_linked) + account.forGlasses -> stringResource(R.string.accounts_linked_as, account.name) + else -> stringResource(R.string.accounts_off_for_glasses, account.name) + }, locked = !ready, busy = busy == row, ) { diff --git a/app/src/main/res/values-it/strings_accounts.xml b/app/src/main/res/values-it/strings_accounts.xml index bc8b3632..aef4da15 100644 --- a/app/src/main/res/values-it/strings_accounts.xml +++ b/app/src/main/res/values-it/strings_accounts.xml @@ -14,4 +14,6 @@ Meta non raggiungibile Gli occhiali non hanno risposto Gli occhiali lo hanno rifiutato + Collegato come %1$s + %1$s, disattivato per gli occhiali diff --git a/app/src/main/res/values/strings_accounts.xml b/app/src/main/res/values/strings_accounts.xml index ce280a30..36c752a8 100644 --- a/app/src/main/res/values/strings_accounts.xml +++ b/app/src/main/res/values/strings_accounts.xml @@ -14,4 +14,6 @@ Meta could not be reached The glasses did not answer The glasses refused it + Linked as %1$s + %1$s, turned off for glasses diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/LinkedAccounts.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/LinkedAccounts.kt index 8cf2dbe4..da2e8245 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/LinkedAccounts.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/LinkedAccounts.kt @@ -5,7 +5,6 @@ import com.macedonga.visor.core.identity.UrlConnectionPost import com.macedonga.visor.core.identity.pyUrlencode import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.jsonObject @@ -17,8 +16,8 @@ enum class LinkedAccountType(val wire: Int, val graphName: String, val appId: St THREADS(3, "THREADS", "1590413528774085"), } -/** A linked account as the Meta account lists it: the id the Graph queries take. */ -class LinkedAccountRef(val type: LinkedAccountType, val obfuscatedId: String) +/** A linked account as the Meta account lists it: the id the Graph queries take, its name, and whether glasses may use it. */ +class LinkedAccountRef(val type: LinkedAccountType, val obfuscatedId: String, val name: String, val forGlasses: Boolean) /** A linked account's access token and, when Meta gives one, its first-party token. Secrets: never log them. */ class LinkedTokens(val accessToken: String, val firstPartyToken: String?) @@ -30,25 +29,19 @@ private const val LINKED_TOKENS_ROOT = "fx_ar_linked_access_and_first_party_toke private const val TOKEN_PACKAGE = "com.facebook.stella" /** - * The provider accounts linked to the Meta account, from the accounts app's token minted by [exchangeAccountToken]. Each - * node that carries an obfuscated id and an account type counts. + * The provider accounts linked to the Meta account: data.wearable_linked_accounts [{obfuscated_id, account_type, + * display_name, is_enabled_for_wearable}], as Meta answered it on 2026-10-23. */ -fun fetchLinkedAccounts(accountsToken: String, http: HttpPost = UrlConnectionPost): List { - val found = mutableListOf() - fun walk(element: JsonElement) { - when (element) { - is JsonObject -> { - val id = element.text("obfuscated_account_id") - val type = element.text("account_type")?.let { name -> LinkedAccountType.entries.firstOrNull { it.graphName == name.uppercase() } } - if (id != null && type != null) found += LinkedAccountRef(type, id) - element.values.forEach(::walk) - } - is JsonArray -> element.forEach(::walk) - else -> Unit - } - } - walk(graphql(accountsToken, "MetaLinkedAccounts", LINKED_LIST_DOC, "{}", http)) - return found.distinctBy { it.type } +fun fetchLinkedAccounts(accessToken: String, http: HttpPost = UrlConnectionPost): List { + val json = graphql(accessToken, "MetaLinkedAccounts", LINKED_LIST_DOC, "{}", http) + val list = (json["data"] as? JsonObject)?.get("wearable_linked_accounts") as? JsonArray ?: return emptyList() + return list.mapNotNull { element -> + val node = element as? JsonObject ?: return@mapNotNull null + val id = node.text("obfuscated_id") ?: return@mapNotNull null + val type = node.text("account_type")?.let { name -> LinkedAccountType.entries.firstOrNull { it.graphName == name.uppercase() } } + ?: return@mapNotNull null + LinkedAccountRef(type, id, node.text("display_name").orEmpty(), (node["is_enabled_for_wearable"] as? JsonPrimitive)?.content == "true") + }.distinctBy { it.type } } /** The tokens of one linked account, from the accounts app's token; null when Meta gives no access token for it. */ From 26d2b58bfd5e337723f60d05f7aac5f5a91ab762 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 20:44:40 +0200 Subject: [PATCH 79/90] Display: save a Wi-Fi network as one SaveNetwork (type 1). The Display's cloud configuration service knows only that request; it answered the Meta app's newer SaveNetworkList (type 18) with 0xD001, unknown type, so no network could be added. The list stays as the fallback. --- .../main/res/values-it/strings_bandfirmware.xml | 16 ---------------- app/src/main/res/values-it/strings_haptics.xml | 4 ---- app/src/main/res/values/strings_bandfirmware.xml | 16 ---------------- app/src/main/res/values/strings_haptics.xml | 4 ---- .../visor/core/connection/GlassesWifi.kt | 5 ++++- 5 files changed, 4 insertions(+), 41 deletions(-) delete mode 100644 app/src/main/res/values-it/strings_bandfirmware.xml delete mode 100644 app/src/main/res/values-it/strings_haptics.xml delete mode 100644 app/src/main/res/values/strings_bandfirmware.xml delete mode 100644 app/src/main/res/values/strings_haptics.xml diff --git a/app/src/main/res/values-it/strings_bandfirmware.xml b/app/src/main/res/values-it/strings_bandfirmware.xml deleted file mode 100644 index dd2f8119..00000000 --- a/app/src/main/res/values-it/strings_bandfirmware.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - Firmware - Firmware del braccialetto - Versione - Aggiornamento - Cerca aggiornamenti - Ricerca - Aggiornato - Installa - Installare %1$s sul braccialetto? - Installazione - Installazione, %1$d%% - Installato - Errore: %1$s - diff --git a/app/src/main/res/values-it/strings_haptics.xml b/app/src/main/res/values-it/strings_haptics.xml deleted file mode 100644 index a5fa09db..00000000 --- a/app/src/main/res/values-it/strings_haptics.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - Il Neural Band non ha vibrato - diff --git a/app/src/main/res/values/strings_bandfirmware.xml b/app/src/main/res/values/strings_bandfirmware.xml deleted file mode 100644 index 9b7eaca8..00000000 --- a/app/src/main/res/values/strings_bandfirmware.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - Firmware - Band firmware - Version - Update - Check for update - Checking - Up to date - Install - Install %1$s on the band? - Installing - Installing, %1$d%% - Installed - Failed: %1$s - diff --git a/app/src/main/res/values/strings_haptics.xml b/app/src/main/res/values/strings_haptics.xml deleted file mode 100644 index 76e81ec5..00000000 --- a/app/src/main/res/values/strings_haptics.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - The band did not buzz - diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt index 516e6ede..fafd92f2 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt @@ -16,6 +16,7 @@ private const val MT_REMOVE_NETWORK = 3 private const val MT_REMOVE_NETWORK_RESULT = 4 private const val MT_QUERY_NETWORKS = 5 private const val MT_QUERY_NETWORKS_RESULT = 6 +private const val MT_SAVE_NETWORK = 1 private const val MT_SAVE_NETWORK_LIST = 18 private const val MT_SAVE_NETWORK_RESULT = 2 @@ -72,7 +73,9 @@ class GlassesWifi(private val connection: GlassesConnection) { Wire.fieldVarint(5, 1) + Wire.fieldVarint(6, RANDOMIZATION_PUBLIC) + Wire.fieldVarint(9, REQUEST_CONNECT) - val answer = ask(MT_SAVE_NETWORK_LIST, Wire.fieldBytes(1, network), MT_SAVE_NETWORK_RESULT) + // The Display's firmware knows only the single SaveNetwork (type 1) and answers the list (18) with 0xD001. + val answer = ask(MT_SAVE_NETWORK, network, MT_SAVE_NETWORK_RESULT) + ?: ask(MT_SAVE_NETWORK_LIST, Wire.fieldBytes(1, network), MT_SAVE_NETWORK_RESULT) answer?.let { Wire.parse(it).varint(1).toInt().also { status -> note("save $ssid: status $status") } } } From e2ac8e3e0d931da2b3e77ed161cf6944dc928ed0 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 20:44:40 +0200 Subject: [PATCH 80/90] Neural Band: show the band as the Display pairs it. The band belongs to the Display; the page now asks the Display over svc41109 (GET_BLE_PAIRING_INFO) whether a band is paired and connected, and shows its battery and fit from the Display's state. The phone-side rows are gone: Vibrate, Wrist, Firmware and Set up worked only on a band the phone owned, which a band paired with the Display is not. --- .../com/macedonga/visor/band/BandPeers.kt | 65 +++++++++ .../com/macedonga/visor/settings/BandPart.kt | 134 +++--------------- .../visor/settings/SettingsScreen.kt | 15 +- app/src/main/res/values-it/strings.xml | 17 +-- app/src/main/res/values/strings.xml | 17 +-- 5 files changed, 98 insertions(+), 150 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/band/BandPeers.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandPeers.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandPeers.kt new file mode 100644 index 00000000..4495f5b1 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandPeers.kt @@ -0,0 +1,65 @@ +package com.macedonga.visor.band + +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.LinkState +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.onSubscription +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withTimeoutOrNull + +/** A band the Display has paired: its address, whether it is connected, and whether directly or through the phone. */ +data class PairedBand(val address: String, val connected: Boolean, val direct: Boolean) + +/** + * svc41109, bt_peer_connection: the Display's own pairing with the Neural Band. GET_BLE_PAIRING_INFO (1) is answered + * with BLE_PAIRING_INFO (2): {3 pairedPeers [ConnectionState {1 peerIdentityAddr BleAddr {2 addr}, 2 isConnected}], + * 5 activeLink [ActiveLink {1 link (1 relay, 2 direct), 2 peerIdentityAddr}]}. + */ +@Singleton +class BandPeers @Inject constructor(private val connection: GlassesConnection) { + private val lock = Mutex() + private val _bands = MutableStateFlow?>(null) + + /** The bands the Display has paired, as it last said; null until it has answered. */ + val bands: StateFlow?> = _bands.asStateFlow() + + /** Asks the Display again; without an answer [bands] stays as it was. */ + suspend fun refresh() = lock.withLock { + if (connection.state.value.link != LinkState.READY) return@withLock + val answer = withTimeoutOrNull(ANSWER_MS) { + connection.serviceData + .onSubscription { connection.submit { it.sendOnService(SVC, MT_GET_PAIRING_INFO, ByteArray(0)) } } + .first { it.serviceId == SVC && it.messageType == MT_PAIRING_INFO } + .payload + } ?: return@withLock + runCatching { parse(answer) }.getOrNull()?.let { _bands.value = it } + } + + private fun parse(payload: ByteArray): List { + val info = Wire.parse(payload) + val links = info.repeatedBytes(5).map(Wire::parse).associate { address(it.bytes(2)) to it.varint(1) } + return info.repeatedBytes(3).map(Wire::parse).map { peer -> + val address = address(peer.bytes(1)) + PairedBand(address, peer.varint(2) == 1L, links[address] == LINK_DIRECT) + } + } + + /** BleAddr {2 addr}, least significant byte first. */ + private fun address(bleAddr: ByteArray): String = + Wire.parse(bleAddr).bytes(2).reversed().joinToString(":") { "%02X".format(it) } + + private companion object { + const val SVC = 41109 + const val MT_GET_PAIRING_INFO = 1 + const val MT_PAIRING_INFO = 2 + const val LINK_DIRECT = 2L + const val ANSWER_MS = 8_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt index 0d5d91bd..2e35aad7 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt @@ -1,63 +1,36 @@ package com.macedonga.visor.settings -import androidx.compose.foundation.layout.Arrangement -import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.padding import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.remember -import androidx.compose.runtime.rememberCoroutineScope -import androidx.compose.runtime.setValue -import androidx.compose.ui.Modifier import androidx.compose.ui.res.stringResource -import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.collectAsStateWithLifecycle import com.macedonga.visor.R -import com.macedonga.visor.band.BandSetup -import com.macedonga.visor.band.BandUpdate -import com.macedonga.visor.band.BandWork import com.macedonga.visor.core.session.LinkState -import kotlinx.coroutines.launch -/** The Neural Band that goes with this Display: set it up, or its battery and the wrist it is on. */ +/** The Neural Band as the Display holds it: whether it is paired and connected, its battery and its fit. */ @Composable internal fun BandPart(model: SettingsViewModel) { - val pair by model.connection.pair.collectAsStateWithLifecycle() - val all by model.bands.all.collectAsStateWithLifecycle() - val band by model.band.state.collectAsStateWithLifecycle() val glasses by model.connection.state.collectAsStateWithLifecycle() - // The Display reports its band; the phone's own reading is for when the Display is away. + val paired by model.bandPeers.bands.collectAsStateWithLifecycle() + val ready = glasses.link == LinkState.READY + LaunchedEffect(ready) { if (ready) model.bandPeers.refresh() } val device = glasses.device - val battery = device?.bandBattery ?: band.battery - val charging = device?.bandCharging ?: band.charging - val setup by model.bandSetup.status.collectAsStateWithLifecycle() - val running = setup is BandSetup.Status.Running - val problem = (setup as? BandSetup.Status.Failed)?.let { failWords(it) } - val scope = rememberCoroutineScope() - var refused by remember { mutableStateOf(false) } - var firmwareOpen by remember { mutableStateOf(false) } - @Composable - fun SetUpRow(title: Int) = ValueRow( - stringResource(title), - if (running) stringResource(R.string.band_setting_up) else null, - onClick = if (running) null else ({ model.bandSetup.start() }), - busy = running, - problem = problem, - ) - if (all.none { it.glasses == pair }) { - // A band the Display already holds reports through it; Visor's own link to the band needs the setup. - Rows( - battery?.let { b -> row { ValueRow(stringResource(R.string.band_battery), batteryWords(b, charging), onClick = null) } }, - fitWords(device?.bandFit)?.let { fit -> row { ValueRow(stringResource(R.string.band_fit), fit, onClick = null) } }, - { SetUpRow(R.string.band_set_up) }, - ) - return - } + val band = paired?.firstOrNull() Rows( - { ValueRow(stringResource(R.string.band_battery), battery?.let { batteryWords(it, charging) }, onClick = null) }, - { ValueRow(stringResource(R.string.band_firmware), band.firmware?.toString() ?: stringResource(R.string.settings_unknown), onClick = { firmwareOpen = true }) }, + { + ValueRow( + stringResource(R.string.band_glasses), + when { + !ready || paired == null -> stringResource(R.string.settings_unknown) + band == null -> stringResource(R.string.band_not_paired) + band.connected -> stringResource(R.string.band_paired_connected) + else -> stringResource(R.string.band_paired_away) + }, + onClick = null, + ) + }, + device?.bandBattery?.let { b -> row { ValueRow(stringResource(R.string.band_battery), batteryWords(b, device.bandCharging), onClick = null) } }, fitWords(device?.bandFit)?.let { fit -> row { ValueRow( @@ -66,64 +39,7 @@ internal fun BandPart(model: SettingsViewModel) { ) } }, - { - PillsRow( - stringResource(R.string.band_wrist), - listOf(true to stringResource(R.string.band_left), false to stringResource(R.string.band_right)), - band.leftHanded, - onPick = { model.band.setLeftHanded(it) }, - ) - }, - { - ValueRow( - stringResource(R.string.band_buzz), null, - onClick = { scope.launch { refused = !model.band.vibrate() } }, - problem = if (refused) stringResource(R.string.band_buzz_refused) else null, - ) - }, - { SetUpRow(R.string.band_set_up_again) }, ) - if (firmwareOpen) BandFirmwareSheet(model.bandUpdate, band.firmware) { firmwareOpen = false } -} - -/** The band's firmware: its version, a check for a newer build and the install, as the glasses' sheet does it. */ -@Composable -private fun BandFirmwareSheet(update: BandUpdate, version: Long?, onClose: () -> Unit) { - val checked by update.checked.collectAsStateWithLifecycle() - val offer by update.offer.collectAsStateWithLifecycle() - val work by update.work.collectAsStateWithLifecycle() - var confirming by remember { mutableStateOf(false) } - val busy = work == BandWork.Checking || work is BandWork.Installing - VisorSheet({ onClose(); confirming = false; update.dismiss() }) { - SheetTitle(stringResource(R.string.band_firmware_title)) - val found = offer - Rows( - { ValueRow(stringResource(R.string.band_firmware_current), version?.toString() ?: stringResource(R.string.settings_unknown), onClick = null) }, - found?.let { o -> @Composable { ValueRow(stringResource(R.string.band_firmware_update), o.displayName.ifEmpty { o.targetVersion }, onClick = null) } }, - ) - val status: String? = when (val w = work) { - is BandWork.Failed -> stringResource(R.string.band_firmware_failed, w.reason) - BandWork.Installed -> stringResource(R.string.band_firmware_installed) - is BandWork.Installing -> w.percent?.let { stringResource(R.string.band_firmware_progress, it) } ?: stringResource(R.string.band_firmware_installing) - BandWork.Checking -> stringResource(R.string.band_firmware_checking) - BandWork.Idle -> if (checked && found == null) stringResource(R.string.band_firmware_up_to_date) else null - } - Column(Modifier.padding(horizontal = SIDE, vertical = 16.dp)) { - if (status != null) Footnote(status) - if (confirming && found != null) { - Row(Modifier.padding(top = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp)) { - ActionPill(stringResource(R.string.band_firmware_install), { confirming = false; update.install() }) - ActionPill(stringResource(R.string.language_cancel), { confirming = false }, quiet = true) - } - Footnote(stringResource(R.string.band_firmware_confirm, found.displayName.ifEmpty { found.targetVersion })) - } else { - Row(Modifier.padding(top = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp)) { - ActionPill(stringResource(R.string.band_firmware_check), { update.check() }, quiet = true, enabled = !busy) - if (found != null) ActionPill(stringResource(R.string.band_firmware_install), { confirming = true }, enabled = !busy) - } - } - } - } } @Composable @@ -136,15 +52,3 @@ private fun fitWords(fit: String?): String? = when (fit) { "TIGHT" -> stringResource(R.string.band_fit_tight) else -> null } - -@Composable -internal fun wristWords(left: Boolean): String = stringResource(if (left) R.string.band_left else R.string.band_right) - -@Composable -private fun failWords(failed: BandSetup.Status.Failed): String = when (failed.reason) { - BandSetup.Reason.NO_GLASSES -> stringResource(R.string.band_fail_no_glasses) - BandSetup.Reason.NOT_PAIRED -> stringResource(R.string.band_fail_not_paired) - BandSetup.Reason.NO_ANSWER -> stringResource(R.string.band_fail_no_answer) - BandSetup.Reason.WRONG_OWNER -> stringResource(R.string.band_fail_wrong_owner) - BandSetup.Reason.FAILED -> stringResource(R.string.band_fail, failed.detail) -} diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt index 103a713a..2d6498e2 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt @@ -176,8 +176,7 @@ class SettingsViewModel @Inject constructor( val hearing: com.macedonga.visor.core.assistant.Hearing, val band: com.macedonga.visor.band.BandLink, val bands: com.macedonga.visor.band.Bands, - val bandSetup: com.macedonga.visor.band.BandSetup, - val bandUpdate: com.macedonga.visor.band.BandUpdate, + val bandPeers: com.macedonga.visor.band.BandPeers, val developer: com.macedonga.visor.outside.DeveloperMode, ) : ViewModel() @@ -567,14 +566,12 @@ private fun summary(topic: Topic, model: SettingsViewModel, state: SettingsState shown("tap_and_hold_mapping")?.let { stringResource(R.string.sum_controls_hold, it) }, ).joinToString(" · ") Topic.BAND -> { - val band by model.band.state.collectAsStateWithLifecycle() - val pair by model.connection.pair.collectAsStateWithLifecycle() + // The Display reports the band it holds. val glasses by model.connection.state.collectAsStateWithLifecycle() - // The Display reports the band it holds, set up in Visor or not. - val battery = glasses.device?.bandBattery ?: band.battery - val charging = glasses.device?.bandCharging ?: band.charging - if (model.bands.forGlasses(pair) == null && battery == null) stringResource(R.string.band_none) - else listOfNotNull(battery?.let { batteryWords(it, charging) }, band.leftHanded?.let { wristWords(it) }).joinToString(" · ") + val paired by model.bandPeers.bands.collectAsStateWithLifecycle() + val device = glasses.device + device?.bandBattery?.let { batteryWords(it, device.bandCharging) } + ?: if (paired?.isEmpty() == true) stringResource(R.string.band_not_paired) else "" } Topic.SOUND -> listOfNotNull( shown("notification_volume")?.let { stringResource(R.string.sum_sound_volume, it) }, diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index e8f72af7..3344e6df 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -376,20 +376,8 @@ Privacy Accessi del telefono Neural Band - Configura - Configura di nuovo - Configurazione… - Non configurato Batteria %1$d%% - Polso - Sinistro - Destro - Configura prima gli occhiali - Associa prima il bracciale a questo telefono - Il bracciale non ha risposto - Il bracciale appartiene a un altro account Meta - Configurazione non riuscita: %1$s Cosa arriva agli occhiali, e cosa dicono ad alta voce. Cosa fanno il tasto di cattura e il touchpad sull\'asta destra. Come gli occhiali capiscono di essere indossati, e come risparmiano energia. @@ -994,7 +982,6 @@ Tocca due volte insieme pollice e medio App Tieni uniti pollice e medio - Fai vibrare Vestibilità Larga Aderente @@ -1003,4 +990,8 @@ Una VPN tiene Visor fuori dalla rete degli occhiali. Escludi Visor dalla VPN. Lo streaming si interrompe spesso Usa il Wi-Fi degli occhiali + Occhiali + Non associato + Associato, collegato + Associato, non collegato diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 4db22405..f9448f64 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -377,20 +377,8 @@ Privacy Phone access Neural Band - Set up - Set up again - Setting up… - Not set up Battery %1$d%% - Wrist - Left - Right - Set up the glasses first - Pair the band with this phone first - The band did not answer - The band belongs to another Meta account - Setup failed: %1$s What reaches the glasses, and what they say out loud. What the capture button and the touchpad on the right temple do. How the glasses notice they are on, and how they save power. @@ -996,7 +984,6 @@ Double tap your thumb and middle finger together Apps Hold your thumb and middle finger together - Vibrate Fit Loose Snug @@ -1005,4 +992,8 @@ A VPN keeps Visor off the glasses\' network. Exclude Visor from it. The stream keeps dropping Use the glasses\' Wi-Fi + Glasses + Not paired + Paired, connected + Paired, not connected From af51bc0e9324a5ebb0b05ece3635abac5a637b9b Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 21:07:02 +0200 Subject: [PATCH 81/90] Display: "Open on phone" opens the Accounts page. A Display app's linking button sends svc70 type 28 (TriggerProviderLinking: provider, request id, source, scenario), which Visor did not serve, so the button did nothing. As the Meta app does, the phone now posts a notification (link or reconnect WhatsApp, Instagram, Messenger, Facebook or Threads) that opens Visor's Accounts page, and answers type 29 with whether it was shown. --- .../com/macedonga/visor/MainActivity.kt | 1 + .../kotlin/com/macedonga/visor/VisorNav.kt | 1 + .../macedonga/visor/connection/LinkPrompt.kt | 70 +++++++++++++++++++ .../visor/connection/PhoneMessages.kt | 2 + .../main/res/values-it/strings_accounts.xml | 4 ++ app/src/main/res/values/strings_accounts.xml | 4 ++ 6 files changed, 82 insertions(+) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/LinkPrompt.kt diff --git a/app/src/main/kotlin/com/macedonga/visor/MainActivity.kt b/app/src/main/kotlin/com/macedonga/visor/MainActivity.kt index ddc5321a..99e109dc 100644 --- a/app/src/main/kotlin/com/macedonga/visor/MainActivity.kt +++ b/app/src/main/kotlin/com/macedonga/visor/MainActivity.kt @@ -84,5 +84,6 @@ class MainActivity : ComponentActivity() { const val OPEN_LIVE = "live" const val OPEN_META_AI = "metaai" const val OPEN_ONBOARDING = "onboarding" + const val OPEN_ACCOUNTS = "accounts" } } diff --git a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt index a8984cdc..a2f0dbae 100644 --- a/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt +++ b/app/src/main/kotlin/com/macedonga/visor/VisorNav.kt @@ -184,6 +184,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) if (place == MainActivity.OPEN_GALLERY && backStack.lastOrNull() != MediaKey) backStack.add(MediaKey) if (place == MainActivity.OPEN_META_AI && backStack.lastOrNull() != MetaAiKey) backStack.add(MetaAiKey) if (place == MainActivity.OPEN_ONBOARDING && backStack.lastOrNull() !is OnboardingKey) backStack.add(OnboardingKey(welcome = true)) + if (place == MainActivity.OPEN_ACCOUNTS && backStack.lastOrNull() != AccountsKey) backStack.add(AccountsKey) if (place == MainActivity.OPEN_LIVE && backStack.lastOrNull() != LiveKey) { stage.homeToTop = true backStack.add(LiveKey) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/LinkPrompt.kt b/app/src/main/kotlin/com/macedonga/visor/connection/LinkPrompt.kt new file mode 100644 index 00000000..bac39146 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/LinkPrompt.kt @@ -0,0 +1,70 @@ +package com.macedonga.visor.connection + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.content.Context +import android.content.Intent +import com.macedonga.visor.MainActivity +import com.macedonga.visor.R +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * svc70 type 28, TriggerProviderLinking {1 provider, 2 requestId, 3 source, 4 scenario (0 connect, 1 reconnect)}: a + * Display app's "Open on phone". As the Meta app does, the phone posts a notification that opens its linking page, and + * answers type 29 {1 requestId, 2 success}. + */ +@Singleton +class LinkPrompt @Inject constructor( + @ApplicationContext private val context: Context, + private val log: Provider, +) { + suspend fun trigger(request: Wire.Fields, reply: Int, connection: DataXConnection) { + val name = PROVIDERS[request.varint(1)] + val reconnect = request.varint(4) == SCENARIO_RECONNECT + val shown = name != null && show(name, reconnect) + log.get().add("Accounts: the glasses asked to link ${name ?: "provider ${request.varint(1)}"} on the phone" + if (shown) "" else "; not shown") + val answer = Wire.fieldBytes(1, request.bytes(2)) + Wire.fieldVarint(2, if (shown) 1L else 0L) + connection.send(reply, answer, messageType = MT_RESPONSE) + } + + private fun show(name: String, reconnect: Boolean): Boolean { + val manager = context.getSystemService(NotificationManager::class.java) + if (!manager.areNotificationsEnabled()) return false + manager.createNotificationChannel( + NotificationChannel(CHANNEL, context.getString(R.string.link_channel), NotificationManager.IMPORTANCE_DEFAULT), + ) + val open = PendingIntent.getActivity( + context, 2, + Intent(context, MainActivity::class.java).putExtra(MainActivity.EXTRA_OPEN, MainActivity.OPEN_ACCOUNTS).addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP), + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, + ) + val notification = Notification.Builder(context, CHANNEL) + .setSmallIcon(R.drawable.ic_stat_visor) + .setContentTitle(context.getString(if (reconnect) R.string.link_reconnect_title else R.string.link_title, name)) + .setContentText(context.getString(R.string.link_text)) + .setContentIntent(open) + .setAutoCancel(true) + .build() + manager.notify(NOTIFICATION_ID, notification) + return true + } + + companion object { + const val MT_TRIGGER = 28 + private const val MT_RESPONSE = 29 + private const val SCENARIO_RECONNECT = 1L + private const val CHANNEL = "linking" + private const val NOTIFICATION_ID = 7 + + // CommunicateProvider, as the Meta app numbers it. + private val PROVIDERS = mapOf(1L to "Messenger", 2L to "WhatsApp", 4L to "Instagram", 6L to "Facebook", 7L to "Threads") + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt index eab52331..6143f752 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt @@ -61,6 +61,7 @@ class PhoneMessages @Inject constructor( private val calls: PhoneCalls, private val control: PhoneCallControl, private val recent: RecentChats, + private val linkPrompt: LinkPrompt, ) : Responder { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Messages: failed: $e") }) @@ -73,6 +74,7 @@ class PhoneMessages @Inject constructor( val reply = DataXConnection.mirror(message.info.channelId) // The Phone app's recent calls, asked once per provider, each waiting for its answer. if (type == MT_LOAD_CALL_LOG) return calls.loadCallLog(request, reply, connection) + if (type == LinkPrompt.MT_TRIGGER) return linkPrompt.trigger(request, reply, connection) if (type !in SERVED) return log.get().add("Messages: type $type not served") // The phone's texts, and replies to a chat notification (Messenger, WhatsApp, Instagram) through its reply action. val provider = request.varint(1) diff --git a/app/src/main/res/values-it/strings_accounts.xml b/app/src/main/res/values-it/strings_accounts.xml index aef4da15..96b5791e 100644 --- a/app/src/main/res/values-it/strings_accounts.xml +++ b/app/src/main/res/values-it/strings_accounts.xml @@ -16,4 +16,8 @@ Gli occhiali lo hanno rifiutato Collegato come %1$s %1$s, disattivato per gli occhiali + Collegamento account + Collega %1$s ai tuoi occhiali + Ricollega %1$s sui tuoi occhiali + Te lo chiedono gli occhiali. Tocca per aprire Account. diff --git a/app/src/main/res/values/strings_accounts.xml b/app/src/main/res/values/strings_accounts.xml index 36c752a8..6fa5c84c 100644 --- a/app/src/main/res/values/strings_accounts.xml +++ b/app/src/main/res/values/strings_accounts.xml @@ -16,4 +16,8 @@ The glasses refused it Linked as %1$s %1$s, turned off for glasses + Account linking + Link %1$s to your glasses + Reconnect %1$s on your glasses + Your glasses asked for it. Tap to open Accounts. From 153b7daacec2b36dfe27403fbad822c4b0165c6c Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 21:12:51 +0200 Subject: [PATCH 82/90] Meta AI: the Display's answer card carries the answer as its title. The Display's log shows the card arriving and parsed as a dialog card (SimpleTemplate {title, subtitle}), which shows the title only: with the question there, the wearer saw the question and no answer. The title is now the answer; the Display shows the question from the turn itself. --- .../kotlin/com/macedonga/visor/core/assistant/Assistant.kt | 2 +- .../main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt index 1dc3e89b..37270c4e 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt @@ -1034,7 +1034,7 @@ class Assistant( preamble(l, interaction, transcript, text) _phase.value = AssistantPhase.SPEAKING val voiceId = glassesVoice() - val card = if (display()) answerCard(transcript.trim().take(80).ifEmpty { "Meta AI" }, text) else null + val card = if (display()) answerCard(text) else null val answer = netEvent(EV_VOICE_ACTION_RESPONSE, varm(interaction, transcript, text, onGlasses = true, voiceId = voiceId, card = card != null)) // An answer that asks something has the glasses listen for the reply, without the wake word. val complete = netEvent( diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt index dead841d..f7f0db9c 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt @@ -166,8 +166,9 @@ internal fun varm( * A simple card for a Display, as ev60's onDeviceUxMessage carries it: base64 of a compact AssistantTemplate * (type 0, a SimpleTemplate inside). The Display throws on a template without hideAfterMs. */ -internal fun answerCard(title: String, text: String): String { - val simple = ThriftWriter().str(1, title).str(2, text).done() +internal fun answerCard(text: String): String { + // The dialog card shows its title only, in a scrolling block: the answer. The Display shows the question itself. + val simple = ThriftWriter().str(1, text).done() val hideAfterMs = (8_000L + 60L * text.length).coerceAtMost(30_000L) val template = ThriftWriter().i32(1, 0).str(2, simple).str(9, text).i64(10, hideAfterMs).done() return java.util.Base64.getEncoder().encodeToString(template) From 1e350e08a3ddd75300e5fcd5b03fb25c9a0e7546 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 21:45:13 +0200 Subject: [PATCH 83/90] Display: grant the Wi-Fi link the Display asks for, by moving the session. WhatsApp and Instagram on the Display wait for a svc41107 link lease that needs Wi-Fi; refused, WhatsApp showed "loading chats" forever. As the Meta app's link manager does, the one session now moves to Wi-Fi while a lease holds it: the main connection itself reconnects on the Wi-Fi link, so every service, notifications and the app's state go with it (the earlier attempt ran a second session and left the app thinking the glasses were gone). The link is the network the phone is on when the glasses keep its passphrase (JoinAccessPoint with it, so they stay on that network and its internet), else a Wi-Fi Direct group of the phone's. The session comes back to Bluetooth 30 s after the last lease ends, when it does not come up on Wi-Fi in 20 s, or when it drops there. The glasses page says when the link is Wi-Fi. A debug command (WIFI_LINK) holds and lets go of a lease for testing. Not tried on the Display yet. --- .../macedonga/visor/debug/DebugReceiver.kt | 7 + .../visor/connection/LeaseService.kt | 32 ++-- .../macedonga/visor/connection/WifiLink.kt | 166 ++++++++++++++++++ .../macedonga/visor/settings/GlassesPage.kt | 1 + app/src/main/res/values-it/strings.xml | 1 + app/src/main/res/values/strings.xml | 1 + .../core/connection/GlassesConnection.kt | 45 ++++- .../visor/core/connection/GlassesWifi.kt | 8 + 8 files changed, 250 insertions(+), 11 deletions(-) create mode 100644 app/src/main/kotlin/com/macedonga/visor/connection/WifiLink.kt diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index 08977758..70f6b950 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -68,6 +68,7 @@ class DebugReceiver : BroadcastReceiver() { @Inject lateinit var internet: com.macedonga.visor.connection.InternetSwitch @Inject lateinit var developer: com.macedonga.visor.outside.DeveloperMode @Inject lateinit var texts: com.macedonga.visor.connection.PhoneMessages + @Inject lateinit var wifiLink: com.macedonga.visor.connection.WifiLink override fun onReceive(context: Context, intent: Intent) { val action = intent.action?.substringAfterLast('.').orEmpty() @@ -477,6 +478,12 @@ class DebugReceiver : BroadcastReceiver() { mcp.signIn.first { it is com.macedonga.visor.core.assistant.mcp.SignIn.Done || it is com.macedonga.visor.core.assistant.mcp.SignIn.Failed } }.toString() } + // A lease that needs Wi-Fi, held and let go as the Display's would be. + "WIFI_LINK" -> resultData = if (intent.getBooleanExtra("on", true)) { + wifiLink.hold("debug").let { "held: " + (wifiLink.refusal() ?: "moving") } + } else { + wifiLink.release("debug").let { "released" } + } "MCP_REMOVE" -> { mcp.store.remove(intent.getStringExtra("id").orEmpty()) resultData = "removed" diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt index 93d37d3b..d4a73ef8 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt @@ -13,11 +13,13 @@ import javax.inject.Singleton /** * svc41107, lease_service: the glasses ask for internet (type 1, InternetLeaseRequest) and for a link (type 4, * LinkLeaseRequest), and end a lease (type 3, no answer). Internet leases are answered as the Meta app answers them - * (2, use the glasses' own Wi-Fi); a link lease (5) on Bluetooth is granted, one that needs Wi-Fi is refused for now. + * (2, use the glasses' own Wi-Fi); a link lease (5) on Bluetooth is granted, and one that needs Wi-Fi is granted + * when the phone can move the session to Wi-Fi ([WifiLink]), refused otherwise. */ @Singleton class LeaseService @Inject constructor( private val log: Provider, + private val wifiLink: Provider, ) : Responder { // Leases the glasses hold, by lease id, so an expiry can be named in the log. private val held = ConcurrentHashMap() @@ -47,22 +49,32 @@ class LeaseService @Inject constructor( request.has(6) -> request.varint(6) >= 1 else -> null } - val reply = when (wifi) { - null -> Wire.fieldBytes(1, id) + refusal(ERROR_BAD_REQUEST, "Invalid Request") - true -> Wire.fieldBytes(1, id) + refusal(ERROR_WIFI_DIRECT_NOT_AVAILABLE, "Wi-Fi is not available for this link.") - false -> Wire.fieldBytes(1, id) + val unavailable = if (wifi == true) wifiLink.get().refusal() else null + val reply = when { + wifi == null -> Wire.fieldBytes(1, id) + refusal(ERROR_BAD_REQUEST, "Invalid Request") + unavailable != null -> Wire.fieldBytes(1, id) + refusal(ERROR_WIFI_DIRECT_NOT_AVAILABLE, "Wi-Fi is not available for this link.") + else -> Wire.fieldBytes(1, id) } log.get().add( - when (wifi) { - null -> "Lease: the glasses asked for a link with no requirement; refused" - true -> "Lease: the glasses asked for Wi-Fi; not available yet, refused" - false -> "Lease: the glasses asked for a link on Bluetooth; answered OK" + when { + wifi == null -> "Lease: the glasses asked for a link with no requirement; refused" + unavailable != null -> "Lease: the glasses asked for Wi-Fi; refused, $unavailable" + wifi == true -> "Lease: the glasses asked for Wi-Fi; answered OK" + else -> "Lease: the glasses asked for a link on Bluetooth; answered OK" }, ) if (wifi == false) held[key] = "link" connection.send(channel, reply, messageType = MT_LINK_RESPONSE) + // Answered on the Bluetooth session first: the move to Wi-Fi ends it. + if (wifi == true && unavailable == null) { + held[key] = "Wi-Fi link" + wifiLink.get().hold(key) + } + } + MT_LEASE_EXPIRATION -> { + held.remove(key)?.let { log.get().add("Lease: a $it lease ended") } + wifiLink.get().release(key) } - MT_LEASE_EXPIRATION -> held.remove(key)?.let { log.get().add("Lease: a $it lease ended") } else -> log.get().add("Lease: message type ${message.info.messageType} is not handled") } } diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/WifiLink.kt b/app/src/main/kotlin/com/macedonga/visor/connection/WifiLink.kt new file mode 100644 index 00000000..0ef548ef --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/WifiLink.kt @@ -0,0 +1,166 @@ +package com.macedonga.visor.connection + +import android.content.Context +import android.net.wifi.WifiManager +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.connection.GlassesLink +import com.macedonga.visor.core.connection.GlassesTarget +import com.macedonga.visor.core.connection.GlassesWifi +import com.macedonga.visor.core.media.BAND_2GHZ +import com.macedonga.visor.core.media.BW_20MHZ +import com.macedonga.visor.core.media.BW_UNSPECIFIED +import com.macedonga.visor.core.media.JoinResult +import com.macedonga.visor.core.media.MT_JOIN_ACCESS_POINT_RESULT +import com.macedonga.visor.core.media.PhoneGroup +import com.macedonga.visor.core.media.SVC_ATC +import com.macedonga.visor.core.media.WifiUnavailable +import com.macedonga.visor.core.media.joinAccessPoint +import com.macedonga.visor.core.media.parseJoinResult +import com.macedonga.visor.core.media.wifiDirectAllowed +import com.macedonga.visor.core.media.wifiIsOn +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.onSubscription +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withTimeoutOrNull + +/** + * The Wi-Fi link the Display asks for with a link lease (svc41107), for WhatsApp, Instagram and other heavy traffic. + * As the Meta app's link manager does, the one session moves to Wi-Fi while a lease holds it and comes back after: on + * the network the phone is on when the glasses have it saved (they stay on it, with their own internet), else on a + * Wi-Fi Direct group of the phone's. Back on Bluetooth when no lease has held Wi-Fi for [IDLE_MS], or when the Wi-Fi + * session fails. + */ +@Singleton +class WifiLink @Inject constructor( + @ApplicationContext private val context: Context, + private val connection: GlassesConnection, + private val networks: GlassesWifi, + private val log: Provider, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default + CoroutineExceptionHandler { _, e -> say("failed: $e") }) + private val lock = Mutex() + private val leases = ConcurrentHashMap.newKeySet() + private var group: PhoneGroup? = null + private var idle: Job? = null + + /** Why the phone cannot give a Wi-Fi link now, or null when it can. */ + fun refusal(): String? = when { + !wifiIsOn(context) -> "the phone's Wi-Fi is off" + connection.state.value.link != LinkState.READY -> "the glasses are not connected" + else -> null + } + + /** A granted lease that needs Wi-Fi: the session moves there unless it is already. */ + fun hold(id: String) { + leases += id + idle?.cancel() + scope.launch { lock.withLock { if (!connection.onWifi && leases.isNotEmpty()) moveToWifi() } } + } + + /** A lease ended: with none left, back to Bluetooth after [IDLE_MS]. */ + fun release(id: String) { + if (!leases.remove(id) || leases.isNotEmpty()) return + idle?.cancel() + idle = scope.launch { + delay(IDLE_MS) + lock.withLock { if (leases.isEmpty() && connection.onWifi) back("no lease needs Wi-Fi") } + } + } + + private suspend fun moveToWifi() { + val serial = connection.state.value.target?.serial ?: return + val joined = sameNetwork() ?: phoneGroup() ?: return say("staying on Bluetooth") + say("moving the session to ${joined.address}:${joined.port}") + connection.moveToWifi(GlassesTarget(GlassesLink.Wifi(checkNotNull(joined.address), joined.port), serial)) + scope.launch { watch() } + } + + /** Back on Bluetooth if the session does not come up on Wi-Fi, or when it drops there. */ + private suspend fun watch() { + val up = withTimeoutOrNull(UP_MS) { connection.state.first { it.target?.link is GlassesLink.Wifi && it.link == LinkState.READY } } + if (up == null) return lock.withLock { if (connection.onWifi) back("the session did not come up over Wi-Fi") } + say("the session runs over Wi-Fi") + connection.state.first { it.link != LinkState.READY || !connection.onWifi } + lock.withLock { + // Live or a media import took the glasses for a Wi-Fi session of their own, and their group with it. + if (!connection.onWifi) group = null else back("the Wi-Fi session dropped") + } + } + + private suspend fun back(why: String) { + say("back to Bluetooth: $why") + leases.clear() + connection.backToBluetooth() + group?.remove() + group = null + } + + /** The glasses joined to the network the phone is on, when they keep its passphrase. */ + private suspend fun sameNetwork(): JoinResult? { + val ssid = phoneSsid() ?: return null + val key = networks.keyOf(ssid) ?: return null + return join(ssid, key, BW_UNSPECIFIED)?.also { say("the glasses are on $ssid with the phone") } + } + + /** The glasses joined to a Wi-Fi Direct group of the phone's. */ + private suspend fun phoneGroup(): JoinResult? { + if (!wifiDirectAllowed(context)) return null.also { say("Wi-Fi Direct is not allowed") } + val g = PhoneGroup(context) { say(it) } + try { + g.create(BAND_2GHZ) + } catch (unavailable: WifiUnavailable) { + return null.also { say("Wi-Fi Direct did not come up: ${unavailable.message}") } + } + group = g + return join(g.ssid, g.passphrase, BW_20MHZ) ?: null.also { + g.remove() + group = null + } + } + + // Fresh out of a group, or of a Wi-Fi restart, the glasses answer INTERNAL_ERROR for a while. + private suspend fun join(ssid: String, passphrase: String, bandwidth: Int): JoinResult? { + for (attempt in 1..JOIN_TRIES) { + if (attempt > 1) delay(RETRY_MS) + val result = withTimeoutOrNull(JOIN_MS) { + connection.serviceData + .onSubscription { connection.submit { joinAccessPoint(it, ssid, passphrase, bandwidth) } } + .first { it.serviceId == SVC_ATC && it.messageType == MT_JOIN_ACCESS_POINT_RESULT } + .let { parseJoinResult(it.payload) } + } + if (result?.ok == true) return result + say(if (result == null) "the glasses did not answer the join to $ssid" else "the glasses could not join $ssid (${result.statusName})") + } + return null + } + + @Suppress("DEPRECATION") + private fun phoneSsid(): String? = + context.getSystemService(WifiManager::class.java)?.connectionInfo?.ssid?.removeSurrounding("\"")?.takeIf { it.isNotEmpty() && it != UNKNOWN_SSID } + + private fun say(text: String) = log.get().add("Wi-Fi link: $text") + + private companion object { + const val IDLE_MS = 30_000L + const val UP_MS = 20_000L + const val JOIN_MS = 30_000L + const val RETRY_MS = 4_000L + const val JOIN_TRIES = 3 + const val UNKNOWN_SSID = "" + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt index 11bb83ea..05f2095a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt @@ -201,6 +201,7 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( RowFrame( stringResource( when { + ready && glasses.target?.link is GlassesLink.Wifi -> R.string.glasses_link_wifi ready -> if (le) R.string.glasses_link_le else R.string.glasses_link_rfcomm running && glasses.link != LinkState.DISCONNECTED && glasses.link != LinkState.LOST -> R.string.glasses_searching else -> R.string.glasses_link_off diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 3344e6df..0a81bd97 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -455,6 +455,7 @@ Collegamento Collegati via Bluetooth LE Collegati via Bluetooth + Collegati via Wi-Fi Non collegati Scollega Collega diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index f9448f64..45a91bd7 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -456,6 +456,7 @@ Connection Connected over Bluetooth LE Connected over Bluetooth + Connected over Wi-Fi Not connected Disconnect Connect diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt index 5c2f4019..bd8d0133 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt @@ -93,8 +93,47 @@ class GlassesConnection( private var holdingStatus = true private var statusTimer: Job? = null + // While a lease holds the session on Wi-Fi, the Bluetooth target it came from. + @Volatile + private var bluetooth: GlassesTarget? = null + + /** True while the session runs on Wi-Fi for a lease, with Bluetooth to come back to. */ + val onWifi: Boolean get() = bluetooth != null + /** Choose a target, open a session on it, and keep the state current until [stop]. */ fun start(scope: CoroutineScope, override: GlassesTarget? = null) { + bluetooth = null + open(scope, override) + } + + /** + * Move the session to [target], a Wi-Fi link, as the Meta app's link manager does for a lease: the glasses take one + * session at a time, so it moves rather than opening a second, and every service moves with it. + */ + suspend fun moveToWifi(target: GlassesTarget) { + val current = _state.value.target ?: return + if (bluetooth == null) bluetooth = current.takeIf { it.link is GlassesLink.Bluetooth } ?: return + reopen(target) + } + + /** Back on the Bluetooth link the session left for [moveToWifi]. */ + suspend fun backToBluetooth() { + val target = bluetooth ?: return + bluetooth = null + reopen(target) + } + + private suspend fun reopen(target: GlassesTarget) { + val scope = lastScope ?: return + if (!_running.value) return + host?.stop() + host = null + job?.cancelAndJoin() + job = null + open(scope, target) + } + + private fun open(scope: CoroutineScope, override: GlassesTarget?) { lastScope = scope lastOverride = override _running.value = true @@ -125,6 +164,9 @@ class GlassesConnection( /** Let go of the link without ending the connection, so another session can have the glasses. */ suspend fun pause(why: String) { + // The session that takes over runs on Wi-Fi of its own; this one comes back on Bluetooth. + bluetooth?.let { lastOverride = it } + bluetooth = null host?.stop() host = null job?.cancelAndJoin() @@ -135,7 +177,7 @@ class GlassesConnection( /** Reconnect after [pause], unless [stop] ran in between. */ fun resume() { val scope = lastScope ?: return - if (_running.value && job == null) start(scope, lastOverride) + if (_running.value && job == null) open(scope, lastOverride) } /** Ask svc28 again on the session's own status channel: the glasses answer with a fresh full snapshot. */ @@ -158,6 +200,7 @@ class GlassesConnection( * has stopped. */ suspend fun stop() { + bluetooth = null host?.stop() host = null job?.cancelAndJoin() diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt index fafd92f2..3a2edf54 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt @@ -52,6 +52,14 @@ class GlassesWifi(private val connection: GlassesConnection) { if (answer == null) null else readList(answer) } + /** The passphrase the glasses keep for [ssid] (QueryNetworksResult.Result 11), or null. A secret: never logged. */ + suspend fun keyOf(ssid: String): String? = lock.withLock { + val answer = ask(MT_QUERY_NETWORKS, ByteArray(0), MT_QUERY_NETWORKS_RESULT) ?: return null + val fields = Wire.parse(answer) + if (fields.varint(1) != 0L) return null + fields.repeatedBytes(2).map(Wire::parse).firstOrNull { it.bytes(1).decodeToString() == ssid }?.takeIf { it.has(11) }?.bytes(11)?.decodeToString() + } + private fun readList(answer: ByteArray): List? { val fields = Wire.parse(answer) if (fields.varint(1) != 0L) { From ed0e35c806c002bddf9448b87da99ebb84afb4b0 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Fri, 23 Oct 2026 22:05:06 +0200 Subject: [PATCH 84/90] Display: keep a session moved to Wi-Fi from being moved again. The service moves a session that is not on LE onto LE, and a session on Wi-Fi is not on LE either: within a minute it would have reopened the session on the same Wi-Fi link and lost the Bluetooth one to come back to. That move and the tile's retry now leave Wi-Fi alone, the retry going back on Bluetooth. The Wi-Fi link is refused while the Live or a media import uses Wi-Fi, and when the session is reopened on Bluetooth some other way the link's own Wi-Fi Direct group is removed, unless the Live or an import paused the session and took the glasses on a group of its own. --- .../visor/connection/GlassesService.kt | 6 +++-- .../macedonga/visor/connection/WifiLink.kt | 23 +++++++++++++++++-- .../core/connection/GlassesConnection.kt | 7 ++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt index 07485e16..8bfbd32b 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt @@ -191,7 +191,8 @@ class GlassesService : Service() { if (live.running.value || media.importing.value) return scope.launch { connection.stop() - connection.start(scope, state.target) + // Tried again on Bluetooth: a Wi-Fi link that dropped is not one to wait for. + connection.start(scope, state.target?.takeIf { it.link is GlassesLink.Bluetooth }) } } @@ -219,7 +220,8 @@ class GlassesService : Service() { scope.launch { var last = 0L combine(connection.state, kind.overLe, keeper.connected) { state, le, held -> - held && !le && state.link == LinkState.READY && state.device?.streamingActive != true + // A session moved to Wi-Fi for a lease is not on LE either, and stays where it is. + held && !le && state.link == LinkState.READY && state.device?.streamingActive != true && state.target?.link !is GlassesLink.Wifi } .distinctUntilChanged() .collect { move -> diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/WifiLink.kt b/app/src/main/kotlin/com/macedonga/visor/connection/WifiLink.kt index 0ef548ef..d729ddd2 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/WifiLink.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/WifiLink.kt @@ -10,7 +10,9 @@ import com.macedonga.visor.core.media.BAND_2GHZ import com.macedonga.visor.core.media.BW_20MHZ import com.macedonga.visor.core.media.BW_UNSPECIFIED import com.macedonga.visor.core.media.JoinResult +import com.macedonga.visor.core.media.LiveStream import com.macedonga.visor.core.media.MT_JOIN_ACCESS_POINT_RESULT +import com.macedonga.visor.core.media.MediaImport import com.macedonga.visor.core.media.PhoneGroup import com.macedonga.visor.core.media.SVC_ATC import com.macedonga.visor.core.media.WifiUnavailable @@ -50,6 +52,8 @@ class WifiLink @Inject constructor( @ApplicationContext private val context: Context, private val connection: GlassesConnection, private val networks: GlassesWifi, + private val live: Provider, + private val media: Provider, private val log: Provider, ) { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default + CoroutineExceptionHandler { _, e -> say("failed: $e") }) @@ -62,6 +66,8 @@ class WifiLink @Inject constructor( fun refusal(): String? = when { !wifiIsOn(context) -> "the phone's Wi-Fi is off" connection.state.value.link != LinkState.READY -> "the glasses are not connected" + live.get().running.value -> "the Live view is using Wi-Fi" + media.get().importing.value -> "a media import is using Wi-Fi" else -> null } @@ -97,11 +103,24 @@ class WifiLink @Inject constructor( say("the session runs over Wi-Fi") connection.state.first { it.link != LinkState.READY || !connection.onWifi } lock.withLock { - // Live or a media import took the glasses for a Wi-Fi session of their own, and their group with it. - if (!connection.onWifi) group = null else back("the Wi-Fi session dropped") + when { + connection.onWifi -> back("the Wi-Fi session dropped") + // The Live or a media import took the glasses for a Wi-Fi session of their own, and the group with it. + connection.paused -> forget() + // The session was opened again on Bluetooth some other way. + else -> { + group?.remove() + forget() + } + } } } + private fun forget() { + group = null + leases.clear() + } + private suspend fun back(why: String) { say("back to Bluetooth: $why") leases.clear() diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt index bd8d0133..1dd90ab0 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt @@ -100,6 +100,11 @@ class GlassesConnection( /** True while the session runs on Wi-Fi for a lease, with Bluetooth to come back to. */ val onWifi: Boolean get() = bluetooth != null + /** True from [pause] until the session opens again: the Live or a media import has the glasses on Wi-Fi of its own. */ + @Volatile + var paused = false + private set + /** Choose a target, open a session on it, and keep the state current until [stop]. */ fun start(scope: CoroutineScope, override: GlassesTarget? = null) { bluetooth = null @@ -134,6 +139,7 @@ class GlassesConnection( } private fun open(scope: CoroutineScope, override: GlassesTarget?) { + paused = false lastScope = scope lastOverride = override _running.value = true @@ -167,6 +173,7 @@ class GlassesConnection( // The session that takes over runs on Wi-Fi of its own; this one comes back on Bluetooth. bluetooth?.let { lastOverride = it } bluetooth = null + paused = true host?.stop() host = null job?.cancelAndJoin() From 914b9b3311e73185e8179ad0d8cd97d36248e817 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Sat, 24 Oct 2026 10:31:45 +0200 Subject: [PATCH 85/90] Landscape mode turns on Meta's landscape switch too (#35). The glasses shoot landscape only with their own landscape_mode_enabled and Meta's mwa_ios_landscape_mode.enabled both on, and Meta's server had turned its switch off: the Landscape mode switch did nothing. With the switch on, Visor now writes Meta's switch on in the glasses' configuration (reading it first if this run has not); the camera reads it at boot, so the glasses need a restart after. Not tried on the glasses. --- .../visor/connection/GlassesService.kt | 1 + .../com/macedonga/visor/settings/Camera.kt | 49 +++++++++++++++++++ .../visor/core/settings/settings_ui.json | 4 +- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt index 8bfbd32b..4a6be475 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt @@ -145,6 +145,7 @@ class GlassesService : Service() { com.macedonga.visor.language.keepAssistantConfig(scope, connection, glassesConfig, media, glassesSettings, assistantConfig, log) developer.keep(scope) com.macedonga.visor.settings.keepCameraInLine(scope, glassesSettings, assistantConfig, log) + com.macedonga.visor.settings.keepLandscapeFlag(scope, connection, glassesSettings, glassesConfig, media, log) notifications = scope.launch { var shown: Any? = null combine(connection.state, liveNow(), outside.glimpse, ::Triple).collect { (state, now, glimpse) -> diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt b/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt index 11e1d78d..f5c31e46 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt @@ -1,18 +1,32 @@ package com.macedonga.visor.settings import com.macedonga.visor.core.assistant.AssistantConfigStore +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.media.GlassesConfig +import com.macedonga.visor.core.media.MediaImport +import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.core.settings.GlassesSettings +import com.macedonga.visor.core.settings.Kind import com.macedonga.visor.core.settings.same import com.macedonga.visor.log.EventLog import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.collectLatest import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.map import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull /** The glasses' own camera switches for Meta AI. With either off, they refuse every question about what the wearer sees. */ val GLASSES_CAMERA_KEYS = setOf("cloud_ai_understanding_enabled", "camera_use_enabled") +/** Meta's server switch for landscape capture. The glasses shoot landscape only with it and their own switch both on. */ +const val LANDSCAPE_FLAG = "mwa_ios_landscape_mode.enabled" + +private const val LANDSCAPE_MODE = "landscape_mode_enabled" +private const val READ_MS = 5 * 60_000L + /** Visor's "Use the camera" is the one switch: whenever it changes, and after every read, the glasses' two follow it. */ fun keepCameraInLine(scope: CoroutineScope, settings: GlassesSettings, store: AssistantConfigStore, log: EventLog) { scope.launch { @@ -31,3 +45,38 @@ fun keepCameraInLine(scope: CoroutineScope, settings: GlassesSettings, store: As } } } + +/** + * With "Landscape mode" on, Meta's landscape switch in the glasses' configuration goes on too: Meta's server turns it + * off, and the glasses' own switch then does nothing. The camera reads it at boot, so it takes a restart. + */ +fun keepLandscapeFlag( + scope: CoroutineScope, + connection: GlassesConnection, + settings: GlassesSettings, + config: GlassesConfig, + media: MediaImport, + log: EventLog, +) { + scope.launch { + combine( + connection.state.map { it.link == LinkState.READY }.distinctUntilChanged(), + settings.state.map { s -> same(Kind.BOOL, s.pending[LANDSCAPE_MODE] ?: s.stored[LANDSCAPE_MODE], "true") }.distinctUntilChanged(), + ) { ready, on -> ready && on }.collectLatest { wanted -> + if (!wanted) return@collectLatest + if (config.state.value.params.isEmpty()) { + // The configuration comes in a bug report: never on top of photos to bring over. + combine(media.importing, media.captures) { importing, captures -> !importing && captures.none { it.isNew && it.isMedia } } + .first { it } + log.add("Camera: reading the glasses' configuration for landscape mode") + config.read() + withTimeoutOrNull(READ_MS) { config.state.first { it.params.isNotEmpty() } } ?: return@collectLatest + } + val state = config.state.first { it.busy.isEmpty() } + if (LANDSCAPE_FLAG !in state.params || state.values[LANDSCAPE_FLAG] == true) return@collectLatest + log.add("Camera: turning on Meta's landscape switch for landscape mode") + val why = config.send(mapOf(LANDSCAPE_FLAG to true)) + log.add(if (why == null) "Camera: landscape mode is on; restart the glasses to use it" else "Camera: Meta's landscape switch did not go: $why") + } + } +} diff --git a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json index ffd08bd3..e0fafc77 100644 --- a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json +++ b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json @@ -10790,7 +10790,7 @@ { "key":"landscape_mode_enabled", "label":"Landscape mode", -"note":"Capture photos and videos in landscape orientation.", +"note":"Capture photos and videos in landscape orientation. Turning it on also turns on Meta's landscape switch in the glasses' configuration; restart the glasses after.", "group":"Camera", "invert":false, "user_facing":true, @@ -10799,7 +10799,7 @@ "type":"toggle" }, "label_it":"Modalità orizzontale", -"note_it":"Acquisisci foto e video in modalità orizzontale.", +"note_it":"Acquisisci foto e video in modalità orizzontale. Attivandola si attiva anche l'interruttore orizzontale di Meta nella configurazione degli occhiali; poi riavvia gli occhiali.", "not_products":[ 258, 259 From 03512ed47c8991bac0a6729468c35afb0fd98733 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Sat, 24 Oct 2026 11:00:10 +0200 Subject: [PATCH 86/90] Landscape mode sets the media orientation the camera follows (#35). On firmware 7238 the camera takes the orientation from capture_aspect_ratio (1 landscape, 0 portrait), which the glasses keep at portrait while only landscape_mode_enabled is on: their log shows the camera app watching it, and a photo taken with the switch on, Meta's flag on and two restarts still came out tagged portrait. The switch now sets capture_aspect_ratio to landscape, and back to portrait when it is turned off; Media orientation keeps its own row. The settings table gets the key's namespace and default (individual_device_settings, 0). Meta's configuration flag still goes on for the older path. Not tried on the glasses yet. --- .../visor/connection/GlassesService.kt | 2 +- .../com/macedonga/visor/settings/Camera.kt | 28 ++++++++++++++----- .../visor/core/settings/settings.tsv | 2 +- .../visor/core/settings/settings_ui.json | 4 +-- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt index 4a6be475..c9f46d78 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt @@ -145,7 +145,7 @@ class GlassesService : Service() { com.macedonga.visor.language.keepAssistantConfig(scope, connection, glassesConfig, media, glassesSettings, assistantConfig, log) developer.keep(scope) com.macedonga.visor.settings.keepCameraInLine(scope, glassesSettings, assistantConfig, log) - com.macedonga.visor.settings.keepLandscapeFlag(scope, connection, glassesSettings, glassesConfig, media, log) + com.macedonga.visor.settings.keepLandscape(scope, connection, glassesSettings, glassesConfig, media, log) notifications = scope.launch { var shown: Any? = null combine(connection.state, liveNow(), outside.glimpse, ::Triple).collect { (state, now, glimpse) -> diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt b/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt index f5c31e46..2773578c 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt @@ -21,10 +21,12 @@ import kotlinx.coroutines.withTimeoutOrNull /** The glasses' own camera switches for Meta AI. With either off, they refuse every question about what the wearer sees. */ val GLASSES_CAMERA_KEYS = setOf("cloud_ai_understanding_enabled", "camera_use_enabled") -/** Meta's server switch for landscape capture. The glasses shoot landscape only with it and their own switch both on. */ +/** Meta's server switch for landscape capture, which the older landscape path needs with landscape_mode_enabled. */ const val LANDSCAPE_FLAG = "mwa_ios_landscape_mode.enabled" private const val LANDSCAPE_MODE = "landscape_mode_enabled" +// What the camera follows since 7238: 1 landscape, 0 portrait. +private const val ASPECT = "capture_aspect_ratio" private const val READ_MS = 5 * 60_000L /** Visor's "Use the camera" is the one switch: whenever it changes, and after every read, the glasses' two follow it. */ @@ -47,10 +49,12 @@ fun keepCameraInLine(scope: CoroutineScope, settings: GlassesSettings, store: As } /** - * With "Landscape mode" on, Meta's landscape switch in the glasses' configuration goes on too: Meta's server turns it - * off, and the glasses' own switch then does nothing. The camera reads it at boot, so it takes a restart. + * "Landscape mode" sets landscape the way the firmware reads it now. The 7238 camera takes the orientation from + * capture_aspect_ratio, which stays portrait with only landscape_mode_enabled on: the switch sets it to landscape, and + * back to portrait when turned off. Meta's landscape switch in the glasses' configuration goes on too, for the older + * path; Meta's server turns it off. The camera reads that one at boot. */ -fun keepLandscapeFlag( +fun keepLandscape( scope: CoroutineScope, connection: GlassesConnection, settings: GlassesSettings, @@ -59,11 +63,21 @@ fun keepLandscapeFlag( log: EventLog, ) { scope.launch { + var was: Boolean? = null combine( connection.state.map { it.link == LinkState.READY }.distinctUntilChanged(), settings.state.map { s -> same(Kind.BOOL, s.pending[LANDSCAPE_MODE] ?: s.stored[LANDSCAPE_MODE], "true") }.distinctUntilChanged(), - ) { ready, on -> ready && on }.collectLatest { wanted -> - if (!wanted) return@collectLatest + ) { ready, on -> on.takeIf { ready } }.collectLatest { on -> + on ?: return@collectLatest + val turnedOff = was == true && !on + was = on + val aspect = settings.state.value.pending[ASPECT] ?: settings.value(ASPECT) + // Off at start leaves the orientation as it is: it has its own row, Media orientation. + val want = if (on) "1" else "0".takeIf { turnedOff } + if (want != null && !same(Kind.INT, aspect, want) && settings.write(ASPECT, want)) { + log.add("Camera: media orientation follows Landscape mode, ${if (on) "landscape" else "portrait"}") + } + if (!on) return@collectLatest if (config.state.value.params.isEmpty()) { // The configuration comes in a bug report: never on top of photos to bring over. combine(media.importing, media.captures) { importing, captures -> !importing && captures.none { it.isNew && it.isMedia } } @@ -76,7 +90,7 @@ fun keepLandscapeFlag( if (LANDSCAPE_FLAG !in state.params || state.values[LANDSCAPE_FLAG] == true) return@collectLatest log.add("Camera: turning on Meta's landscape switch for landscape mode") val why = config.send(mapOf(LANDSCAPE_FLAG to true)) - log.add(if (why == null) "Camera: landscape mode is on; restart the glasses to use it" else "Camera: Meta's landscape switch did not go: $why") + log.add(if (why == null) "Camera: Meta's landscape switch is on; it applies once the glasses restart" else "Camera: Meta's landscape switch did not go: $why") } } } diff --git a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings.tsv b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings.tsv index 4ad7e761..bb32698a 100644 --- a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings.tsv +++ b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings.tsv @@ -83,7 +83,7 @@ CAPTOUCH2D_2FINGER_SWIPE_REAR str KEYCODE_VOLUME_DOWN individual_device_setting CAPTOUCH2D_2FINGER_SWIPE_UP str KEYCODE_UNKNOWN individual_device_settings CAPTOUCH2D_2FINGER_TAP_RELEASE str KEYCODE_BACK individual_device_settings captouch_enabled 148 bool true individual_device_settings 1 -capture_aspect_ratio 252 int +capture_aspect_ratio 252 int 0 individual_device_settings capture_auto_review 118 bool true individual_device_settings capture_button_assistant_shortcut 50 str individual_device_settings capture_button_double_press 57 int 0 individual_device_settings 1 diff --git a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json index e0fafc77..42c14349 100644 --- a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json +++ b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json @@ -10790,7 +10790,7 @@ { "key":"landscape_mode_enabled", "label":"Landscape mode", -"note":"Capture photos and videos in landscape orientation. Turning it on also turns on Meta's landscape switch in the glasses' configuration; restart the glasses after.", +"note":"Capture photos and videos in landscape orientation. Turning it on also sets Media orientation to Landscape and turns on Meta's landscape switch in the glasses' configuration.", "group":"Camera", "invert":false, "user_facing":true, @@ -10799,7 +10799,7 @@ "type":"toggle" }, "label_it":"Modalità orizzontale", -"note_it":"Acquisisci foto e video in modalità orizzontale. Attivandola si attiva anche l'interruttore orizzontale di Meta nella configurazione degli occhiali; poi riavvia gli occhiali.", +"note_it":"Acquisisci foto e video in modalità orizzontale. Attivandola si imposta anche l'orientamento su Orizzontale e si attiva l'interruttore orizzontale di Meta nella configurazione degli occhiali.", "not_products":[ 258, 259 From 43842dc3f6cdfb06255d706e9efb7e5d5e0296eb Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Sat, 24 Oct 2026 11:55:35 +0200 Subject: [PATCH 87/90] Landscape that works: the phone crops the captures the glasses mark landscape (#35). With Media orientation at landscape (capture_aspect_ratio 1) the glasses still take the whole portrait frame and only mark the capture, a byte in its metadata (slot 34 of the capture's info): the Meta app crops it on the phone (LandscapePhotoCropEngine for photos, its native import for videos). Visor now reads that mark and, as it brings a capture over, keeps the middle 4:3 band across the whole width, which is the Meta app's crop when it has nothing to follow: a photo's JPEG is cut from the developed still, a video is re-encoded once with Media3 Transformer (HEVC, the sound as it was), and a video this phone cannot crop is kept whole. Media orientation is now the one control: Landscape mode leaves the camera page, and it and Meta's landscape flag follow Media orientation. Measured on the Gen 2 (7238): photos and a video taken at landscape came marked, a test video came out 2448x1836 and upright. --- .../macedonga/visor/debug/DebugReceiver.kt | 2 +- .../com/macedonga/visor/settings/Camera.kt | 30 +++--- .../macedonga/visor/settings/TopicScreen.kt | 2 +- .../com/macedonga/visor/settings/Topics.kt | 2 +- core/media/build.gradle.kts | 3 + .../macedonga/visor/core/media/Assembly.kt | 38 +++++-- .../macedonga/visor/core/media/Captures.kt | 15 ++- .../macedonga/visor/core/media/Landscape.kt | 100 ++++++++++++++++++ .../visor/core/settings/settings_ui.json | 10 +- gradle/libs.versions.toml | 4 + 10 files changed, 172 insertions(+), 34 deletions(-) create mode 100644 core/media/src/main/kotlin/com/macedonga/visor/core/media/Landscape.kt diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index 70f6b950..abc4d9c0 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -301,7 +301,7 @@ class DebugReceiver : BroadcastReceiver() { } } "MEDIA" -> resultData = media.captures.value.joinToString("\n") { c -> - "${c.handle} kind=${c.kindCode} ${c.takenAtMillis} " + c.assets.joinToString(" ") { "${it.id}:0x${it.typeCode.toString(16)}:${it.size}" } + "${c.handle} kind=${c.kindCode} ${c.takenAtMillis}${if (c.landscape) " landscape" else ""} " + c.assets.joinToString(" ") { "${it.id}:0x${it.typeCode.toString(16)}:${it.size}" } }.ifEmpty { "no captures listed; open Media once or wait for the list" } "IMPORT" -> { val handles = intent.getStringExtra("handle")?.split(",").orEmpty() diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt b/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt index 2773578c..8895c65d 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt @@ -25,7 +25,7 @@ val GLASSES_CAMERA_KEYS = setOf("cloud_ai_understanding_enabled", "camera_use_en const val LANDSCAPE_FLAG = "mwa_ios_landscape_mode.enabled" private const val LANDSCAPE_MODE = "landscape_mode_enabled" -// What the camera follows since 7238: 1 landscape, 0 portrait. +// What the camera follows and marks each capture with: 1 landscape, 0 portrait. private const val ASPECT = "capture_aspect_ratio" private const val READ_MS = 5 * 60_000L @@ -49,10 +49,10 @@ fun keepCameraInLine(scope: CoroutineScope, settings: GlassesSettings, store: As } /** - * "Landscape mode" sets landscape the way the firmware reads it now. The 7238 camera takes the orientation from - * capture_aspect_ratio, which stays portrait with only landscape_mode_enabled on: the switch sets it to landscape, and - * back to portrait when turned off. Meta's landscape switch in the glasses' configuration goes on too, for the older - * path; Meta's server turns it off. The camera reads that one at boot. + * Media orientation is the one landscape control. The camera follows capture_aspect_ratio and marks each capture with + * it, still taking the whole portrait frame; Visor crops the landscape ones as it brings them over. The older + * landscape_mode_enabled follows it, and with landscape Meta's landscape switch in the glasses' configuration goes on + * too, which Meta's server turns off. */ fun keepLandscape( scope: CoroutineScope, @@ -63,32 +63,30 @@ fun keepLandscape( log: EventLog, ) { scope.launch { - var was: Boolean? = null combine( connection.state.map { it.link == LinkState.READY }.distinctUntilChanged(), - settings.state.map { s -> same(Kind.BOOL, s.pending[LANDSCAPE_MODE] ?: s.stored[LANDSCAPE_MODE], "true") }.distinctUntilChanged(), + settings.state.map { s -> + // Unknown until the first read; never stored, the camera takes portrait. + if (s.readAt == null) null else same(Kind.INT, s.pending[ASPECT] ?: s.stored[ASPECT] ?: settings.catalogue[ASPECT]?.default, "1") + }.distinctUntilChanged(), ) { ready, on -> on.takeIf { ready } }.collectLatest { on -> on ?: return@collectLatest - val turnedOff = was == true && !on - was = on - val aspect = settings.state.value.pending[ASPECT] ?: settings.value(ASPECT) - // Off at start leaves the orientation as it is: it has its own row, Media orientation. - val want = if (on) "1" else "0".takeIf { turnedOff } - if (want != null && !same(Kind.INT, aspect, want) && settings.write(ASPECT, want)) { - log.add("Camera: media orientation follows Landscape mode, ${if (on) "landscape" else "portrait"}") + val legacy = settings.state.value.pending[LANDSCAPE_MODE] ?: settings.value(LANDSCAPE_MODE) + if (!same(Kind.BOOL, legacy, on.toString()) && settings.write(LANDSCAPE_MODE, on.toString())) { + log.add("Camera: landscape mode follows Media orientation, ${if (on) "on" else "off"}") } if (!on) return@collectLatest if (config.state.value.params.isEmpty()) { // The configuration comes in a bug report: never on top of photos to bring over. combine(media.importing, media.captures) { importing, captures -> !importing && captures.none { it.isNew && it.isMedia } } .first { it } - log.add("Camera: reading the glasses' configuration for landscape mode") + log.add("Camera: reading the glasses' configuration for landscape") config.read() withTimeoutOrNull(READ_MS) { config.state.first { it.params.isNotEmpty() } } ?: return@collectLatest } val state = config.state.first { it.busy.isEmpty() } if (LANDSCAPE_FLAG !in state.params || state.values[LANDSCAPE_FLAG] == true) return@collectLatest - log.add("Camera: turning on Meta's landscape switch for landscape mode") + log.add("Camera: turning on Meta's landscape switch") val why = config.send(mapOf(LANDSCAPE_FLAG to true)) log.add(if (why == null) "Camera: Meta's landscape switch is on; it applies once the glasses restart" else "Camera: Meta's landscape switch did not go: $why") } diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/TopicScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/TopicScreen.kt index f879a7f5..e936a933 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/TopicScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/TopicScreen.kt @@ -61,7 +61,7 @@ private fun sectionsOf(topic: Topic): List>> = when (top ) Topic.CAMERA -> listOf( R.string.section_video to listOf( - "resolution_and_framerate", "eis_mode", "video_duration_ms", "capture_aspect_ratio", "landscape_mode_enabled", "motion_blur_reduction_enabled", + "resolution_and_framerate", "eis_mode", "video_duration_ms", "capture_aspect_ratio", "motion_blur_reduction_enabled", ), R.string.section_picture to listOf( "quick_clips_photo_capture_enabled", "media_auto_leveling_enabled", "media_occlusion_removal_enabled", "occlusion_detection_enabled", "capture_auto_review", diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt b/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt index b3c5535b..96564acf 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt @@ -51,7 +51,7 @@ enum class Topic(val title: Int, val icon: ImageVector, val keys: List = CAMERA( R.string.topic_camera, VisorIcons.Camera, listOf( - "resolution_and_framerate", "video_duration_ms", "eis_mode", "capture_aspect_ratio", "landscape_mode_enabled", + "resolution_and_framerate", "video_duration_ms", "eis_mode", "capture_aspect_ratio", "quick_clips_photo_capture_enabled", "media_auto_leveling_enabled", "media_occlusion_removal_enabled", "capture_auto_review", "audio_playback_during_capture_enabled", "music_concurrency_enabled", "motion_blur_reduction_enabled", "occlusion_detection_enabled", "audio_noise_reduction_enabled", diff --git a/core/media/build.gradle.kts b/core/media/build.gradle.kts index fcfa407b..8a1fff38 100644 --- a/core/media/build.gradle.kts +++ b/core/media/build.gradle.kts @@ -8,4 +8,7 @@ android { dependencies { api(project(":core:connection")) + implementation(libs.androidx.media3.transformer) + implementation(libs.androidx.media3.effect) + implementation(libs.androidx.media3.common) } diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Assembly.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Assembly.kt index 228be241..93eb615c 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Assembly.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Assembly.kt @@ -46,7 +46,10 @@ class Assembly( val name = partName(capture, assetId) ?: return val dir = File(staging, capture.handle).apply { mkdirs() } file.copyTo(File(dir, name), overwrite = true) - File(dir, META).writeText(JSONObject().put("type", capture.type.name).put("takenAt", capture.takenAtMillis ?: System.currentTimeMillis()).toString()) + File(dir, META).writeText( + JSONObject().put("type", capture.type.name).put("takenAt", capture.takenAtMillis ?: System.currentTimeMillis()) + .put("landscape", capture.landscape).toString(), + ) if (capture.wanted.mapNotNull { partName(capture, it.id) }.any { !File(dir, it).exists() }) return queue.trySend { safely(dir) } } @@ -85,19 +88,25 @@ class Assembly( val handle = dir.name val type = CaptureType.valueOf(meta.getString("type")) val takenAt = meta.getLong("takenAt") + val landscape = meta.optBoolean("landscape") val place = places.near(takenAt) val keep = index.originals(handle).apply { mkdirs() } val main = File(dir, MAIN) var format = if (type == CaptureType.VIDEO) Format.MP4 else Format.JPEG val kept = when (type) { CaptureType.VIDEO -> { - val (uri, where) = insert(format, takenAt) { out -> main.inputStream().use { it.copyTo(out) } } - say("Saved a video: $where") + val band = if (landscape) landscapeVideo(main, handle) else null + val (uri, where) = try { + insert(format, takenAt) { out -> (band ?: main).inputStream().use { it.copyTo(out) } } + } finally { + band?.delete() + } + say(if (band != null) "Saved a landscape video: $where" else "Saved a video: $where") Kept(handle, uri.toString(), type, takenAt, lat = place?.lat, lon = place?.lon) } else -> { main.copyTo(File(keep, STILL), overwrite = true) - runCatching { developed(dir, handle, type, takenAt, place) }.getOrElse { error -> + runCatching { developed(dir, handle, type, takenAt, place, landscape) }.getOrElse { error -> // A photo this phone cannot develop still goes to the gallery, as the glasses' own HEIC. say("Could not develop ${handle.take(8)} (${error.message}): saving the HEIC as it came") format = Format.HEIC @@ -122,12 +131,14 @@ class Assembly( } /** A photo or live photo as the gallery keeps it: a JPEG, with the clip after it for a live photo. */ - private fun developed(dir: File, handle: String, type: CaptureType, takenAt: Long, place: Place?): Kept { + private fun developed(dir: File, handle: String, type: CaptureType, takenAt: Long, place: Place?, landscape: Boolean): Kept { val main = File(dir, MAIN) - val bitmap = decodeStill(main) + val whole = decodeStill(main) val motion = File(dir, MOTION).takeIf { type == CaptureType.LIVE && it.exists() } val frames = readFrames(File(dir, FRAMES)).orEmpty() - val key = motion?.let { matchFrame(it, bitmap) } + // The clip is matched against the whole frame, which it shows. + val key = motion?.let { matchFrame(it, whole) } + val bitmap = if (landscape) Landscape.crop(whole).also { if (it !== whole) whole.recycle() } else whole val jpeg = try { jpegWithExif(bitmap, main, takenAt, place, File(scratch, "$handle.jpg")) } finally { @@ -142,13 +153,24 @@ class Assembly( } val bytes = if (motion != null) motionPhoto(jpeg, motion.readBytes(), keyUs) else jpeg val (uri, where) = insert(Format.JPEG, takenAt) { it.write(bytes) } - say(if (motion != null) "Saved a live photo: $where" else "Saved a photo: $where") + val what = if (motion != null) "live photo" else "photo" + say(if (landscape) "Saved a landscape $what: $where" else "Saved a $what: $where") return Kept( handle, uri.toString(), if (motion != null) CaptureType.LIVE else CaptureType.PHOTO, takenAt, clipBytes = motion?.length() ?: 0, keyFrameUs = keyUs, shutterNs = shutterNs, lat = place?.lat, lon = place?.lon, ) } + /** A landscape video's 4:3 band, or null to keep it whole when this phone cannot make one. */ + private fun landscapeVideo(main: File, handle: String): File? { + val band = File(scratch, "$handle.mp4").apply { parentFile?.mkdirs() } + return runCatching { Landscape.crop(context, main, band) }.map { band }.getOrElse { + say("Could not crop ${handle.take(8)} to landscape (${it.message}): saving it whole") + band.delete() + null + } + } + private enum class Format(val ext: String, val mime: String, val video: Boolean) { JPEG("jpg", "image/jpeg", false), HEIC("heic", "image/heic", false), diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt index e552a1d3..0ff8e337 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt @@ -44,7 +44,14 @@ enum class CaptureType { PHOTO, LIVE, VIDEO, OTHER } * One entry of the glasses' list. A photo carries a thumbnail, the HEIC still and two sensor logs; a live photo * (kind 0) carries a short HEVC clip as well; a video carries the video, one poster still and the logs. */ -data class Capture(val handle: String, val kindCode: Long, val assets: List, val timestampNs: Long) { +data class Capture( + val handle: String, + val kindCode: Long, + val assets: List, + val timestampNs: Long, + // Taken with the media orientation at landscape: the glasses keep the portrait frame, the phone crops it. + val landscape: Boolean = false, +) { private val stills: List get() = assets.filter { it.kind == KIND_STILL }.sortedBy { it.size } /** The full-resolution still. A video's only still is its poster, not a photo. */ @@ -119,7 +126,9 @@ fun parseCapture(payload: ByteArray): Capture? { } val meta = f.field(info, 12)?.let(f::target)?.takeIf { f.isTable(it.toLong()) } val timestamp = meta?.let { f.number(f.field(it, 4), wide = true) } ?: 0L - return Capture(handle, f.number(f.field(info, 8)), assets, timestamp) + // The capture's aspect ratio, a byte: 0 portrait 3:4, 1 landscape 4:3. + val landscape = meta?.let { f.byte(f.field(it, 34)) } == 1 + return Capture(handle, f.number(f.field(info, 8)), assets, timestamp, landscape) } /** A flatbuffer read by its own vtables. Port of capture_list.py's `_Buf`. */ @@ -183,6 +192,8 @@ private class Flat(private val b: ByteArray) { } } + fun byte(fp: Int?): Int = if (fp != null && fits(fp.toLong(), 1)) b[fp].toInt() and 0xFF else 0 + fun number(fp: Int?, wide: Boolean = false): Long = when { fp == null -> 0 wide && fits(fp.toLong(), 8) -> bb.getLong(fp) diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Landscape.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Landscape.kt new file mode 100644 index 00000000..8c578614 --- /dev/null +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Landscape.kt @@ -0,0 +1,100 @@ +package com.macedonga.visor.core.media + +import android.content.Context +import android.graphics.Bitmap +import android.media.MediaMetadataRetriever +import android.net.Uri +import android.os.Handler +import android.os.HandlerThread +import androidx.annotation.OptIn +import androidx.media3.common.MediaItem +import androidx.media3.common.MimeTypes +import androidx.media3.common.util.UnstableApi +import androidx.media3.effect.Crop +import androidx.media3.transformer.Composition +import androidx.media3.transformer.DefaultEncoderFactory +import androidx.media3.transformer.EditedMediaItem +import androidx.media3.transformer.Effects +import androidx.media3.transformer.ExportException +import androidx.media3.transformer.ExportResult +import androidx.media3.transformer.Transformer +import androidx.media3.transformer.VideoEncoderSettings +import java.io.File +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +/** + * Landscape captures. The glasses always take the full portrait frame and mark the capture landscape; the phone keeps + * its middle 4:3 band, the whole width across, as the Meta app's landscape crop does when it has nothing to follow. + */ +internal object Landscape { + /** The band's share of a portrait frame's height. */ + private fun share(width: Int, height: Int) = width * 3f / 4f / height + + fun crop(bitmap: Bitmap): Bitmap { + if (bitmap.width >= bitmap.height) return bitmap + val band = bitmap.width * 3 / 4 + return Bitmap.createBitmap(bitmap, 0, (bitmap.height - band) / 2, bitmap.width, band) + } + + /** [input]'s middle band, re-encoded into [output]; the sound goes across as it is. Throws when it cannot. */ + @OptIn(UnstableApi::class) + fun crop(context: Context, input: File, output: File) { + val retriever = MediaMetadataRetriever() + val (width, height, bitrate) = try { + retriever.setDataSource(input.path) + fun int(key: Int) = retriever.extractMetadata(key)?.toIntOrNull() ?: 0 + val turned = int(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION) % 180 != 0 + val w = int(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH) + val h = int(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT) + Triple(if (turned) h else w, if (turned) w else h, int(MediaMetadataRetriever.METADATA_KEY_BITRATE)) + } finally { + retriever.release() + } + check(width > 0 && height > 0) { "no video track" } + if (width >= height) { + input.copyTo(output, overwrite = true) + return + } + val share = share(width, height) + val item = EditedMediaItem.Builder(MediaItem.fromUri(Uri.fromFile(input))) + .setEffects(Effects(emptyList(), listOf(Crop(-1f, 1f, -share, share)))) + .build() + val encoders = DefaultEncoderFactory.Builder(context) + .apply { if (bitrate > 0) setRequestedVideoEncoderSettings(VideoEncoderSettings.Builder().setBitrate((bitrate * share).toInt()).build()) } + .build() + val thread = HandlerThread("landscape").apply { start() } + val done = CountDownLatch(1) + var failure: ExportException? = null + var transformer: Transformer? = null + try { + // A Transformer lives on the thread whose looper it was built with. + val handler = Handler(thread.looper) + handler.post { + transformer = Transformer.Builder(context) + .setLooper(thread.looper) + .setVideoMimeType(MimeTypes.VIDEO_H265) + .setEncoderFactory(encoders) + .addListener(object : Transformer.Listener { + override fun onCompleted(composition: Composition, exportResult: ExportResult) = done.countDown() + + override fun onError(composition: Composition, exportResult: ExportResult, exportException: ExportException) { + failure = exportException + done.countDown() + } + }) + .build() + .also { it.start(item, output.path) } + } + if (!done.await(EXPORT_MINUTES, TimeUnit.MINUTES)) { + handler.post { transformer?.cancel() } + error("the crop took over $EXPORT_MINUTES minutes") + } + failure?.let { throw it } + } finally { + thread.quitSafely() + } + } + + private const val EXPORT_MINUTES = 15L +} diff --git a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json index 42c14349..c4dc8b4c 100644 --- a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json +++ b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json @@ -4796,7 +4796,7 @@ { "key":"capture_aspect_ratio", "label":"Media orientation", -"note":"Portrait (3:4) or landscape (4:3) photos and videos.", +"note":"Portrait (3:4) or landscape (4:3) photos and videos. The glasses always take the whole portrait frame; Visor crops landscape ones to 4:3 as it brings them over.", "group":"Camera", "invert":false, "user_facing":true, @@ -4829,7 +4829,7 @@ 258, 259 ], -"note_it":"Foto e video in verticale (3:4) o in orizzontale (4:3)." +"note_it":"Foto e video in verticale (3:4) o in orizzontale (4:3). Gli occhiali scattano sempre l'intera inquadratura verticale; Visor ritaglia quelli orizzontali a 4:3 quando li importa." }, { "key":"capture_auto_review", @@ -10790,16 +10790,16 @@ { "key":"landscape_mode_enabled", "label":"Landscape mode", -"note":"Capture photos and videos in landscape orientation. Turning it on also sets Media orientation to Landscape and turns on Meta's landscape switch in the glasses' configuration.", +"note":"The older landscape switch. Visor keeps it in line with Media orientation.", "group":"Camera", "invert":false, -"user_facing":true, +"user_facing":false, "confidence":"medium", "control":{ "type":"toggle" }, "label_it":"Modalità orizzontale", -"note_it":"Acquisisci foto e video in modalità orizzontale. Attivandola si imposta anche l'orientamento su Orizzontale e si attiva l'interruttore orizzontale di Meta nella configurazione degli occhiali.", +"note_it":"Il vecchio interruttore orizzontale. Visor lo tiene allineato all'orientamento dei contenuti multimediali.", "not_products":[ 258, 259 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ed54ad74..18731efa 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -10,6 +10,7 @@ hilt = "2.60.1" hiltNavigationCompose = "1.4.0" androidxCore = "1.19.0" +media3 = "1.8.0" androidxActivity = "1.13.0" androidxLifecycle = "2.11.0" androidxWebkit = "1.17.1" @@ -53,6 +54,9 @@ minSdk = "29" [libraries] androidx-core-ktx = { module = "androidx.core:core-ktx", version.ref = "androidxCore" } +androidx-media3-transformer = { module = "androidx.media3:media3-transformer", version.ref = "media3" } +androidx-media3-effect = { module = "androidx.media3:media3-effect", version.ref = "media3" } +androidx-media3-common = { module = "androidx.media3:media3-common", version.ref = "media3" } androidx-activity-compose = { module = "androidx.activity:activity-compose", version.ref = "androidxActivity" } androidx-lifecycle-runtime-ktx = { module = "androidx.lifecycle:lifecycle-runtime-ktx", version.ref = "androidxLifecycle" } From fca6a3fa9d518a4160b4bbbf0d46b3879c8a8f2c Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Sat, 24 Oct 2026 12:40:54 +0200 Subject: [PATCH 88/90] Stabilize on the phone the videos the glasses leave shaking (#35). Firmware 7238 steadies only 1080p (its camera app watches the two-stage settings, none of the 3K or 60 fps ones), so 3K, and any video with two-stage stabilization off, came over unsteadied; the Meta app stabilizes those in its own import from the motion log each video carries. Visor now does the same as it saves a video: the gyro gives the head's rotation at each frame, the path is smoothed, and each frame is turned by the difference inside a crop margin, one GL pass in the Media3 re-encode that also cuts landscape captures. A video whose size is one of the glasses' own two-stage outputs is left alone. The capture's own stabilization (Auto, High, Medium, Low, read from its metadata) picks margin and smoothing: 15%/0.2 s, 20%/0.3 s, 15%/0.2 s, 10%/0.12 s. Calibrated against a shaky 1080p walk from this pair: the IMU's axes are Android's, a frame's moment is its mid-exposure plus 10 ms, the focal length 0.78 of the width; on the phone that walk lost 44% of its sideways shake, 66% of its vertical and 39% of its roll. Where a frame would leave the margin, the smoothed path is drawn toward the head there and smoothed again, which kept the shake removal that scaling the turn down threw away. 3K's focal length is assumed to scale with width; not yet checked on a moving 3K video. A debug action, STEADY, runs it again on a pushed file. --- .../macedonga/visor/debug/DebugReceiver.kt | 7 + .../visor/connection/GlassesService.kt | 1 + .../com/macedonga/visor/settings/Camera.kt | 12 + .../macedonga/visor/core/media/Assembly.kt | 25 +- .../macedonga/visor/core/media/Captures.kt | 7 +- .../macedonga/visor/core/media/Landscape.kt | 226 ++++++++++++---- .../macedonga/visor/core/media/MediaImport.kt | 15 + .../com/macedonga/visor/core/media/Steady.kt | 256 ++++++++++++++++++ 8 files changed, 479 insertions(+), 70 deletions(-) create mode 100644 core/media/src/main/kotlin/com/macedonga/visor/core/media/Steady.kt diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index abc4d9c0..be5add5d 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -114,6 +114,13 @@ class DebugReceiver : BroadcastReceiver() { } "LIVE_REPLAY" -> answerLater { "replay ${live.outputs.replay.heldUs / 1000} ms: ${live.outputs.saveReplay()}" } "LIVE_FRAME" -> answerLater { "frame: ${live.outputs.saveFrame()}" } + // --es file --es handle [--ez landscape] [--ei strength]: the phone's stabilization run again, + // into steady_out.mp4 beside the pushed file. + "STEADY" -> answerLater { + val input = pushed(context, intent.getStringExtra("file")) ?: return@answerLater "no such pushed file" + val out = java.io.File(input.parentFile, "steady_out.mp4") + media.finishAgain(input, intent.getStringExtra("handle").orEmpty(), out, intent.getBooleanExtra("landscape", false), intent.getIntExtra("strength", 0)) + } "BUGREPORT" -> { media.requestBugReport(if (intent.hasExtra("mode")) intent.getIntExtra("mode", 0) else null) resultData = "asked; the zip goes to Download/Visor" diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt index c9f46d78..89bc7e41 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt @@ -146,6 +146,7 @@ class GlassesService : Service() { developer.keep(scope) com.macedonga.visor.settings.keepCameraInLine(scope, glassesSettings, assistantConfig, log) com.macedonga.visor.settings.keepLandscape(scope, connection, glassesSettings, glassesConfig, media, log) + media.steadiedOnGlasses = com.macedonga.visor.settings.steadiedOnGlasses(glassesSettings) notifications = scope.launch { var shown: Any? = null combine(connection.state, liveNow(), outside.glimpse, ::Triple).collect { (state, now, glimpse) -> diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt b/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt index 8895c65d..a3c6005c 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/Camera.kt @@ -92,3 +92,15 @@ fun keepLandscape( } } } + +/** + * Whether a video this size came out of the glasses' own two-stage stabilization, from the output sizes in their + * settings (1080p, and 3K and 60 fps on firmware that has those). Any other video is steadied on the phone. + */ +fun steadiedOnGlasses(settings: GlassesSettings): (Int, Int) -> Boolean = { w, h -> + listOf("", "_3k", "_60fps").any { suffix -> + val ow = settings.value("two_stage_eis_output_width$suffix")?.trim()?.toIntOrNull() + val oh = settings.value("two_stage_eis_output_height$suffix")?.trim()?.toIntOrNull() + ow != null && oh != null && minOf(w, h) == minOf(ow, oh) && maxOf(w, h) == maxOf(ow, oh) + } +} diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Assembly.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Assembly.kt index 93eb615c..59290bd9 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Assembly.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Assembly.kt @@ -30,6 +30,8 @@ class Assembly( private val onSaved: (String) -> Unit, /** A capture that could not be saved, and why. Its pieces stay for the next try. */ private val onFailed: (String, String) -> Unit, + /** Whether a video this size came out of the glasses' own stabilization. */ + private val steadiedOnGlasses: (Int, Int) -> Boolean, ) { private val staging = File(context.filesDir, "staging") private val scratch = File(context.cacheDir, "develop") @@ -48,7 +50,7 @@ class Assembly( file.copyTo(File(dir, name), overwrite = true) File(dir, META).writeText( JSONObject().put("type", capture.type.name).put("takenAt", capture.takenAtMillis ?: System.currentTimeMillis()) - .put("landscape", capture.landscape).toString(), + .put("landscape", capture.landscape).put("steadiness", capture.steadiness).toString(), ) if (capture.wanted.mapNotNull { partName(capture, it.id) }.any { !File(dir, it).exists() }) return queue.trySend { safely(dir) } @@ -95,13 +97,15 @@ class Assembly( var format = if (type == CaptureType.VIDEO) Format.MP4 else Format.JPEG val kept = when (type) { CaptureType.VIDEO -> { - val band = if (landscape) landscapeVideo(main, handle) else null + val made = File(scratch, "$handle.mp4").apply { parentFile?.mkdirs() } + val finished = finished(dir, main, made, handle, landscape, meta.optInt("steadiness")) val (uri, where) = try { - insert(format, takenAt) { out -> (band ?: main).inputStream().use { it.copyTo(out) } } + insert(format, takenAt) { out -> (if (finished != null) made else main).inputStream().use { it.copyTo(out) } } } finally { - band?.delete() + made.delete() } - say(if (band != null) "Saved a landscape video: $where" else "Saved a video: $where") + val what = listOfNotNull("steadied".takeIf { finished?.steadied == true }, "landscape".takeIf { finished?.landscape == true }) + say("Saved a ${(what + "video").joinToString(" ")}: $where") Kept(handle, uri.toString(), type, takenAt, lat = place?.lat, lon = place?.lon) } else -> { @@ -161,15 +165,12 @@ class Assembly( ) } - /** A landscape video's 4:3 band, or null to keep it whole when this phone cannot make one. */ - private fun landscapeVideo(main: File, handle: String): File? { - val band = File(scratch, "$handle.mp4").apply { parentFile?.mkdirs() } - return runCatching { Landscape.crop(context, main, band) }.map { band }.getOrElse { - say("Could not crop ${handle.take(8)} to landscape (${it.message}): saving it whole") - band.delete() + /** The video steadied or cut to landscape into [made], or null to keep it as it came: nothing to do, or this phone could not. */ + private fun finished(dir: File, main: File, made: File, handle: String, landscape: Boolean, steadiness: Int): Finished? = + runCatching { finishVideo(context, main, made, File(dir, IMU), File(dir, FRAMES), steadiness, landscape, steadiedOnGlasses) }.getOrElse { + say("Could not steady or crop ${handle.take(8)} (${it.message}): saving it as it came") null } - } private enum class Format(val ext: String, val mime: String, val video: Boolean) { JPEG("jpg", "image/jpeg", false), diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt index 0ff8e337..b9824d38 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt @@ -51,6 +51,8 @@ data class Capture( val timestampNs: Long, // Taken with the media orientation at landscape: the glasses keep the portrait frame, the phone crops it. val landscape: Boolean = false, + // The video stabilization it was taken with: 0 Auto-select, 1 High, 2 Medium, 3 Low. + val steadiness: Int = 0, ) { private val stills: List get() = assets.filter { it.kind == KIND_STILL }.sortedBy { it.size } @@ -128,7 +130,10 @@ fun parseCapture(payload: ByteArray): Capture? { val timestamp = meta?.let { f.number(f.field(it, 4), wide = true) } ?: 0L // The capture's aspect ratio, a byte: 0 portrait 3:4, 1 landscape 4:3. val landscape = meta?.let { f.byte(f.field(it, 34)) } == 1 - return Capture(handle, f.number(f.field(info, 8)), assets, timestamp, landscape) + // The capture's video settings, a table: 4 the stabilization, 6 the resolution mode. + val video = meta?.let { f.field(it, 32) }?.let(f::target)?.takeIf { f.isTable(it.toLong()) } + val steadiness = video?.let { f.byte(f.field(it, 4)) } ?: 0 + return Capture(handle, f.number(f.field(info, 8)), assets, timestamp, landscape, steadiness) } /** A flatbuffer read by its own vtables. Port of capture_list.py's `_Buf`. */ diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Landscape.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Landscape.kt index 8c578614..119df131 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Landscape.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Landscape.kt @@ -2,15 +2,24 @@ package com.macedonga.visor.core.media import android.content.Context import android.graphics.Bitmap +import android.media.MediaExtractor +import android.media.MediaFormat import android.media.MediaMetadataRetriever import android.net.Uri +import android.opengl.GLES20 import android.os.Handler import android.os.HandlerThread import androidx.annotation.OptIn import androidx.media3.common.MediaItem import androidx.media3.common.MimeTypes +import androidx.media3.common.VideoFrameProcessingException +import androidx.media3.common.util.GlProgram +import androidx.media3.common.util.GlUtil +import androidx.media3.common.util.Size import androidx.media3.common.util.UnstableApi -import androidx.media3.effect.Crop +import androidx.media3.effect.BaseGlShaderProgram +import androidx.media3.effect.GlEffect +import androidx.media3.effect.GlShaderProgram import androidx.media3.transformer.Composition import androidx.media3.transformer.DefaultEncoderFactory import androidx.media3.transformer.EditedMediaItem @@ -28,73 +37,176 @@ import java.util.concurrent.TimeUnit * its middle 4:3 band, the whole width across, as the Meta app's landscape crop does when it has nothing to follow. */ internal object Landscape { - /** The band's share of a portrait frame's height. */ - private fun share(width: Int, height: Int) = width * 3f / 4f / height - fun crop(bitmap: Bitmap): Bitmap { if (bitmap.width >= bitmap.height) return bitmap val band = bitmap.width * 3 / 4 return Bitmap.createBitmap(bitmap, 0, (bitmap.height - band) / 2, bitmap.width, band) } +} + +/** What [finishVideo] did to a video. */ +internal class Finished(val steadied: Boolean, val landscape: Boolean) + +/** + * [input] as the gallery keeps it, in [output]: steadied on the phone when the glasses did not ([steadiedOnGlasses] says + * by its size) and its motion logs are here, cut to a 4:3 band when [landscape]. One re-encode, HEVC, the sound as it + * was. Null when there is nothing to do; throws when it cannot be done. + */ +@OptIn(UnstableApi::class) +internal fun finishVideo( + context: Context, + input: File, + output: File, + imu: File, + frames: File, + strength: Int, + landscape: Boolean, + steadiedOnGlasses: (Int, Int) -> Boolean, +): Finished? { + val retriever = MediaMetadataRetriever() + val (width, height, bitrate) = try { + retriever.setDataSource(input.path) + fun int(key: Int) = retriever.extractMetadata(key)?.toIntOrNull() ?: 0 + // The glasses write portrait frames with no rotation; one that has it is not theirs to change. + if (int(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION) != 0) return null + Triple(int(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH), int(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT), int(MediaMetadataRetriever.METADATA_KEY_BITRATE)) + } finally { + retriever.release() + } + check(width > 0 && height > 0) { "no video track" } + val cut = landscape && width < height + val steady = !steadiedOnGlasses(width, height) + if (!cut && !steady) return null + val warps = Steady.plan(width, height, frameTimes(input), readImu(imu), readFrames(frames), strength, steady, cut) ?: return null + val item = EditedMediaItem.Builder(MediaItem.fromUri(Uri.fromFile(input))) + .setEffects(Effects(emptyList(), listOf(WarpEffect(warps)))) + .build() + val share = warps.outWidth.toDouble() * warps.outHeight / (width.toDouble() * height) + val encoders = DefaultEncoderFactory.Builder(context) + .apply { if (bitrate > 0) setRequestedVideoEncoderSettings(VideoEncoderSettings.Builder().setBitrate((bitrate * share).toInt()).build()) } + .build() + val thread = HandlerThread("finish-video").apply { start() } + val done = CountDownLatch(1) + var failure: ExportException? = null + var transformer: Transformer? = null + try { + // A Transformer lives on the thread whose looper it was built with. + val handler = Handler(thread.looper) + handler.post { + transformer = Transformer.Builder(context) + .setLooper(thread.looper) + .setVideoMimeType(MimeTypes.VIDEO_H265) + .setEncoderFactory(encoders) + .addListener(object : Transformer.Listener { + override fun onCompleted(composition: Composition, exportResult: ExportResult) = done.countDown() - /** [input]'s middle band, re-encoded into [output]; the sound goes across as it is. Throws when it cannot. */ - @OptIn(UnstableApi::class) - fun crop(context: Context, input: File, output: File) { - val retriever = MediaMetadataRetriever() - val (width, height, bitrate) = try { - retriever.setDataSource(input.path) - fun int(key: Int) = retriever.extractMetadata(key)?.toIntOrNull() ?: 0 - val turned = int(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION) % 180 != 0 - val w = int(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH) - val h = int(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT) - Triple(if (turned) h else w, if (turned) w else h, int(MediaMetadataRetriever.METADATA_KEY_BITRATE)) - } finally { - retriever.release() + override fun onError(composition: Composition, exportResult: ExportResult, exportException: ExportException) { + failure = exportException + done.countDown() + } + }) + .build() + .also { it.start(item, output.path) } } - check(width > 0 && height > 0) { "no video track" } - if (width >= height) { - input.copyTo(output, overwrite = true) - return + if (!done.await(EXPORT_MINUTES, TimeUnit.MINUTES)) { + handler.post { transformer?.cancel() } + error("it took over $EXPORT_MINUTES minutes") } - val share = share(width, height) - val item = EditedMediaItem.Builder(MediaItem.fromUri(Uri.fromFile(input))) - .setEffects(Effects(emptyList(), listOf(Crop(-1f, 1f, -share, share)))) - .build() - val encoders = DefaultEncoderFactory.Builder(context) - .apply { if (bitrate > 0) setRequestedVideoEncoderSettings(VideoEncoderSettings.Builder().setBitrate((bitrate * share).toInt()).build()) } - .build() - val thread = HandlerThread("landscape").apply { start() } - val done = CountDownLatch(1) - var failure: ExportException? = null - var transformer: Transformer? = null + failure?.let { throw it } + } finally { + thread.quitSafely() + } + return Finished(warps.steadied, cut) +} + +private const val EXPORT_MINUTES = 15L + +/** The video track's frame times, in order. */ +private fun frameTimes(file: File): LongArray { + val extractor = MediaExtractor() + try { + extractor.setDataSource(file.path) + val track = (0 until extractor.trackCount).firstOrNull { extractor.getTrackFormat(it).getString(MediaFormat.KEY_MIME)?.startsWith("video/") == true } + ?: return LongArray(0) + extractor.selectTrack(track) + val times = ArrayList() + while (true) { + val t = extractor.sampleTime + if (t < 0) break + times += t + if (!extractor.advance()) break + } + return times.sorted().toLongArray() + } finally { + extractor.release() + } +} + +/** Each frame drawn through its own warp. */ +@OptIn(UnstableApi::class) +private class WarpEffect(private val warps: Warps) : GlEffect { + override fun toGlShaderProgram(context: Context, useHdr: Boolean): GlShaderProgram = try { + WarpProgram(warps, useHdr) + } catch (e: GlUtil.GlException) { + throw VideoFrameProcessingException(e) + } +} + +@OptIn(UnstableApi::class) +private class WarpProgram(private val warps: Warps, useHdr: Boolean) : BaseGlShaderProgram(useHdr, 1) { + private val program = GlProgram(VERTEX, FRAGMENT).apply { + setBufferAttribute("aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE) + } + + override fun configure(inputWidth: Int, inputHeight: Int) = Size(warps.outWidth, warps.outHeight) + + override fun drawFrame(inputTexId: Int, presentationTimeUs: Long) { try { - // A Transformer lives on the thread whose looper it was built with. - val handler = Handler(thread.looper) - handler.post { - transformer = Transformer.Builder(context) - .setLooper(thread.looper) - .setVideoMimeType(MimeTypes.VIDEO_H265) - .setEncoderFactory(encoders) - .addListener(object : Transformer.Listener { - override fun onCompleted(composition: Composition, exportResult: ExportResult) = done.countDown() + program.use() + program.setSamplerTexIdUniform("uTexSampler", inputTexId, 0) + program.setFloatsUniform("uWarp", warps.at(presentationTimeUs)) + program.setFloatsUniform("uOutSize", floatArrayOf(warps.outWidth.toFloat(), warps.outHeight.toFloat())) + program.setFloatsUniform("uInSize", floatArrayOf(warps.inWidth.toFloat(), warps.inHeight.toFloat())) + program.bindAttributesAndUniforms() + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4) + GlUtil.checkGlError() + } catch (e: GlUtil.GlException) { + throw VideoFrameProcessingException(e, presentationTimeUs) + } + } - override fun onError(composition: Composition, exportResult: ExportResult, exportException: ExportException) { - failure = exportException - done.countDown() - } - }) - .build() - .also { it.start(item, output.path) } - } - if (!done.await(EXPORT_MINUTES, TimeUnit.MINUTES)) { - handler.post { transformer?.cancel() } - error("the crop took over $EXPORT_MINUTES minutes") - } - failure?.let { throw it } - } finally { - thread.quitSafely() + override fun release() { + super.release() + try { + program.delete() + } catch (e: GlUtil.GlException) { + throw VideoFrameProcessingException(e) } } - private const val EXPORT_MINUTES = 15L + private companion object { + const val VERTEX = """ +attribute vec4 aFramePosition; +varying vec2 vTexSamplingCoord; +void main() { + gl_Position = aFramePosition; + vTexSamplingCoord = aFramePosition.xy * 0.5 + 0.5; +} +""" + + // Texture rows run bottom up, the warps' pixels top down. + const val FRAGMENT = """ +precision highp float; +uniform sampler2D uTexSampler; +uniform mat3 uWarp; +uniform vec2 uOutSize; +uniform vec2 uInSize; +varying vec2 vTexSamplingCoord; +void main() { + vec3 p = uWarp * vec3(vTexSamplingCoord.x * uOutSize.x, (1.0 - vTexSamplingCoord.y) * uOutSize.y, 1.0); + vec2 q = p.xy / p.z; + gl_FragColor = texture2D(uTexSampler, clamp(vec2(q.x / uInSize.x, 1.0 - q.y / uInSize.y), 0.0, 1.0)); +} +""" + } } diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/MediaImport.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/MediaImport.kt index 859b82a7..58d68757 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/MediaImport.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/MediaImport.kt @@ -129,6 +129,7 @@ class MediaImport( _problem.update { p -> p?.copy(handles = p.handles - handle)?.takeIf { it.handles.isNotEmpty() } } }, onFailed = { handle, why -> report(handle, ImportProblem.Kind.NOT_SAVED, why) }, + steadiedOnGlasses = { w, h -> steadiedOnGlasses(w, h) }, ) /** [handle] is no longer coming: it joins the problem on show, or starts one of [kind]. */ @@ -478,6 +479,20 @@ class MediaImport( @Volatile var bugReportReader: ((java.io.File) -> Unit)? = null + /** + * Whether a video this size came out of the glasses' own two-stage stabilization; any other is steadied on the phone. + * The app sets it from the glasses' settings; this is their default output. + */ + @Volatile + var steadiedOnGlasses: (Int, Int) -> Boolean = { w, h -> minOf(w, h) == 1512 && maxOf(w, h) == 2016 } + + /** [input] steadied or cut as an import would, into [output], with the logs kept for [handle]: for checking by hand. */ + fun finishAgain(input: java.io.File, handle: String, output: java.io.File, landscape: Boolean, steadiness: Int): String { + val logs = index.originals(handle) + val done = finishVideo(context, input, output, java.io.File(logs, Assembly.IMU), java.io.File(logs, Assembly.FRAMES), steadiness, landscape, steadiedOnGlasses) + return done?.let { "steadied ${it.steadied}, landscape ${it.landscape}" } ?: "nothing to do" + } + /** * Ask the glasses for a bug report, the way the Meta app does: svc7006 REQUEST_BUG_REPORT. The zip * comes back over airship in the "bugreport" namespace and goes to Download/Visor. diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Steady.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Steady.kt new file mode 100644 index 00000000..b8afffda --- /dev/null +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Steady.kt @@ -0,0 +1,256 @@ +package com.macedonga.visor.core.media + +import kotlin.math.atan2 +import kotlin.math.cos +import kotlin.math.exp +import kotlin.math.sin +import kotlin.math.sqrt + +/** + * Per frame, which input pixel each output pixel shows: column-major 3x3 matrices on pixels with y down, one per frame + * of the video in presentation order. + */ +internal class Warps( + val inWidth: Int, + val inHeight: Int, + val outWidth: Int, + val outHeight: Int, + private val ptsUs: LongArray, + private val matrices: List, + /** Turned frame by frame, not only cut. */ + val steadied: Boolean, +) { + private var offset: Long? = null + + /** The warp for the frame shown at [presentationTimeUs], the first one setting where the file's clock starts. */ + fun at(presentationTimeUs: Long): FloatArray { + if (matrices.size == 1) return matrices[0] + val base = offset ?: (presentationTimeUs - ptsUs[0]).also { offset = it } + val t = presentationTimeUs - base + val found = ptsUs.binarySearch(t) + val i = if (found >= 0) found else { + val after = -found - 1 + when { + after == 0 -> 0 + after >= ptsUs.size -> ptsUs.size - 1 + t - ptsUs[after - 1] <= ptsUs[after] - t -> after - 1 + else -> after + } + } + return matrices[i] + } +} + +/** + * Stabilization on the phone, for videos the glasses leave shaking (3K, which firmware 7238 does not steady, or any video + * with two-stage stabilization off), as the Meta app does when it brings one over. The motion log gives the head's + * rotation at each frame; the path is smoothed and each frame turned by the difference, inside a crop margin. + * + * Measured on the Gen 2 against a shaky 1080p walk: the IMU's axes are Android's (x right, y up, z toward the wearer), + * a frame's moment is its mid-exposure plus 10 ms, and the focal length is 0.78 of the video's width. Within the margin, + * shake fell by 40 to 60 percent, roll by a third. Rolling shutter is left alone: modelling it, along either axis, made + * no measurable difference. + */ +internal object Steady { + private const val FOCAL_SHARE = 0.78125 + private const val LAG_NS = 10_000_000L + private const val ROUNDS = 8 + + /** + * The warps for a [width] x [height] video whose frames show at [ptsUs]: steadied when [steady] and the logs cover it + * frame for frame, cut to a 4:3 band across the whole width when [landscape]. Null when neither happens. + */ + fun plan( + width: Int, + height: Int, + ptsUs: LongArray, + imu: List?, + frames: List?, + strength: Int, + steady: Boolean, + landscape: Boolean, + ): Warps? { + val path = if (steady && imu != null && imu.size > 1 && frames != null && frames.size == ptsUs.size && frames.size > 2) { + Path(imu, frames) + } else { + null + } + if (path == null && !landscape) return null + val (keep, sigmaSeconds) = if (path != null) feel(strength) else 1.0 to 0.0 + val ow = even(width * keep) + val oh = if (landscape) even(ow * 3.0 / 4.0) else even(height * keep) + val f = FOCAL_SHARE * width + val geometry = Geometry(width, height, ow, oh, f) + val turns = path?.corrections(sigmaSeconds, geometry) ?: listOf(DoubleArray(3)) + return Warps(width, height, ow, oh, ptsUs, turns.map(geometry::matrix), steadied = path != null) + } + + /** + * The share of each side kept, and the seconds of smoothing, for the capture's stabilization: 1 High, 2 Medium, + * 3 Low, 0 Auto-select. On the walk they were measured against, any of them took out about half the shake; more + * margin buys steadier sway, not less shake. + */ + private fun feel(strength: Int) = when (strength) { + 1 -> 0.8 to 0.3 + 3 -> 0.9 to 0.12 + else -> 0.85 to 0.2 + } + + private fun even(v: Double) = (v.toInt() / 2) * 2 + + /** Pinhole cameras sharing a focal length: the video and the centred crop the output is. */ + private class Geometry(val w: Int, val h: Int, val ow: Int, val oh: Int, val f: Double) { + /** Output pixel to input pixel, for the camera turned by rotation vector [r]: K_in R K_out⁻¹, column-major. */ + fun matrix(r: DoubleArray): FloatArray { + val m = rotation(r) + // K_out⁻¹ maps (x, y) to ((x - ow/2) / f, (y - oh/2) / f, 1); K_in maps (X, Y, Z) to (f X + w/2 Z, f Y + h/2 Z, Z). + val a = Array(3) { DoubleArray(3) } + for (row in 0 until 3) { + a[row][0] = m[row][0] / f + a[row][1] = m[row][1] / f + a[row][2] = m[row][2] - m[row][0] * ow / 2 / f - m[row][1] * oh / 2 / f + } + val k = arrayOf(doubleArrayOf(f, 0.0, w / 2.0), doubleArrayOf(0.0, f, h / 2.0), doubleArrayOf(0.0, 0.0, 1.0)) + val out = FloatArray(9) + for (row in 0 until 3) for (col in 0 until 3) { + out[col * 3 + row] = (k[row][0] * a[0][col] + k[row][1] * a[1][col] + k[row][2] * a[2][col]).toFloat() + } + return out + } + + /** Whether every edge of the output, turned by [r], still falls inside the video. */ + fun covered(r: DoubleArray): Boolean { + val m = matrix(r) + for (x in listOf(0.0, ow / 2.0, ow.toDouble())) for (y in listOf(0.0, oh / 2.0, oh.toDouble())) { + val z = m[2] * x + m[5] * y + m[8] + if (z <= 0) return false + val u = (m[0] * x + m[3] * y + m[6]) / z + val v = (m[1] * x + m[4] * y + m[7]) / z + if (u < 0 || u > w || v < 0 || v > h) return false + } + return true + } + } + + /** The head's rotation at each frame, from the motion log, in the camera's axes (x right, y down, z ahead). */ + private class Path(imu: List, frames: List) { + private val at: List + private val fps: Double + + init { + val q = ArrayList(imu.size) + q += doubleArrayOf(1.0, 0.0, 0.0, 0.0) + for (k in 1 until imu.size) { + val dt = (imu[k].timeNs - imu[k - 1].timeNs) / 1e9 + val a = imu[k - 1] + val b = imu[k] + val step = doubleArrayOf((a.gx + b.gx) / 2.0 * dt, -(a.gy + b.gy) / 2.0 * dt, -(a.gz + b.gz) / 2.0 * dt) + q += normalize(multiply(q[k - 1], quaternionOf(step))) + } + val times = LongArray(imu.size) { imu[it].timeNs } + at = frames.map { frame -> + val t = frame.timeNs + frame.exposureNs / 2 + LAG_NS + val i = times.binarySearch(t).let { if (it >= 0) it else (-it - 1) }.coerceIn(1, times.size - 1) + val share = ((t - times[i - 1]).toDouble() / (times[i] - times[i - 1]).coerceAtLeast(1)).coerceIn(0.0, 1.0) + normalize(DoubleArray(4) { c -> q[i - 1][c] * (1 - share) + aligned(q[i - 1], q[i])[c] * share }) + } + val span = (frames.last().timeNs - frames.first().timeNs).coerceAtLeast(1) + fps = (frames.size - 1) / (span / 1e9) + } + + /** + * Per frame, the turn from where the head was to the smoothed path. Where a frame would leave the margin, the path + * is drawn toward the head there and smoothed again, a few times over: the slow sway is followed, and the fast + * shake still goes. Scaling each turn down instead kept a fraction of the shake with it. + */ + fun corrections(sigmaSeconds: Double, geometry: Geometry): List { + val sigma = (sigmaSeconds * fps).coerceAtLeast(1.0) + var path = smooth(at, sigma) + repeat(ROUNDS) { + val fits = DoubleArray(at.size) { i -> fit(geometry, turn(i, path)) } + if (fits.all { it >= 1.0 }) return List(at.size) { turn(it, path) } + path = smooth(List(at.size) { i -> normalize(multiply(at[i], quaternionOf(scale(turn(i, path), fits[i])))) }, (sigma / 4).coerceAtLeast(1.0)) + } + return List(at.size) { i -> turn(i, path).let { scale(it, fit(geometry, it)) } } + } + + private fun turn(i: Int, path: List) = vectorOf(multiply(conjugate(at[i]), path[i])) + + private fun fit(geometry: Geometry, turn: DoubleArray) = largest { s -> geometry.covered(scale(turn, s)) } + + /** [qs] averaged over a Gaussian [sigma] frames wide. */ + private fun smooth(qs: List, sigma: Double): List { + val reach = (3 * sigma).toInt() + val weights = DoubleArray(2 * reach + 1) { exp(-((it - reach) * (it - reach)) / (2 * sigma * sigma)) } + val n = qs.size + return List(n) { i -> + val sum = DoubleArray(4) + for (k in -reach..reach) { + val q = aligned(qs[i], qs[(i + k).coerceIn(0, n - 1)]) + for (c in 0 until 4) sum[c] += weights[k + reach] * q[c] + } + normalize(sum) + } + } + + /** The largest share in [0, 1] that [ok] accepts, by halving. */ + private fun largest(ok: (Double) -> Boolean): Double { + if (ok(1.0)) return 1.0 + var low = 0.0 + var high = 1.0 + repeat(12) { + val mid = (low + high) / 2 + if (ok(mid)) low = mid else high = mid + } + return low + } + } + + private fun scale(r: DoubleArray, s: Double) = doubleArrayOf(r[0] * s, r[1] * s, r[2] * s) + + private fun multiply(a: DoubleArray, b: DoubleArray) = doubleArrayOf( + a[0] * b[0] - a[1] * b[1] - a[2] * b[2] - a[3] * b[3], + a[0] * b[1] + a[1] * b[0] + a[2] * b[3] - a[3] * b[2], + a[0] * b[2] - a[1] * b[3] + a[2] * b[0] + a[3] * b[1], + a[0] * b[3] + a[1] * b[2] - a[2] * b[1] + a[3] * b[0], + ) + + private fun conjugate(q: DoubleArray) = doubleArrayOf(q[0], -q[1], -q[2], -q[3]) + + private fun normalize(q: DoubleArray): DoubleArray { + val n = sqrt(q.sumOf { it * it }) + return DoubleArray(4) { q[it] / n } + } + + /** [q], or its negative, whichever is on [reference]'s side: the same rotation, fit to average with it. */ + private fun aligned(reference: DoubleArray, q: DoubleArray): DoubleArray = + if ((0 until 4).sumOf { reference[it] * q[it] } < 0) DoubleArray(4) { -q[it] } else q + + /** The unit quaternion turning by rotation vector [r]. */ + private fun quaternionOf(r: DoubleArray): DoubleArray { + val angle = sqrt(r[0] * r[0] + r[1] * r[1] + r[2] * r[2]) + if (angle < 1e-12) return doubleArrayOf(1.0, r[0] / 2, r[1] / 2, r[2] / 2) + val s = sin(angle / 2) / angle + return doubleArrayOf(cos(angle / 2), r[0] * s, r[1] * s, r[2] * s) + } + + /** The rotation vector of unit quaternion [q], the shorter way round. */ + private fun vectorOf(q: DoubleArray): DoubleArray { + val p = if (q[0] < 0) DoubleArray(4) { -q[it] } else q + val v = sqrt(p[1] * p[1] + p[2] * p[2] + p[3] * p[3]) + if (v < 1e-12) return doubleArrayOf(2 * p[1], 2 * p[2], 2 * p[3]) + val angle = 2 * atan2(v, p[0]) + return doubleArrayOf(p[1] / v * angle, p[2] / v * angle, p[3] / v * angle) + } + + /** The rotation matrix of rotation vector [r], rows first. */ + private fun rotation(r: DoubleArray): Array { + val q = quaternionOf(r) + val (w, x, y, z) = q.toList() + return arrayOf( + doubleArrayOf(1 - 2 * (y * y + z * z), 2 * (x * y - w * z), 2 * (x * z + w * y)), + doubleArrayOf(2 * (x * y + w * z), 1 - 2 * (x * x + z * z), 2 * (y * z - w * x)), + doubleArrayOf(2 * (x * z - w * y), 2 * (y * z + w * x), 1 - 2 * (x * x + y * y)), + ) + } +} From aec105b9efb8ed30e1c2bedbd590d53da086ca62 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Sun, 25 Oct 2026 17:17:59 +0100 Subject: [PATCH 89/90] Debug actions HEARING (start hearing correction and wait for it, stop it, or show its state) and CWW_CLEAR (delete the enrolled custom wake word). The wake word goes as the Meta app removes it: NetEvent 365 with index 0 of 1, an empty phrase and recording, and a new id; on the pair the glasses then report no enrollment and no longer turn custom_wake_word_enabled back on at assistant start. Used for #42. --- .../com/macedonga/visor/debug/DebugReceiver.kt | 15 +++++++++++++++ .../visor/core/assistant/CustomWakeWord.kt | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index be5add5d..070366e8 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -69,6 +69,7 @@ class DebugReceiver : BroadcastReceiver() { @Inject lateinit var developer: com.macedonga.visor.outside.DeveloperMode @Inject lateinit var texts: com.macedonga.visor.connection.PhoneMessages @Inject lateinit var wifiLink: com.macedonga.visor.connection.WifiLink + @Inject lateinit var hearing: com.macedonga.visor.core.assistant.Hearing override fun onReceive(context: Context, intent: Intent) { val action = intent.action?.substringAfterLast('.').orEmpty() @@ -213,6 +214,20 @@ class DebugReceiver : BroadcastReceiver() { }) } } + // Delete the enrolled custom wake word from the glasses. + "CWW_CLEAR" -> answerLater { if (assistant.wakeWord.clear()) "sent; the glasses answer with a result in the log" else "not sent" } + // Hearing correction: --es do correction (start, waited for) | stop | state. + "HEARING" -> answerLater { + when (intent.getStringExtra("do") ?: "state") { + "correction" -> { + hearing.startCorrection() + val on = hearing.awaitRunning(com.macedonga.visor.core.assistant.HearingMode.CORRECTION, 10_000) + "running=$on ${hearing.state.value}" + } + "stop" -> { hearing.stop(); hearing.awaitStopped(); "${hearing.state.value}" } + else -> "${hearing.state.value}" + } + } // Restart only the glasses' assistant (it re-reads MobileConfig): live_caption_voice_id away and back. "ASSISTANT_RESTART" -> answerLater { if (com.macedonga.visor.language.restartGlassesAssistant(settings)) "restarting the glasses' assistant" else "not connected" } // Developer mode on or off: --ez on false. diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/CustomWakeWord.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/CustomWakeWord.kt index 98e12a6b..eca03b2f 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/CustomWakeWord.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/CustomWakeWord.kt @@ -173,6 +173,10 @@ class CustomWakeWord internal constructor( return withTimeoutOrNull(ENROLL_MS) { waiting.await() } } + /** Delete the enrolled phrase, as the Meta app does: an empty utterance under a new id. The glasses answer with a result. */ + suspend fun clear(): Boolean = + send(EV_UTTERANCE, ThriftWriter().i32(1, 0).i32(2, 1).str(3, "").str(4, ByteArray(0)).str(5, UUID.randomUUID().toString()).done()) + /** Close the session; false when the close could not go out. */ suspend fun end(session: String, reason: Int = 0): Boolean = send(EV_SESSION_END, ThriftWriter().str(1, session).i32(2, reason).done()) @@ -204,6 +208,7 @@ class CustomWakeWord internal constructor( private const val BYTES_PER_MS = 32 // A voice near the glasses peaks around a third of full scale. private const val LOUD = 12_000f + private const val EV_UTTERANCE = 365 private const val EV_BATCH_ENROLLMENT = 386 private const val EV_SESSION_BEGIN = 390 private const val EV_SESSION_END = 391 From ccc54c0b4c1de028948cebe13692f6efcb08a9e1 Mon Sep 17 00:00:00 2001 From: Marco Ceccon Date: Sun, 25 Oct 2026 18:48:57 +0100 Subject: [PATCH 90/90] Display: a quiet turn (#48) draws no answer card. --- .../main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt index 90be17d1..a205d24a 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt @@ -1078,7 +1078,7 @@ class Assistant( preamble(l, interaction, transcript, text) _phase.value = AssistantPhase.SPEAKING val voiceId = glassesVoice() - val card = if (display()) answerCard(text) else null + val card = if (display() && text.isNotEmpty()) answerCard(text) else null val answer = netEvent(EV_VOICE_ACTION_RESPONSE, varm(interaction, transcript, text, onGlasses = true, voiceId = voiceId, card = card != null)) // An answer that asks something has the glasses listen for the reply, without the wake word. val complete = netEvent(