diff --git a/CLAUDE.md b/CLAUDE.md index 61e647c4..0aed14a9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,6 +37,27 @@ the frame style and language packs. Works on real glasses as of 2026-10-05. `s` low-half. Any mistake shows up as `0x1043 INVALID_APP_IDENTITY`. - Airship: accepting a file (or refusing it) must end with a drop on type 7, or the glasses' send queue stalls behind it. +- DataX OK on the wire is status 0. 0x1000 is svc36's own result; sent as OK, it made the Display's + image loader retry every image. +- Replies on a channel the glasses opened go on `DataXConnection.mirror`. Replies on a channel Visor + opened keep the originator bit. + +Display: + +- svc28 device state comes wrapped in HNDeviceState `{1: stateCommon}`. Unwrapped as a plain + DeviceState, the battery reads 0%. +- svc41108: the Display takes the phone's internet state only on a channel the phone opens. Until + then its apps stay off the phone's internet. +- svc26 must be opened with an empty type 3 on every connection, or every share fails ("No remote + channel at identifier: MWA"). +- svc70 folder names are lowercase. With "SENT" a sent text reloads as still sending. +- A text notification needs app id `com.android.sms`, provider 3 and the SMS thread id, or the + Messages app drops it. +- Captions and translation turn on only through a svc29 pack install and FEATURE_CHANGE. The same + pack on airship `language_resource` turns nothing on. +- Each Meta `/login` exchange ends the older sessions. Keep one token per run. +- With an Android phone the music screen ignores AVRCP cover art. The cover comes only over svc9012. +- The Phone app's recent calls wait on svc70 type 11, once per provider. svc11006 does not fill them. ## Build diff --git a/README.md b/README.md index 7ca30645..449da223 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,13 @@ Visor works with Ray-Ban Meta, Oakley Meta and the other Meta glasses. It speaks | **Settings** | Every setting the glasses have, the way the Meta app shows them. Advanced holds the rest, assistant language packs and the firmware's feature flags. | | **Outside the app** | A home screen widget, a Quick Settings tile, launcher shortcuts, and a Live Update notification while the Live view runs. | | **Developer mode** | Other apps and automations can take photos and videos, import captures and hear about new ones. See [Developer mode](docs/developer-mode.md). | +| **More than one pair** | Keep several pairs on one phone. Home swipes between them, each with its name, batteries and a Connect button, and ends with a page to set up another. | +| **Meta Ray-Ban Display** | A teleprompter, live captions and translation, the Neural Band's gestures, the Briefing panel and saved Wi-Fi networks. The glasses' own apps reach the internet through the phone, with Instagram's Reels and messages among them. | +| **Neural Band** | Set up from the Display's settings, with its battery on Home and its wrist in Settings. | ## Requirements -- **Glasses:** a pair of Meta glasses, such as Ray-Ban Meta or Oakley Meta. Support for the Meta Ray-Ban Display is limited and lives on the [`display`](https://github.com/macedonga/visor/tree/display) branch. +- **Glasses:** a pair of Meta glasses, such as Ray-Ban Meta, Oakley Meta or the Meta Ray-Ban Display. Support for the Display and its Neural Band is newer and does not yet cover everything the Meta AI app does. - **Phone:** Android 10 or later. - **Meta account**, for setup. Log in with a password or an email code. Passkeys do not work. - **For the assistant:** an API key from Anthropic or from an OpenAI-compatible service. diff --git a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt index d0ad373d..070366e8 100644 --- a/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt +++ b/app/src/debug/kotlin/com/macedonga/visor/debug/DebugReceiver.kt @@ -37,6 +37,7 @@ import kotlinx.coroutines.withTimeoutOrNull * READ_SETTINGS, MEDIA, IMPORT (--es handle a,b: by handle prefix), ASK (--es q), ONBOARD (--es page --ef t), * MCP_PROBE / MCP_ADD (--es url --es name --es auth NONE|TOKEN|HEADER --es secret --es header), MCP_STATUS, * MCP_TOOLS (what the model would get), MCP_CALL (--es tool exposed-name --es args json), MCP_REMOVE (--es id), + * DEMO_PAIR (--ez on false takes it away): a second pair on the home, a black Skyler, never connected or looked at. * PHONE (--es do directions --es to --es mode | play --es q --es app | media --es c pause|resume|next|... | now). */ @AndroidEntryPoint @@ -54,14 +55,30 @@ class DebugReceiver : BroadcastReceiver() { @Inject lateinit var config: com.macedonga.visor.core.media.GlassesConfig @Inject lateinit var liveSettings: com.macedonga.visor.core.media.LiveSettingsStore @Inject lateinit var mcp: com.macedonga.visor.core.assistant.mcp.McpHub + @Inject lateinit var pairs: com.macedonga.visor.pairs.Pairs + @Inject lateinit var looks: com.macedonga.visor.stage.Looks @Inject lateinit var phone: com.macedonga.visor.phone.PhoneActions + @Inject lateinit var prompter: com.macedonga.visor.core.connection.Teleprompter + @Inject lateinit var captions: com.macedonga.visor.core.connection.Captions + @Inject lateinit var languages: com.macedonga.visor.language.LanguageInstaller + @Inject lateinit var captionPacks: com.macedonga.visor.display.CaptionPacks + @Inject lateinit var screencast: com.macedonga.visor.core.media.Screencast + @Inject lateinit var requests: com.macedonga.visor.connection.RequestLog + @Inject lateinit var login: com.macedonga.visor.login.MetaLogin + @Inject lateinit var internet: com.macedonga.visor.connection.InternetSwitch @Inject lateinit var developer: com.macedonga.visor.outside.DeveloperMode + @Inject lateinit var texts: com.macedonga.visor.connection.PhoneMessages + @Inject lateinit var wifiLink: com.macedonga.visor.connection.WifiLink + @Inject lateinit var hearing: com.macedonga.visor.core.assistant.Hearing override fun onReceive(context: Context, intent: Intent) { val action = intent.action?.substringAfterLast('.').orEmpty() Log.i("Visor", "API: $action") when (action) { "STATUS" -> resultData = status() + // The phone's thread list as Visor reads it for the glasses' Messages: --ei count. + "THREADS" -> resultData = texts.describeThreads(intent.getIntExtra("count", 15)) + "DEMO_PAIR" -> resultData = demoPair(intent.getBooleanExtra("on", true)) "CONNECT" -> resultData = runCatching { GlassesService.start(context); "connecting" }.getOrElse { "could not start the service: $it" } "DISCONNECT" -> { GlassesService.stop(context) @@ -117,8 +134,10 @@ class DebugReceiver : BroadcastReceiver() { "refused: unknown namespace or not connected" } } - // A notification built by hand: --ei type --ei provider --es title --es text --es sender --es pkg. + // A notification built by hand: --ei type --ei provider --es title --es text --es sender --es pkg --es thread + // (a text's thread in the SMS store, whose number becomes the sender's id). "NOTIFY" -> { + val thread = intent.getStringExtra("thread").orEmpty() val n = com.macedonga.visor.notify.GlassesNotification( key = "visor|" + System.currentTimeMillis(), pkg = intent.getStringExtra("pkg") ?: "org.telegram.messenger", @@ -129,7 +148,8 @@ class DebugReceiver : BroadcastReceiver() { type = intent.getIntExtra("type", 1), provider = intent.getIntExtra("provider", 3), sender = intent.getStringExtra("sender").orEmpty(), - senderId = intent.getStringExtra("sid").orEmpty(), + senderId = intent.getStringExtra("sid") ?: texts.addressesOf(thread).firstOrNull().orEmpty(), + threadId = thread, ) connection.submit { it.sendOnService(com.macedonga.visor.notify.GlassesNotification.SVC, 1, n.envelope()) } resultData = "sent type ${n.type} provider ${n.provider}" @@ -194,6 +214,20 @@ class DebugReceiver : BroadcastReceiver() { }) } } + // Delete the enrolled custom wake word from the glasses. + "CWW_CLEAR" -> answerLater { if (assistant.wakeWord.clear()) "sent; the glasses answer with a result in the log" else "not sent" } + // Hearing correction: --es do correction (start, waited for) | stop | state. + "HEARING" -> answerLater { + when (intent.getStringExtra("do") ?: "state") { + "correction" -> { + hearing.startCorrection() + val on = hearing.awaitRunning(com.macedonga.visor.core.assistant.HearingMode.CORRECTION, 10_000) + "running=$on ${hearing.state.value}" + } + "stop" -> { hearing.stop(); hearing.awaitStopped(); "${hearing.state.value}" } + else -> "${hearing.state.value}" + } + } // Restart only the glasses' assistant (it re-reads MobileConfig): live_caption_voice_id away and back. "ASSISTANT_RESTART" -> answerLater { if (com.macedonga.visor.language.restartGlassesAssistant(settings)) "restarting the glasses' assistant" else "not connected" } // Developer mode on or off: --ez on false. @@ -313,6 +347,106 @@ class DebugReceiver : BroadcastReceiver() { resultData = "asked for $id" } } + "PROMPTER" -> answerLater { + if (intent.getBooleanExtra("close", false)) prompter.close() + else prompter.show( + intent.getStringExtra("text") ?: "Testing the teleprompter from Visor.", + com.macedonga.visor.core.connection.PrompterLook(paged = intent.getBooleanExtra("paged", false), followVoice = intent.getBooleanExtra("voice", false)), + ) + kotlinx.coroutines.delay(300) + withTimeoutOrNull(8_000) { + prompter.state.first { it.phase != com.macedonga.visor.core.connection.PrompterPhase.STARTING && it.phase != com.macedonga.visor.core.connection.PrompterPhase.CLOSING } + } + prompter.state.value.let { "${it.phase} refusal=${it.refusal}" } + } + // --es features a,b: what Meta has for these constellation features on this pair. + "PACKS" -> answerLater { + val features = intent.getStringExtra("features").orEmpty().split(',').map { it.trim() }.filter { it.isNotEmpty() } + runCatching { languages.offersFor(features) }.fold( + { offers -> offers.joinToString("\n") { "${it.feature} ${it.resource} ${it.version} ${it.type} app ${it.appId} ${it.size} ${it.url}" }.ifEmpty { "none" } }, + { "failed: ${it.message}" }, + ) + } + // --ei svc --ei type --es hex: one message on a service; the answer is what the glasses said within 5 s. + "SEND" -> answerLater { + val svc = intent.getIntExtra("svc", 0) + val body = intent.getStringExtra("hex").orEmpty().chunked(2).filter { it.length == 2 }.map { it.toInt(16).toByte() }.toByteArray() + val heard = java.util.Collections.synchronizedList(mutableListOf()) + val watch = scope.launch { + launch { connection.serviceData.collect { if (it.serviceId == svc) heard += "type ${it.messageType}: " + it.payload.joinToString("") { b -> "%02x".format(b) } } } + launch { connection.warnings.collect { if (it.contains("service $svc ")) heard += it } } + } + connection.submit { it.sendOnService(svc, intent.getIntExtra("type", 0), body) } + kotlinx.coroutines.delay(5_000) + watch.cancel() + heard.joinToString("\n").ifEmpty { "nothing" } + } + // --es doc --es name --es vars : one query on Meta's ar GraphQL, as the + // user; the answer's first 4000 characters. + "GRAPHQL" -> answerLater { + val text = login.graphql( + intent.getStringExtra("name").orEmpty(), + intent.getStringExtra("doc").orEmpty(), + intent.getStringExtra("vars64")?.let { String(android.util.Base64.decode(it, 0)) } ?: intent.getStringExtra("vars") ?: "{}", + ) + if (!intent.getBooleanExtra("shape", false)) text.take(4000) else { + // Every path with its type, a string's length and start: the structure without the bulk. + val out = StringBuilder() + fun walk(path: String, v: Any?) { + when (v) { + is org.json.JSONObject -> v.keys().forEach { walk("$path.$it", v.opt(it)) } + is org.json.JSONArray -> { out.append("$path [${v.length()}]\n"); if (v.length() > 0) walk("$path[0]", v.opt(0)) } + is String -> out.append("$path str(${v.length}) ${v.take(60).replace('\n', ' ')}\n") + else -> out.append("$path = $v\n") + } + } + walk("", org.json.JSONObject(text)) + out.toString().take(4000) + } + } + // --ez on false: tell the Display the phone is offline, so it keeps its apps off the phone's internet. + "INTERNET" -> { + internet.sharing = intent.getBooleanExtra("on", true) + resultData = "sharing=${internet.sharing}" + } + // The request log's newest entries, links cut at their query. + "REQUESTS" -> resultData = requests.items.value.takeLast(intent.getIntExtra("n", 30)).joinToString("\n") { + "${it.source} ${it.outcome ?: "running"} ${it.bytes}B ${it.millis}ms [${it.options}] ${it.url.substringBefore('?').take(160)}" + }.ifEmpty { "none" } + // --es key --ez value true|false: one bool written to the glasses. + "MC_SET" -> answerLater { + config.send(mapOf(intent.getStringExtra("key").orEmpty() to intent.getBooleanExtra("value", false))) ?: "sent" + } + // --ez stop true to stop; the answer is the state once it settles. + "MIRROR" -> answerLater { + if (intent.getBooleanExtra("stop", false)) screencast.stop() else screencast.start() + kotlinx.coroutines.delay(500) + withTimeoutOrNull(40_000) { screencast.state.first { it.phase != com.macedonga.visor.core.media.ScreencastPhase.STARTING } } + screencast.state.value.toString() + } + // --es feature live_captions_en: Meta's pack installed through the glasses' OTA service. + "CAPTION_PACK" -> answerLater { + captionPacks.install(intent.getStringExtra("feature") ?: "live_captions_en") + kotlinx.coroutines.delay(500) + withTimeoutOrNull(600_000) { captionPacks.work.first { it !is com.macedonga.visor.display.PackWork.Working } } + captionPacks.work.value.toString() + } + "CAPTIONS" -> answerLater { + val self = intent.getStringExtra("lang") ?: "en" + when { + intent.getBooleanExtra("stop", false) -> captions.stop() + intent.getBooleanExtra("show", false) -> Unit + else -> captions.start( + if (intent.hasExtra("other")) com.macedonga.visor.core.connection.CaptionsMode.TRANSLATION else com.macedonga.visor.core.connection.CaptionsMode.CAPTIONS, + self, intent.getStringExtra("other") ?: self, + ) + } + kotlinx.coroutines.delay(300) + withTimeoutOrNull(8_000) { + captions.state.first { it.phase == com.macedonga.visor.core.connection.CaptionsPhase.LIVE || it.phase == com.macedonga.visor.core.connection.CaptionsPhase.IDLE } + } + captions.state.value.let { s -> "${s.phase} ${s.mode} refusal=${s.refusal}\n" + s.lines.joinToString("\n") { "${if (it.done) "" else "… "}${it.text}${it.source?.let { src -> " ($src)" }.orEmpty()}" } } + } "PHONE" -> answerLater { when (intent.getStringExtra("do")) { "directions" -> phone.directions(intent.getStringExtra("to").orEmpty(), intent.getStringExtra("mode").orEmpty()).toString() @@ -366,6 +500,12 @@ class DebugReceiver : BroadcastReceiver() { mcp.signIn.first { it is com.macedonga.visor.core.assistant.mcp.SignIn.Done || it is com.macedonga.visor.core.assistant.mcp.SignIn.Failed } }.toString() } + // A lease that needs Wi-Fi, held and let go as the Display's would be. + "WIFI_LINK" -> resultData = if (intent.getBooleanExtra("on", true)) { + wifiLink.hold("debug").let { "held: " + (wifiLink.refusal() ?: "moving") } + } else { + wifiLink.release("debug").let { "released" } + } "MCP_REMOVE" -> { mcp.store.remove(intent.getStringExtra("id").orEmpty()) resultData = "removed" @@ -374,6 +514,19 @@ class DebugReceiver : BroadcastReceiver() { } } + private fun demoPair(on: Boolean): String { + if (!on) { + pairs.demo(null, null) + looks.demo(DEMO_SERIAL, null, null) + return "demo pair gone" + } + val look = com.macedonga.visor.core.identity.language.GlassesLook("skyler", "Skyler", "shiny_black", "Shiny Black", "clear", "Clear", null) + looks.demo(DEMO_SERIAL, look, DEMO_PICTURE) + val seen = com.macedonga.visor.home.PairSeen(glasses = 64, case = 88, at = System.currentTimeMillis() - 2 * 3_600_000L, folded = true) + pairs.demo(com.macedonga.visor.pairs.KnownPair(DEMO_SERIAL, "00:00:00:00:00:00"), seen) + return "demo pair on: swipe the home's stage" + } + private fun status(): String { val s = connection.state.value val b = s.device?.battery @@ -411,5 +564,9 @@ class DebugReceiver : BroadcastReceiver() { private companion object { val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + const val DEMO_SERIAL = "DEMO00SKYLER00" + // Meta's own picture of a black Skyler, as its asset service hands it out. + const val DEMO_PICTURE = "https://scontent.xx.fbcdn.net/mci_ab/uap/asset_manager/name_revision/?ab_b=e&ab_page=AssetManagerNameAndRevision" + + "&ab_entry=RBM2-Skyler-ShinyBlack-Clear-Concord&set=mwa_device_image_assets&revision=1837680503843762&density=3&mode=dark" } } diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index feeeec3b..f19dfd84 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -21,10 +21,22 @@ + + + + + + + + + + + + @@ -152,6 +164,25 @@ android:exported="false" android:foregroundServiceType="connectedDevice" /> + + + + + + + + + + + + + + , markAtLaunch: Boolean) { @@ -161,6 +184,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) if (place == MainActivity.OPEN_GALLERY && backStack.lastOrNull() != MediaKey) backStack.add(MediaKey) if (place == MainActivity.OPEN_META_AI && backStack.lastOrNull() != MetaAiKey) backStack.add(MetaAiKey) if (place == MainActivity.OPEN_ONBOARDING && backStack.lastOrNull() !is OnboardingKey) backStack.add(OnboardingKey(welcome = true)) + if (place == MainActivity.OPEN_ACCOUNTS && backStack.lastOrNull() != AccountsKey) backStack.add(AccountsKey) if (place == MainActivity.OPEN_LIVE && backStack.lastOrNull() != LiveKey) { stage.homeToTop = true backStack.add(LiveKey) @@ -202,6 +226,14 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) }, onMetaAi = { backStack.add(MetaAiKey) }, onSettings = { backStack.add(SettingsKey) }, + onDisplayTool = { tool -> + when (tool) { + com.macedonga.visor.display.DisplayTool.TELEPROMPTER -> backStack.add(PrompterKey) + com.macedonga.visor.display.DisplayTool.CAPTIONS -> backStack.add(CaptionsKey) + com.macedonga.visor.display.DisplayTool.GESTURES -> backStack.add(GesturesKey) + com.macedonga.visor.display.DisplayTool.MIRROR -> backStack.add(MirrorKey) + } + }, ) } entry { @@ -235,7 +267,8 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) entry { NativeLoginPage(onDone = { backStack.removeLastOrNull() }) } - entry { EventLogScreen() } + entry { EventLogScreen(onRequests = { backStack.add(RequestLogKey) }) } + entry { com.macedonga.visor.log.RequestLogScreen() } entry { GalleryScreen() } // The pair travels between the home and Settings on the one stage, so the screens only cross-fade, both ways. entry( @@ -255,6 +288,12 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) entry { AboutScreen() } entry { PhoneAccessScreen() } entry { NotifyAppsScreen() } + entry { com.macedonga.visor.display.PrompterScreen() } + entry { com.macedonga.visor.display.GesturesScreen() } + entry { com.macedonga.visor.display.CaptionsScreen() } + entry { com.macedonga.visor.settings.WifiNetworksScreen() } + entry { com.macedonga.visor.settings.AccountsScreen() } + entry { com.macedonga.visor.settings.WebAppsScreen() } entry { MetaAiScreen( onSettings = { backStack.add(AssistantSettingsKey) }, @@ -281,6 +320,7 @@ fun VisorNav(opens: kotlinx.coroutines.flow.Flow, markAtLaunch: Boolean) ) } entry { LiveSettingsScreen() } + entry { MirrorScreen() } entry { com.macedonga.visor.language.LanguageScreen() } entry { FlagsScreen() } entry { ConversationFocusScreen() } diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt new file mode 100644 index 00000000..c851f5cc --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandLink.kt @@ -0,0 +1,347 @@ +package com.macedonga.visor.band + +import android.content.Context +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.BundleCredentials +import com.macedonga.visor.core.identity.BundleState +import com.macedonga.visor.core.identity.IdentityBundle +import com.macedonga.visor.core.identity.IdentityStore +import com.macedonga.visor.core.identity.IdentityTrustExchange +import com.macedonga.visor.core.identity.nowIso +import com.macedonga.visor.core.protocol.Band +import com.macedonga.visor.core.protocol.ServiceId +import com.macedonga.visor.core.session.GlassesSession +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.LinkStateChanged +import com.macedonga.visor.core.session.ProtocolError +import com.macedonga.visor.core.session.ServiceData +import com.macedonga.visor.core.session.SessionCommand +import com.macedonga.visor.core.transport.BluetoothLeTransport +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.withTimeoutOrNull +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +/** The band as last heard from. */ +data class BandState( + val serial: String? = null, + val link: LinkState = LinkState.DISCONNECTED, + val battery: Int? = null, + val charging: Boolean? = null, + val leftHanded: Boolean? = null, + val at: Long? = null, + // The Display holds the band; the phone steps back, as Meta's app does. + val leased: Boolean = false, + // The band's RPC protocol, from its device info: 2 and up turn streams on by a map. + val protocol: Int? = null, + // A lesson has the band's gestures coming to the phone. + val gestures: Boolean = false, + // The firmware's version, from the band's OTA device info. + val firmware: Long? = null, +) + +/** + * The phone's own link to the Neural Band: the band takes one host at a time, so the phone leaves it to the Display, + * which may hold a lease on it (svc150), and takes it only for a lesson or one errand (the + * wrist, a buzz); the Display reports the band's battery itself. It reads the battery and the wrist over svc52822 and sets the + * wrist; in a lesson it turns the gesture streams on and acks each gesture, as the Meta app does. + */ +@Singleton +class BandLink @Inject constructor( + @ApplicationContext private val context: Context, + private val connection: GlassesConnection, + private val bands: Bands, + private val store: IdentityStore, + private val log: EventLog, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val gate = Mutex() + private val seq = AtomicInteger(1) + // Answers still awaited, by request seq; a haptics request waits for its own. + private val replies = ConcurrentHashMap>() + private val leased = MutableStateFlow(false) + private val held = MutableStateFlow(true) + private val lesson = MutableStateFlow(false) + private val borrowing = MutableStateFlow(false) + private val updating = MutableStateFlow(false) + private val _answers = MutableSharedFlow(extraBufferCapacity = 64) + /** Every RPC answer while the band is up, raw: a firmware update reads its own answers here. */ + val answers: SharedFlow = _answers + private val _gestures = kotlinx.coroutines.flow.MutableSharedFlow(extraBufferCapacity = 32) + val gestures: kotlinx.coroutines.flow.SharedFlow = _gestures + + @Volatile private var session: GlassesSession? = null + private var job: Job? = null + + private val _state = MutableStateFlow(BandState()) + val state: StateFlow = _state.asStateFlow() + + init { + scope.launch { + // A lease lives on the Display's link: it ends with it. + connection.state.collect { if (it.link != LinkState.READY && leased.value) leased.value = false } + } + scope.launch { + // A lesson or an errand takes the band from the Display; when it ends, the band goes back. Never otherwise: + // the Display reaches the band on its own, and a phone holding it while the Display restarts keeps it off. + val stepBack = combine(lesson, borrowing, updating) { learning, errand, update -> !learning && !errand && !update } + combine(connection.running, connection.pair, bands.all, stepBack, held) { running, pair, _, away, hold -> + bands.forGlasses(pair)?.takeIf { running && hold && !away } + }.distinctUntilChanged().collect { band -> gate.withLock { switchTo(band) } } + } + scope.launch { leased.collect { lease -> _state.update { it.copy(leased = lease) } } } + } + + /** The Display's lease on the band, from svc150: a non-relay one sends the phone's link away until it ends. */ + fun leased(on: Boolean) { + leased.value = on + } + + /** Let go of the band for a while, so another session can have it: setting it up again. */ + suspend fun release() { + held.value = false + gate.withLock { switchTo(null) } + } + + fun hold() { + held.value = true + } + + /** Holds the band for [block], which needs it for minutes, then gives it back. */ + suspend fun holdFor(block: suspend () -> T): T { + updating.value = true + try { + if (withTimeoutOrNull(BORROW_MS) { _state.first { it.link == LinkState.READY } } == null) error("the band is not reached") + return block() + } finally { + updating.value = false + } + } + + /** The next seq for a request built by hand. */ + fun nextSeq(): Int = seq.getAndIncrement() + + /** Sends RPC requests on the band now; false when the band is not up. */ + fun sendRaw(requests: List): Boolean = send { requests } + + /** Have the band's gestures come here: the band is taken from the Display for as long as it lasts. */ + fun startLesson() { + if (lesson.value) return + lesson.value = true + log.add("Band: a lesson takes the band") + if (_state.value.link == LinkState.READY) streams(true) + } + + fun endLesson() { + if (!lesson.value) return + if (_state.value.link == LinkState.READY) streams(false) + lesson.value = false + log.add("Band: the lesson gives the band back") + } + + /** A buzz on the band; true when the band answers OK. */ + suspend fun vibrate(duration: Long = BUZZ_DURATION, frequency: Float = BUZZ_HZ, amplitude: Int = BUZZ_AMPLITUDE): Boolean = + haptic { seq -> Band.vibrate(seq, duration, frequency, amplitude) } + + /** Stops a buzz; true when the band answers OK. */ + suspend fun stop(): Boolean = haptic { seq -> Band.stopHaptics(seq) } + + /** One haptics request, taking the band from the Display for it; true when the band answers OK. */ + private suspend fun haptic(request: (Int) -> ByteArray): Boolean { + val borrowed = _state.value.link != LinkState.READY + if (borrowed) borrowing.value = true + try { + if (borrowed && withTimeoutOrNull(BORROW_MS) { _state.first { it.link == LinkState.READY } } == null) { + log.add("Band: not reached for a buzz in ${BORROW_MS / 1000} s") + return false + } + val n = seq.getAndIncrement() + val answer = CompletableDeferred() + replies[n] = answer + val code = if (send { listOf(request(n)) }) withTimeoutOrNull(REPLY_MS) { answer.await() } else null + replies.remove(n) + if (code != null && code != Band.OK) log.add("Band: the buzz was refused, code $code") + if (code == null) log.add("Band: no answer to the buzz") + return code == Band.OK + } finally { + if (borrowed) scope.launch { + delay(ERRAND_MS) + borrowing.value = false + } + } + } + + /** Take the band for one errand, then give it back; at once when the phone already has it. */ + private fun borrow(requests: (AtomicInteger) -> List) { + if (send(requests)) return + scope.launch { + borrowing.value = true + val up = kotlinx.coroutines.withTimeoutOrNull(BORROW_MS) { _state.first { it.link == LinkState.READY } } != null + if (up) { + send(requests) + delay(ERRAND_MS) + } else log.add("Band: not reached for an errand in ${BORROW_MS / 1000} s") + borrowing.value = false + } + } + + /** Put the band on [left] or the right wrist; read back at once. */ + fun setLeftHanded(left: Boolean) { + _state.update { it.copy(leftHanded = left) } + borrow { seq -> listOf(Band.configRequest(seq.getAndIncrement(), left), Band.configRequest(seq.getAndIncrement())) } + } + + private fun streams(on: Boolean) { + val sent = send { seq -> listOf(Band.gestureStreams(seq.getAndIncrement(), on, _state.value.protocol)) } + if (sent) { + _state.update { it.copy(gestures = on) } + log.add("Band: gestures ${if (on) "on" else "off"} (protocol ${_state.value.protocol ?: "not read"})") + } + } + + /** Requests on the band's RPC, in order; false when the band is not connected. */ + private fun send(requests: (AtomicInteger) -> List): Boolean { + val running = session ?: return false + if (_state.value.link != LinkState.READY) return false + running.submit( + SessionCommand { s -> + val mux = s.connection ?: return@SessionCommand + for (r in requests(seq)) mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, r) + }, + ) + return true + } + + private suspend fun switchTo(band: BandRecord?) { + if (band != null && _state.value.serial == band.serial && session != null) return + session?.let { + log.add("Band: link down") + it.stop() + } + job?.cancel() + session = null + job = null + if (band == null) { + _state.update { it.copy(link = LinkState.DISCONNECTED, gestures = false) } + return + } + val key = band.key ?: store.ownerKeyPem() ?: return + if (_state.value.serial != band.serial) _state.value = BandState(serial = band.serial, leased = leased.value) + val identity = IdentityBundle(key, band.address, band.serial, nowIso(), "band", BundleState.VERIFIED) + val opened = GlassesSession( + transport = BluetoothLeTransport(context, band.address, defaultPsm = Band.PSM), + trust = IdentityTrustExchange(BundleCredentials(identity)), + ) + session = opened + job = scope.launch { + launch { poll(opened) } + opened.events.collect { event -> + when (event) { + is LinkStateChanged -> { + _state.update { it.copy(link = event.state, gestures = if (event.state == LinkState.READY) it.gestures else false) } + if (event.state == LinkState.READY) { + log.add("Band: link up") + ask(opened, info = true) + // The gestures wait for the device info, which picks how to ask for them; not for long. + if (lesson.value) launch { + delay(INFO_WAIT_MS) + if (lesson.value && !_state.value.gestures) streams(true) + } + } + } + is ServiceData -> if (event.serviceId == ServiceId.WIRELESS_INPUT.value) onBand(event) + is ProtocolError -> log.add("Band: ${event.detail}") + else -> Unit + } + } + } + log.add("Band: connecting to ${band.serial}") + opened.start(scope) + } + + // The band pushes battery changes too; a slow read keeps the reading fresh when it does not. + private suspend fun poll(opened: GlassesSession) { + while (true) { + delay(POLL_MS) + if (_state.value.link == LinkState.READY && session === opened) ask(opened) + } + } + + private fun ask(opened: GlassesSession, info: Boolean = false) = opened.submit( + SessionCommand { s -> + val mux = s.connection ?: return@SessionCommand + if (info) mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.deviceInfoRequest(seq.getAndIncrement())) + if (info) mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.otaDeviceInfo(seq.getAndIncrement())) + mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.batteryRequest(seq.getAndIncrement())) + mux.sendOnService(ServiceId.WIRELESS_INPUT.value, Band.REQUEST_TYPE, Band.configRequest(seq.getAndIncrement())) + }, + ) + + private fun onBand(event: ServiceData) { + val type = event.messageType and 0xFF + if (Band.isGesture(type)) return onGesture(type, event.payload) + if (type == Band.RESPONSE) { + _answers.tryEmit(event.payload) + runCatching { Band.otaInfo(event.payload) }.getOrNull()?.let { info -> _state.update { it.copy(firmware = info.firmware) } } + } + val battery = runCatching { + when (type) { + Band.RESPONSE -> Band.response(event.payload).also { r -> + replies.remove(r.seq)?.complete(r.haptics ?: r.code) + if (r.leftHanded != null) _state.update { it.copy(leftHanded = r.leftHanded) } + if (r.protocol != null) { + _state.update { it.copy(protocol = r.protocol) } + if (lesson.value && !_state.value.gestures) streams(true) + } + }.battery + Band.STREAM_UPDATE -> Band.streamBattery(event.payload) + else -> null + } + }.onFailure { log.add("Band: unreadable message type ${event.messageType}: ${it.message}") }.getOrNull() ?: return + _state.update { it.copy(battery = battery.percent, charging = battery.charging, at = System.currentTimeMillis()) } + } + + // Every gesture is acked, as the Meta app does; a lesson's gestures go in the log so the names can be checked. + private fun onGesture(type: Int, payload: ByteArray) { + val g = runCatching { Band.gesture(type, payload) }.getOrElse { + log.add("Band: unreadable gesture type $type: ${it.message}") + return + } + send { seq -> listOf(Band.gestureAck(seq.getAndIncrement(), g.seq)) } + if (lesson.value) log.add("Band: gesture type ${g.type}, finger ${g.finger}, action ${g.action}, derived ${g.derived}") + _gestures.tryEmit(g) + } + + private companion object { + const val POLL_MS = 5 * 60_000L + const val INFO_WAIT_MS = 1_500L + const val BORROW_MS = 20_000L + const val ERRAND_MS = 1_500L + const val REPLY_MS = 5_000L + // Nothing in the Meta app sends a buzz, so these are a guess at milliseconds, hertz and a 0-255 strength. + const val BUZZ_DURATION = 200L + const val BUZZ_HZ = 170f + const val BUZZ_AMPLITUDE = 160 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandPeers.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandPeers.kt new file mode 100644 index 00000000..4495f5b1 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandPeers.kt @@ -0,0 +1,65 @@ +package com.macedonga.visor.band + +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.LinkState +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.onSubscription +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withTimeoutOrNull + +/** A band the Display has paired: its address, whether it is connected, and whether directly or through the phone. */ +data class PairedBand(val address: String, val connected: Boolean, val direct: Boolean) + +/** + * svc41109, bt_peer_connection: the Display's own pairing with the Neural Band. GET_BLE_PAIRING_INFO (1) is answered + * with BLE_PAIRING_INFO (2): {3 pairedPeers [ConnectionState {1 peerIdentityAddr BleAddr {2 addr}, 2 isConnected}], + * 5 activeLink [ActiveLink {1 link (1 relay, 2 direct), 2 peerIdentityAddr}]}. + */ +@Singleton +class BandPeers @Inject constructor(private val connection: GlassesConnection) { + private val lock = Mutex() + private val _bands = MutableStateFlow?>(null) + + /** The bands the Display has paired, as it last said; null until it has answered. */ + val bands: StateFlow?> = _bands.asStateFlow() + + /** Asks the Display again; without an answer [bands] stays as it was. */ + suspend fun refresh() = lock.withLock { + if (connection.state.value.link != LinkState.READY) return@withLock + val answer = withTimeoutOrNull(ANSWER_MS) { + connection.serviceData + .onSubscription { connection.submit { it.sendOnService(SVC, MT_GET_PAIRING_INFO, ByteArray(0)) } } + .first { it.serviceId == SVC && it.messageType == MT_PAIRING_INFO } + .payload + } ?: return@withLock + runCatching { parse(answer) }.getOrNull()?.let { _bands.value = it } + } + + private fun parse(payload: ByteArray): List { + val info = Wire.parse(payload) + val links = info.repeatedBytes(5).map(Wire::parse).associate { address(it.bytes(2)) to it.varint(1) } + return info.repeatedBytes(3).map(Wire::parse).map { peer -> + val address = address(peer.bytes(1)) + PairedBand(address, peer.varint(2) == 1L, links[address] == LINK_DIRECT) + } + } + + /** BleAddr {2 addr}, least significant byte first. */ + private fun address(bleAddr: ByteArray): String = + Wire.parse(bleAddr).bytes(2).reversed().joinToString(":") { "%02X".format(it) } + + private companion object { + const val SVC = 41109 + const val MT_GET_PAIRING_INFO = 1 + const val MT_PAIRING_INFO = 2 + const val LINK_DIRECT = 2L + const val ANSWER_MS = 8_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandSetup.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandSetup.kt new file mode 100644 index 00000000..64f7020e --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandSetup.kt @@ -0,0 +1,144 @@ +package com.macedonga.visor.band + +import android.annotation.SuppressLint +import android.bluetooth.BluetoothDevice +import android.bluetooth.BluetoothGatt +import android.bluetooth.BluetoothGattCallback +import android.bluetooth.BluetoothManager +import android.bluetooth.BluetoothProfile +import android.content.Context +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.IdentityStore +import com.macedonga.visor.core.identity.provision.WrongOwnerError +import com.macedonga.visor.core.identity.devicePublicKey +import com.macedonga.visor.core.identity.provision.ProvisionStep +import com.macedonga.visor.core.identity.provision.provision +import com.macedonga.visor.core.protocol.Band +import com.macedonga.visor.core.transport.BluetoothLeTransport +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.login.MetaLogin +import com.macedonga.visor.setup.BondFailed +import com.macedonga.visor.setup.NearbyGlasses +import com.macedonga.visor.setup.words +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull + +/** Change owner on the Neural Band, as on the glasses: same svc36 steps, same owner key, over its own PSM. */ +@Singleton +class BandSetup @Inject constructor( + @ApplicationContext private val context: Context, + private val connection: GlassesConnection, + private val store: IdentityStore, + private val bands: Bands, + private val link: BandLink, + private val nearby: NearbyGlasses, + private val login: MetaLogin, + private val log: EventLog, +) { + sealed interface Status { + data object Idle : Status + data class Running(val step: ProvisionStep?) : Status + data class Done(val serial: String) : Status + data class Failed(val reason: Reason, val detail: String = "") : Status + } + + enum class Reason { NO_GLASSES, NOT_PAIRED, NO_ANSWER, WRONG_OWNER, FAILED } + + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val _status = MutableStateFlow(Status.Idle) + val status: StateFlow = _status.asStateFlow() + + fun start() { + if (_status.value is Status.Running) return + _status.value = Status.Running(null) + scope.launch { + val glasses = connection.pair.value + val owned = withContext(Dispatchers.IO) { store.ownerKeyPem() != null } + if (glasses == null || !owned) return@launch fail(Reason.NO_GLASSES) + link.release() + try { + // Not bonded yet: the band heard nearby is bonded here, as the glasses are. + val candidates = bondedBands().ifEmpty { listOfNotNull(nearby.bandNearby()?.let { nearby.bond(it) }) } + if (candidates.isEmpty()) return@launch fail(Reason.NOT_PAIRED) + // A band paired more than once leaves a bond per pairing; the one that answers is the band. + val address = candidates.firstOrNull { reachable(it) } ?: return@launch fail(Reason.NO_ANSWER) + log.add("Band setup: changing owner on $address") + val result = provision( + transport = BluetoothLeTransport(context, address, defaultPsm = Band.PSM), + // No existing key: the band gets one of its own, as the Meta app keeps one per device. + source = login.source, + ) { step -> + _status.value = Status.Running(step) + log.add("Band setup: ${step.words()}") + } + // The key Meta's final receipt names is the one the Display's lease hints use; the certificate's is a fallback. + val publicKey = result.deviceEcPublicKey ?: result.deviceCert?.let(::devicePublicKey) + bands.save(BandRecord(result.serial, address, publicKey, glasses, key = result.keyPem)) + val from = if (result.deviceEcPublicKey != null) "receipt" else if (publicKey != null) "certificate" else "none" + log.add("Band setup: owner changed. Serial ${result.serial}, device key from $from") + _status.value = Status.Done(result.serial) + } catch (failed: BondFailed) { + fail(if (failed.why == BondFailed.Why.NO_ANSWER) Reason.NO_ANSWER else Reason.FAILED, failed.why.name) + } catch (error: Exception) { + val wrong = error is WrongOwnerError + fail(if (wrong) Reason.WRONG_OWNER else Reason.FAILED, error.message ?: error.javaClass.simpleName) + } finally { + link.hold() + } + } + } + + fun reset() { + if (_status.value !is Status.Running) _status.value = Status.Idle + } + + private fun fail(reason: Reason, detail: String = "") { + log.add("Band setup failed: $reason $detail") + _status.value = Status.Failed(reason, detail) + } + + @SuppressLint("MissingPermission") + private fun bondedBands(): List = runCatching { + context.getSystemService(BluetoothManager::class.java)?.adapter?.bondedDevices.orEmpty() + .filter { it.type != BluetoothDevice.DEVICE_TYPE_CLASSIC && it.name?.startsWith(BAND_NAME) == true } + .map { it.address } + }.getOrDefault(emptyList()) + + /** A GATT connection that comes up within a few seconds: this bond is the band's current one. */ + @SuppressLint("MissingPermission") + private suspend fun reachable(address: String): Boolean { + val device = context.getSystemService(BluetoothManager::class.java)?.adapter?.getRemoteDevice(address) ?: return false + val up = CompletableDeferred() + val gatt = device.connectGatt( + context, false, + object : BluetoothGattCallback() { + override fun onConnectionStateChange(g: BluetoothGatt, status: Int, newState: Int) { + up.complete(newState == BluetoothProfile.STATE_CONNECTED) + } + }, + BluetoothDevice.TRANSPORT_LE, + ) ?: return false + return try { + withTimeoutOrNull(PROBE_MS) { up.await() } == true + } finally { + gatt.disconnect() + gatt.close() + } + } + + private companion object { + const val BAND_NAME = "Meta Band" + const val PROBE_MS = 6_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/band/BandUpdate.kt b/app/src/main/kotlin/com/macedonga/visor/band/BandUpdate.kt new file mode 100644 index 00000000..41b5fb7c --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/band/BandUpdate.kt @@ -0,0 +1,364 @@ +package com.macedonga.visor.band + +import com.macedonga.visor.core.identity.token.withToken +import android.content.Context +import android.util.Base64 +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.firmware.BandOffer +import com.macedonga.visor.core.identity.firmware.fetchBandKek +import com.macedonga.visor.core.identity.firmware.fetchBandOffer +import com.macedonga.visor.core.identity.token.Token +import com.macedonga.visor.core.protocol.Band +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.login.MetaLogin +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.File +import java.io.FileOutputStream +import java.net.HttpURLConnection +import java.net.URL +import java.security.MessageDigest +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull +import org.json.JSONObject + +sealed interface BandWork { + data object Idle : BandWork + data object Checking : BandWork + /** [percent] is null until the band acks the first chunk. */ + data class Installing(val percent: Int? = null) : BandWork + data object Installed : BandWork + data class Failed(val reason: String) : BandWork +} + +/** + * The Neural Band's firmware from Meta. [check] asks Meta for the newest image; [install] downloads it with the user's + * login, checks it against its manifest, and sends it over the band's RPC: Ota on RpcRequest 7 (answers 8) and the bulk + * file transfer on 9 (answers 10). The band is held for the whole update, and any failure aborts it. + */ +@Singleton +class BandUpdate @Inject constructor( + @ApplicationContext private val context: Context, + private val login: MetaLogin, + private val connection: GlassesConnection, + private val bands: Bands, + private val link: BandLink, + private val log: EventLog, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val _checked = MutableStateFlow(false) + /** Meta answered for the band's current build, offer or none. */ + val checked: StateFlow = _checked.asStateFlow() + private val _offer = MutableStateFlow(null) + val offer: StateFlow = _offer.asStateFlow() + private val _work = MutableStateFlow(BandWork.Idle) + val work: StateFlow = _work.asStateFlow() + private val transfers = AtomicInteger(1) + private var job: Job? = null + + init { + // An offer is for one band's build: another pair starts over. + scope.launch { + connection.pair.filterNotNull().distinctUntilChanged().drop(1).collect { + _offer.value = null + _checked.value = false + } + } + } + + fun check() { + if (job?.isActive == true) return + job = scope.launch { + _work.value = BandWork.Checking + try { + val serial = serial() + val firmware = link.holdFor { withSession { deviceInfo().firmware } } + // The Meta app never sends a version below 1. + val found = login.source.withToken { fetchBandOffer(it, serial, firmware.coerceAtLeast(1).toString()) } + _offer.value = found + _checked.value = true + _work.value = BandWork.Idle + log.add("Band firmware: " + (found?.let { "${it.targetVersion} is offered" } ?: "no newer build is offered")) + } catch (error: kotlinx.coroutines.CancellationException) { + _work.value = BandWork.Idle + throw error + } catch (error: Throwable) { + log.add("Band firmware: check failed: ${error.message}") + _work.value = BandWork.Failed(reason(error)) + } + } + } + + fun install() { + val offer = _offer.value ?: return + if (job?.isActive == true) return + job = scope.launch { + val zip = File(File(context.cacheDir, "band-update").apply { mkdirs() }, "band.zip") + try { + val serial = serial() + download(offer, zip) + val blobs = unpack(zip) + _work.value = BandWork.Installing() + log.add("Band firmware: installing ${offer.targetVersion}") + login.source.withToken { token -> link.holdFor { installOn(offer, serial, token, blobs) } } + _offer.value = null + _work.value = BandWork.Installed + log.add("Band firmware: ${offer.targetVersion} is installed") + } catch (error: kotlinx.coroutines.CancellationException) { + _work.value = BandWork.Idle + throw error + } catch (error: Throwable) { + log.add("Band firmware: install failed: ${error.message}") + _work.value = BandWork.Failed(reason(error)) + } finally { + zip.delete() + } + } + } + + fun cancel() { + job?.cancel() + } + + /** Back to the start after an install ended. */ + fun dismiss() { + if (job?.isActive != true) _work.value = BandWork.Idle + } + + private fun serial(): String = bands.forGlasses(connection.pair.value)?.serial ?: error("no band is set up with these glasses") + + private suspend fun installOn(offer: BandOffer, serial: String, token: Token, blobs: List) = withSession { + val target = offer.targetVersion.toLongOrNull() ?: error("the offer's version is not a number") + var begun = false + try { + if (battery() < MIN_BATTERY) error("the band's battery is below $MIN_BATTERY%") + var info = deviceInfo() + if (info.unavailable != Band.OTA_NO_REASON) error("the band cannot update now (reason ${info.unavailable})") + if (info.state == Band.OTA_IN_PROGRESS) { + fire(Band.otaAbort(link.nextSeq())) + delay(ABORT_MS) + info = deviceInfo() + if (info.state == Band.OTA_IN_PROGRESS) error("the band is still in an update") + } + if (info.state == Band.OTA_UNAVAILABLE) error("the band is not ready for an update") + + val items = ArrayList() + if (info.firmware != target) { + val firmware = blobs.firstOrNull { it.manifestType == MANIFEST_FIRMWARE } ?: error("the zip has no firmware") + items += Item(Band.OTA_ITEM_FIRMWARE, firmware) + } + blobs.firstOrNull { it.manifestType == MANIFEST_ML }?.let { ml -> + if (ml.bytes.size >= MIN_ML_BYTES && target > info.ml) items += Item(Band.OTA_ITEM_ML_MODEL, ml) + } + if (items.isEmpty()) { + log.add("Band firmware: already on $target") + } else { + offer.kekId?.takeIf { it != info.kek }?.let { kekId -> + val wrapped = fetchBandKek(token, serial, kekId) + if (wrapped.algorithm != null && wrapped.algorithm != "AES_128") log.add("Band firmware: the key names ${wrapped.algorithm}") + begun = true + val answer = call { Band.otaInstallKek(it, kekId, Base64.decode(wrapped.encrypted, Base64.NO_WRAP)) } + expectOk(answer, "its key") + } + val total = items.sumOf { it.blob.bytes.size.toLong() } + val done = AtomicLong() + for (item in items) { + begun = true + val size = item.blob.bytes.size.toLong() + expectOk(call { Band.otaStartItem(it, item.type, target, size) }, "the update") + transfer(item.blob) { n -> + val now = done.addAndGet(n.toLong()) + _work.value = BandWork.Installing((now * 100 / total).toInt()) + } + expectOk(call(VERIFY_MS) { Band.otaVerifyItem(it, item.type, target) }, "the image") + } + expectOk(call(VERIFY_MS) { Band.otaCommit(it) }, "the commit") + } + } catch (error: Throwable) { + if (begun) fire(Band.otaAbort(link.nextSeq())) + throw error + } + } + + /** One image over the bulk transfer: the info request, then 950-byte chunks in the band's window. */ + private suspend fun Session.transfer(blob: Blob, sent: (Int) -> Unit) { + val id = transfers.getAndIncrement() + val size = blob.bytes.size.toLong() + val info = Band.bulk(call { Band.bulkInfo(it, blob.md5, size, id) }) as? Band.Bulk.Info + ?: error("the band did not answer the transfer") + if (info.transferId != id) error("the band answered another transfer") + when (info.state) { + Band.TRANSFER_FULL -> return + Band.TRANSFER_NEW, Band.TRANSFER_PARTIAL -> Unit + else -> error("the band would not take the file (state ${info.state})") + } + val window = (info.memory / Band.CHUNK_BYTES).toInt().coerceAtLeast(1) + sendChunks(id, blob.bytes, info.resume.toInt(), window, sent) + } + + private suspend fun Session.sendChunks(id: Int, bytes: ByteArray, from: Int, window: Int, sent: (Int) -> Unit) = coroutineScope { + val slots = Semaphore(window) + var offset = from + while (offset < bytes.size) { + slots.acquire() + val at = offset + val end = minOf(bytes.size, at + Band.CHUNK_BYTES) + val piece = bytes.copyOfRange(at, end) + launch { + try { + chunk(id, at, piece) + sent(piece.size) + } finally { + slots.release() + } + } + offset = end + } + } + + /** Sends one chunk and waits for its ack; unanswered, it goes again up to three times. */ + private suspend fun Session.chunk(id: Int, at: Int, piece: ByteArray) { + repeat(CHUNK_SENDS) { + val key = id to at + val ack = CompletableDeferred() + acks[key] = ack + try { + if (!link.sendRaw(listOf(Band.bulkChunk(link.nextSeq(), id, at.toLong(), piece)))) error("the band is not up") + if (withTimeoutOrNull(CHUNK_MS) { ack.await() } != null) return + } finally { + acks.remove(key) + } + } + error("the band did not ack the chunk at $at") + } + + /** The answers of one held update: each request's by seq, each chunk's ack by its transfer and offset. */ + private inner class Session { + private val replies = ConcurrentHashMap>() + val acks = ConcurrentHashMap, CompletableDeferred>() + + /** Sends one request and waits for its answer. */ + suspend fun call(timeout: Long = REPLY_MS, build: (Int) -> ByteArray): ByteArray { + val n = link.nextSeq() + val answer = CompletableDeferred() + replies[n] = answer + try { + if (!link.sendRaw(listOf(build(n)))) error("the band is not up") + return withTimeoutOrNull(timeout) { answer.await() } ?: error("the band did not answer") + } finally { + replies.remove(n) + } + } + + fun fire(request: ByteArray) { + link.sendRaw(listOf(request)) + } + + fun route(payload: ByteArray) { + when (val bulk = runCatching { Band.bulk(payload) }.getOrNull()) { + is Band.Bulk.Ack -> acks.remove(bulk.transferId to bulk.offset.toInt())?.complete(Unit) + is Band.Bulk.Nack -> { + val refused = IllegalStateException("the band refused a chunk: ${bulk.reason.ifEmpty { "no reason" }}") + acks.values.forEach { it.completeExceptionally(refused) } + acks.clear() + } + else -> runCatching { Band.seqOf(payload) }.getOrNull()?.let { replies.remove(it)?.complete(payload) } + } + } + } + + private suspend fun withSession(block: suspend Session.() -> T): T = coroutineScope { + val session = Session() + val router = launch(start = CoroutineStart.UNDISPATCHED) { link.answers.collect { session.route(it) } } + try { + session.block() + } finally { + router.cancel() + } + } + + private suspend fun Session.deviceInfo(): Band.OtaInfo = + Band.otaInfo(call { Band.otaDeviceInfo(it) }) ?: error("the band gave no device info") + + private suspend fun Session.battery(): Int = + Band.response(call { Band.batteryRequest(it) }).battery?.percent ?: error("the band gave no battery") + + private fun Session.expectOk(answer: ByteArray, what: String) { + val code = Band.otaResult(answer) ?: error("the band gave no answer for $what") + if (code != Band.OTA_RESULT_OK) error("the band refused $what (code $code)") + } + + private suspend fun download(offer: BandOffer, file: File) = withContext(Dispatchers.IO) { + val c = URL(offer.url).openConnection() as HttpURLConnection + try { + c.connectTimeout = 30_000 + c.readTimeout = 60_000 + if (c.responseCode != 200) error("the download answered ${c.responseCode}") + c.inputStream.use { input -> FileOutputStream(file).use { input.copyTo(it) } } + } finally { + c.disconnect() + } + if (offer.checksum.isNotEmpty() && sha256(file.readBytes()) != offer.checksum.lowercase()) error("the download did not match Meta's checksum") + } + + /** The zip's images, each checked against the SHA-256 in its manifest. They stay encrypted; the band decrypts them. */ + private fun unpack(zip: File): List = ZipFile(zip).use { z -> + val manifest = JSONObject(z.read(z.getEntry("manifest.json") ?: error("the zip has no manifest")).decodeToString()) + val items = manifest.getJSONArray("items") + (0 until items.length()).map { i -> + val item = items.getJSONObject(i) + val path = item.getString("path") + val bytes = z.read(z.getEntry(path) ?: error("the zip has no $path")) + if (sha256(bytes) != item.getString("digest").lowercase()) error("$path does not match its digest") + Blob(item.getInt("item_type"), bytes, MessageDigest.getInstance("MD5").digest(bytes)) + } + } + + private fun ZipFile.read(entry: ZipEntry): ByteArray = getInputStream(entry).use { it.readBytes() } + + private fun sha256(bytes: ByteArray): String = MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) } + + private fun reason(error: Throwable): String = error.message ?: error::class.java.simpleName + + /** An image from the zip; [manifestType] is its item type in the manifest (1 firmware, 2 ML model). */ + private class Blob(val manifestType: Int, val bytes: ByteArray, val md5: ByteArray) + + /** An image as the band's item: [type] is the wire's item type (0 firmware, 2 ML model). */ + private class Item(val type: Int, val blob: Blob) + + private companion object { + const val MANIFEST_FIRMWARE = 1 + const val MANIFEST_ML = 2 + const val MIN_BATTERY = 10 + const val MIN_ML_BYTES = 10240 + // A chunk is sent once and at most three more times when unanswered. + const val CHUNK_SENDS = 4 + // Guesses: nothing in the Meta app's code sets these times. + const val CHUNK_MS = 3_000L + const val REPLY_MS = 5_000L + const val VERIFY_MS = 60_000L + const val ABORT_MS = 2_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/band/Bands.kt b/app/src/main/kotlin/com/macedonga/visor/band/Bands.kt new file mode 100644 index 00000000..7c3bf079 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/band/Bands.kt @@ -0,0 +1,95 @@ +package com.macedonga.visor.band + +import android.content.Context +import android.util.Base64 +import android.util.Log +import com.macedonga.visor.core.identity.KeystoreSealer +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.File +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import org.json.JSONArray +import org.json.JSONObject + +/** + * A Neural Band this phone owns: where it is, the key its lease hints name, the Display it goes with, and its own owner + * key, as the Meta app keeps one per device. + */ +data class BandRecord(val serial: String, val address: String, val publicKey: ByteArray?, val glasses: String?, val key: ByteArray? = null) { + override fun equals(other: Any?): Boolean = + other is BandRecord && serial == other.serial && address == other.address && glasses == other.glasses && + publicKey.contentEquals(other.publicKey) && key.contentEquals(other.key) + + override fun hashCode(): Int = serial.hashCode() +} + +/** The bands set up here, kept apart from the glasses' identity store so the home never takes one for a pair. */ +@Singleton +class Bands @Inject constructor(@ApplicationContext context: Context) { + private val file = File(context.filesDir, "bands.json") + private val sealer = KeystoreSealer(alias = "visor.band") + // Set when bands.json exists but cannot be read; the next write keeps it aside as bands.json.bad first. + private var unreadable = false + private val _all = MutableStateFlow(read()) + val all: StateFlow> = _all.asStateFlow() + + fun forGlasses(serial: String?): BandRecord? = serial?.let { s -> _all.value.firstOrNull { it.glasses == s } } + + @Synchronized + fun save(band: BandRecord) { + // One band per Display, as the Meta app pairs them. + _all.value = _all.value.filter { it.serial != band.serial && (band.glasses == null || it.glasses != band.glasses) } + band + write() + } + + @Synchronized + fun forget(serial: String) { + _all.value = _all.value.filter { it.serial != serial } + write() + } + + private fun read(): List { + if (!file.exists()) return emptyList() + return runCatching { + val list = JSONArray(file.readText()) + (0 until list.length()).map { i -> + val o = list.getJSONObject(i) + BandRecord( + serial = o.getString("serial"), + address = o.getString("address"), + publicKey = o.optString("public_key").takeIf { it.isNotEmpty() }?.let { Base64.decode(it, Base64.NO_WRAP) }, + glasses = o.optString("glasses").takeIf { it.isNotEmpty() }, + key = o.optString("key").takeIf { it.isNotEmpty() }?.let { sealer.unseal(Base64.decode(it, Base64.NO_WRAP)) }, + ) + } + }.getOrElse { + Log.w("Visor", "bands.json cannot be read", it) + unreadable = true + emptyList() + } + } + + private fun write() { + if (unreadable) { + runCatching { file.copyTo(File(file.parentFile, "${file.name}.bad"), overwrite = true) }.onFailure { + Log.w("Visor", "bands.json could not be kept aside", it) + return + } + unreadable = false + } + val list = JSONArray() + for (b in _all.value) { + list.put( + JSONObject().put("serial", b.serial).put("address", b.address).put("glasses", b.glasses ?: "") + .put("public_key", b.publicKey?.let { Base64.encodeToString(it, Base64.NO_WRAP) } ?: "") + .put("key", b.key?.let { Base64.encodeToString(sealer.seal(it), Base64.NO_WRAP) } ?: ""), + ) + } + val tmp = File(file.parentFile, "${file.name}.tmp") + tmp.writeText(list.toString()) + tmp.renameTo(file) + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/band/DeviceLease.kt b/app/src/main/kotlin/com/macedonga/visor/band/DeviceLease.kt new file mode 100644 index 00000000..87f931a8 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/band/DeviceLease.kt @@ -0,0 +1,47 @@ +package com.macedonga.visor.band + +import com.macedonga.visor.core.identity.keyHintMatches +import com.macedonga.visor.core.protocol.NotifyLease +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import javax.inject.Inject +import javax.inject.Singleton + +/** + * svc150, conn_device_lease, as Meta's DeviceLeaseService answers it: the Display names its band by key hint, the phone + * says OK when it owns that band and steps back from it until the channel closes, BadRequest when it does not. + */ +@Singleton +class DeviceLease @Inject constructor( + private val bands: Bands, + private val link: BandLink, + private val log: EventLog, +) : Responder { + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val info = message.info + val reply = DataXConnection.mirror(info.channelId) + if (info.flags.isClosed && message.payload.isEmpty()) { + if (link.state.value.leased) log.add("Band lease: the Display let go") + link.leased(false) + return + } + if (info.messageType != NotifyLease.MESSAGE_TYPE) { + connection.sendStatus(reply, NotifyLease.UNKNOWN_TYPE) + return + } + val lease = runCatching { NotifyLease.decode(message.payload) }.getOrNull() + val band = lease?.let { l -> bands.all.value.firstOrNull { b -> b.publicKey?.let { keyHintMatches(l.keyHint, it) } == true } } + if (lease == null || band == null) { + val hint = lease?.keyHint?.joinToString("") { "%02x".format(it) } + log.add("Band lease: no band here for hint $hint (${bands.all.value.size} known)") + connection.sendStatus(reply, NotifyLease.BAD_REQUEST) + return + } + connection.sendStatus(reply, NotifyLease.OK) + log.add("Band lease: the Display holds ${band.serial} (type ${lease.type})") + if (!lease.relay) link.leased(true) + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt b/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt new file mode 100644 index 00000000..2a45b0bd --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/AccountLinking.kt @@ -0,0 +1,197 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.identity.token.withToken +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.token.LinkedAccountType +import com.macedonga.visor.core.identity.token.TokenError +import com.macedonga.visor.core.identity.token.exchangeAccountToken +import com.macedonga.visor.core.identity.token.LinkedAccountRef +import com.macedonga.visor.core.identity.token.fetchLinkedAccounts +import com.macedonga.visor.core.identity.token.fetchLinkedTokens +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.ServiceData +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.login.MetaLogin +import java.util.UUID +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull + +/** What a link attempt came to, for the Accounts page to say. */ +enum class LinkOutcome { LINKED, LOGGED_OUT, UNREACHABLE, NO_ANSWER, REFUSED, NOT_LINKED } + +/** + * svc89, wearables_whatsapp_linking: the Meta account and WhatsApp linking the Meta app runs with the Display. Visor + * opens the channel. 25 asks whether a Meta account is present (answered by 26). 5 is the Meta account sync, with the + * user's token minted for the glasses' accounts app (answered by 13, after an ack 17 Visor does not wait for). 6 is a + * linked provider account, Facebook or Instagram, with its tokens (answered by 14, after an ack 18). 22 is the WhatsApp + * QR pairing: the glasses show a code for WhatsApp's Linked devices and answer 23 when it is scanned. + */ +@Singleton +class AccountLinking @Inject constructor( + private val connection: GlassesConnection, + private val login: MetaLogin, + private val log: EventLog, +) { + // One exchange at a time: each answer is taken as the next one of its type. + private val lock = Mutex() + + /** Whether the glasses hold a Meta account; null when they do not answer. */ + suspend fun metaPresent(): Boolean? { + val f = ask(MT_PRESENT_REQUEST, MT_PRESENT_RESPONSE, Wire.fieldBytes(1, str(uuid())), ANSWER_MS) ?: return null + return f.varint(2) == 1L + } + + /** Mints the user's token for the accounts app and sends the Meta account to the glasses. */ + suspend fun syncMeta(): LinkOutcome { + if (!login.source.hasCachedLogin) return LinkOutcome.LOGGED_OUT + val identity = try { + withContext(Dispatchers.IO) { login.source.withToken { exchangeAccountToken(it) } } + } catch (error: TokenError) { + log.add("Accounts: ${error.message}") + return if (error.unreachable) LinkOutcome.UNREACHABLE else LinkOutcome.REFUSED + } + val request = Wire.fieldBytes(1, str(uuid())) + + Wire.fieldBytes(2, str(identity.accessToken)) + + Wire.fieldBytes(3, str(identity.appId)) + + Wire.fieldBytes(4, str(identity.userId)) + + Wire.fieldBytes(5, str(identity.analyticsClaim)) + + Wire.fieldBytes(6, str(SOURCE)) + + Wire.fieldBytes(7, str(identity.accountId)) + + Wire.fieldBytes(8, str(identity.profilePictureUrl)) + + Wire.fieldVarint(9, flag(identity.isEmployee)) + + Wire.fieldVarint(10, flag(identity.isTestUser)) + + Wire.fieldBytes(11, str(identity.name)) + + Wire.fieldBytes(12, str(uuid())) + val f = ask(MT_META_SYNC_REQUEST, MT_META_SYNC_RESPONSE, request, ANSWER_MS) + if (f == null) { + log.add("Accounts: the glasses did not answer the Meta account") + return LinkOutcome.NO_ANSWER + } + if (f.varint(2) != 1L) { + log.add("Accounts: the glasses refused the Meta account, code ${f.varint(3)}") + return LinkOutcome.REFUSED + } + log.add("Accounts: Meta account sent to the glasses") + return LinkOutcome.LINKED + } + + /** The provider accounts linked to the user's Meta account, as Meta lists them; null when they cannot be read. */ + suspend fun linkedAccounts(): List? { + if (!login.source.hasCachedLogin) return null + return try { + withContext(Dispatchers.IO) { fetchLinkedAccounts(login.source.withToken { exchangeAccountToken(it) }.accessToken) } + } catch (error: TokenError) { + log.add("Accounts: ${error.message}") + null + } + } + + /** Sends a provider account of the Meta account, with its tokens, to the glasses. */ + suspend fun syncLinked(type: LinkedAccountType): LinkOutcome { + if (!login.source.hasCachedLogin) return LinkOutcome.LOGGED_OUT + val found = try { + withContext(Dispatchers.IO) { + val accountsToken = login.source.withToken { exchangeAccountToken(it) }.accessToken + val account = fetchLinkedAccounts(accountsToken).firstOrNull { it.type == type } ?: return@withContext null + account to (fetchLinkedTokens(accountsToken, account, login.deviceId) ?: return@withContext null) + } + } catch (error: TokenError) { + log.add("Accounts: ${error.message}") + return if (error.unreachable) LinkOutcome.UNREACHABLE else LinkOutcome.REFUSED + } + if (found == null) { + log.add("Accounts: ${type.name.lowercase()} is not linked to this Meta account") + return LinkOutcome.NOT_LINKED + } + val (account, tokens) = found + val credentials = Wire.fieldBytes(1, str(tokens.accessToken)) + + (tokens.firstPartyToken?.let { Wire.fieldBytes(2, str(it)) } ?: ByteArray(0)) + val request = Wire.fieldBytes(1, str(uuid())) + + Wire.fieldVarint(2, type.wire.toLong()) + + Wire.fieldBytes(4, str(account.obfuscatedId)) + + Wire.fieldVarint(5, flag(true)) + + Wire.fieldBytes(6, str(uuid())) + + Wire.fieldBytes(7, credentials) + val f = ask(MT_LINKED_SYNC_REQUEST, MT_LINKED_SYNC_RESPONSE, request, ANSWER_MS) + if (f == null) { + log.add("Accounts: the glasses did not answer the ${type.name.lowercase()} account") + return LinkOutcome.NO_ANSWER + } + if (f.varint(2) != 1L) { + log.add("Accounts: the glasses refused the ${type.name.lowercase()} account, code ${f.varint(3)}") + return LinkOutcome.REFUSED + } + log.add("Accounts: ${type.name.lowercase()} account sent to the glasses") + return LinkOutcome.LINKED + } + + /** Has the glasses show their WhatsApp code; the answer comes once it is scanned from WhatsApp's Linked devices. */ + suspend fun linkWhatsApp(): LinkOutcome { + val f = ask(MT_QR_PAIRING_REQUEST, MT_QR_PAIRING_RESPONSE, Wire.fieldBytes(1, str(uuid())), SCAN_MS) + if (f == null) { + log.add("Accounts: no WhatsApp pairing answer from the glasses") + return LinkOutcome.NO_ANSWER + } + if (f.varint(2) != 1L) { + log.add("Accounts: the glasses did not pair WhatsApp, code ${f.varint(3)}") + return LinkOutcome.REFUSED + } + log.add("Accounts: WhatsApp paired on the glasses") + return LinkOutcome.LINKED + } + + /** Sends one request on svc89 and waits for the answer of [answer] type; null when none comes in [waitMs]. */ + private suspend fun ask(type: Int, answer: Int, payload: ByteArray, waitMs: Long): Wire.Fields? = lock.withLock { + coroutineScope { + val fromGlasses = Channel(Channel.UNLIMITED) + // Subscribed before the request goes out, so the answer is not missed. + val listening = launch(start = CoroutineStart.UNDISPATCHED) { + connection.serviceData.collect { if (it.serviceId == SVC && it.messageType == answer) fromGlasses.send(it) } + } + try { + connection.submit { it.sendOnService(SVC, type, payload) } + withTimeoutOrNull(waitMs) { runCatching { Wire.parse(fromGlasses.receive().payload) }.getOrNull() } + } finally { + listening.cancel() + fromGlasses.close() + } + } + } + + private fun uuid() = UUID.randomUUID().toString() + + private fun str(s: String) = s.encodeToByteArray() + + private fun flag(on: Boolean) = if (on) 1L else 0L + + companion object { + const val SVC = 89 + + // The phone's default for its account and WhatsApp waits. + private const val ANSWER_MS = 20_000L + + // Until the code is scanned; the glasses refresh it while it waits. + private const val SCAN_MS = 180_000L + + // The Meta app's source constant for the account sync. + private const val SOURCE = "306760944872162" + + private const val MT_META_SYNC_REQUEST = 5 + private const val MT_META_SYNC_RESPONSE = 13 + private const val MT_PRESENT_REQUEST = 25 + private const val MT_PRESENT_RESPONSE = 26 + private const val MT_QR_PAIRING_REQUEST = 22 + private const val MT_QR_PAIRING_RESPONSE = 23 + private const val MT_LINKED_SYNC_REQUEST = 6 + private const val MT_LINKED_SYNC_RESPONSE = 14 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/AfterUpdate.kt b/app/src/main/kotlin/com/macedonga/visor/connection/AfterUpdate.kt new file mode 100644 index 00000000..64116901 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/AfterUpdate.kt @@ -0,0 +1,31 @@ +package com.macedonga.visor.connection + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import com.macedonga.visor.core.identity.IdentityStore +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.phone.observeCompanions +import dagger.hilt.android.AndroidEntryPoint +import javax.inject.Inject +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking + +/** + * MY_PACKAGE_REPLACED: an update kills the service, and the glasses already in reach raise no presence event for it. + * Starts the connection again, as [GlassesPresence] does, and asks Android for the presence once more. + */ +@AndroidEntryPoint +class AfterUpdate : BroadcastReceiver() { + + @Inject lateinit var store: IdentityStore + + @Inject lateinit var log: EventLog + + override fun onReceive(context: Context, intent: Intent) { + if (intent.action != Intent.ACTION_MY_PACKAGE_REPLACED) return + observeCompanions(context, log) + val paired = runBlocking(Dispatchers.IO) { store.all().isNotEmpty() } + if (paired) GlassesService.reconnect(context) + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/BloksRelay.kt b/app/src/main/kotlin/com/macedonga/visor/connection/BloksRelay.kt new file mode 100644 index 00000000..0848b59a --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/BloksRelay.kt @@ -0,0 +1,113 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.login.MetaLogin +import java.io.ByteArrayOutputStream +import java.util.zip.GZIPOutputStream +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import org.json.JSONArray +import org.json.JSONObject + +/** + * svc43, smartglasses_graphql_query: the Display's System UI, Utilities and AI History draw some screens from Meta's + * server through the phone, the Glanceables (weather, sports, stocks) among them. A Bloks component request (type 9) or + * action (type 11) names an app id and its params; the phone runs Meta's persisted query with the user's login and + * hands back the bundle gzipped (type 10 or 12). `com.bloks.fox_edge.*` gets an empty answer, as from the Meta app. + */ +@Singleton +class BloksRelay @Inject constructor( + private val login: Provider, + private val requests: RequestLog, + private val log: Provider, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Bloks: failed: $e") }) + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val type = message.info.messageType + val query = QUERIES[type] ?: return + val request = runCatching { Request.decode(message.payload) }.getOrNull() ?: return + val reply = DataXConnection.mirror(message.info.channelId) + scope.launch { serve(request, query, reply, connection) } + } + + private suspend fun serve(r: Request, q: Query, reply: Int, mux: DataXConnection) { + val id = requests.start("bloks://${r.appId}", SOURCE_BLOKS, r.params["params"]?.let { runCatching { JSONObject(it).optString("data_type") }.getOrNull() }.orEmpty()) + if (r.appId.isEmpty() || r.appId.startsWith("com.bloks.fox_edge.")) { + mux.send(reply, ByteArray(0), messageType = q.answer) + requests.finish(id, "empty, as the Meta app answers", failed = false) + return + } + try { + val bundle = fetch(r, q) + val gz = ByteArrayOutputStream().also { out -> GZIPOutputStream(out).use { it.write(bundle.toByteArray()) } }.toByteArray() + mux.send(reply, Wire.fieldBytes(1, gz), messageType = q.answer) + requests.finish(id, "from Meta", failed = false, bytes = gz.size.toLong()) + } catch (error: Exception) { + requests.finish(id, "failed: ${error.message}", failed = true) + runCatching { mux.sendStatus(reply, INTERNAL_ERROR) } + } + } + + private fun fetch(r: Request, q: Query): String { + val params = JSONObject().put("app_id", r.appId).put("params", JSONObject(r.params as Map<*, *>).toString()) + .put("bloks_versioning_id", r.versioningId) + if (r.prebundled) params.put("is_prebundled", true) + val context = JSONObject().put("pixel_ratio", r.pixelRatio).put("bloks_version", r.versioningId) + .put("theme_params", JSONArray().put(JSONObject().put("design_system_name", r.designSystem).put("value", JSONArray(r.themeValues)))) + val variables = JSONObject().put("params", params).put("bk_context", context) + val answer = login.get().graphql(q.name, q.doc, variables.toString()) + val json = JSONObject(answer) + json.optJSONArray("errors")?.let { error(it.optJSONObject(0)?.optString("message") ?: "Meta refused it") } + // data..: the first string under the first object, whatever Meta names them. + val data = json.optJSONObject("data") ?: error(json.optJSONObject("error")?.optString("message") ?: answer.take(300)) + val root = data.keys().asSequence().mapNotNull { data.optJSONObject(it) }.firstOrNull() ?: error("no answer in the reply") + return root.keys().asSequence().mapNotNull { k -> root.opt(k) as? String }.firstOrNull { it.startsWith("{") } ?: error("no bundle in the reply") + } + + private class Query(val name: String, val doc: String, val answer: Int) + + /** BloksAsync*Request: 1 device context {1 versioning id, 2 pixel ratio, 3 theme {1 name, 2 values}}, 3 app id, 4 params, 5 prebundled. */ + private class Request( + val appId: String, + val params: Map, + val versioningId: String, + val pixelRatio: Double, + val designSystem: String, + val themeValues: List, + val prebundled: Boolean, + ) { + companion object { + fun decode(payload: ByteArray): Request { + val f = Wire.parse(payload) + val device = Wire.parse(f.bytes(1)) + val theme = Wire.parse(device.bytes(3)) + val params = f.repeatedBytes(4).associate { e -> Wire.parse(e).let { it.bytes(1).decodeToString() to it.bytes(2).decodeToString() } } + return Request( + f.bytes(3).decodeToString(), params, device.bytes(1).decodeToString(), device.double(2, 1.0), + theme.bytes(1).decodeToString(), theme.repeatedBytes(2).map { it.decodeToString() }, f.varint(5) != 0L, + ) + } + } + } + + companion object { + const val SVC = 43 + private val QUERIES = mapOf( + 9 to Query("CompanionBloksComponentQuery", "37274295535564761374285150560", 10), + 11 to Query("CompanionBloksAsyncActionQuery", "35737516989272377596036696859", 12), + ) + private const val INTERNAL_ERROR = 0xC004 + private const val SOURCE_BLOKS = -2 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/CaptureState.kt b/app/src/main/kotlin/com/macedonga/visor/connection/CaptureState.kt new file mode 100644 index 00000000..1f66cc90 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/CaptureState.kt @@ -0,0 +1,30 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.media.MediaImport +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * svc2000, media_capture_state: the glasses open it on connect and send a ConcurrentSignal (type 1) whose signal type + * is "ready" first, then video, hyperlapse, slow motion or dynamic photo started and stopped. Nothing is asked of the + * phone, so nothing is answered. A stop refreshes the capture list, and AutoImport takes the new capture from there. + */ +@Singleton +class CaptureState @Inject constructor(private val media: Provider) : Responder { + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed || message.info.messageType != MT_CONCURRENT_SIGNAL) return + // ConcurrentSignal field 6 is signal_type; field 5 is the source. + val signalType = Wire.parse(message.payload).bytes(6).decodeToString() + if (signalType.endsWith("_stopped")) media.get().refresh() + } + + companion object { + const val SVC = 2000 + private const val MT_CONCURRENT_SIGNAL = 1 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt b/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt new file mode 100644 index 00000000..690c2096 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/DataFetcher.kt @@ -0,0 +1,321 @@ +package com.macedonga.visor.connection + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.media.MediaMetadataRetriever +import android.net.Uri +import android.util.Base64 +import org.json.JSONObject +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.ByteArrayOutputStream +import java.io.InputStream +import java.net.HttpURLConnection +import java.net.URL +import java.util.concurrent.ConcurrentHashMap +import java.util.zip.DeflaterOutputStream +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlin.math.max +import kotlin.math.roundToInt +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit + +/** + * svc9000, the data fetcher: the Display's apps (Instagram, messaging, the prompter) ask the phone for a URL and get + * its bytes back on the same channel, a ResponseData {1 data, 2 compressed, 3 total} per chunk, then an OK status. + * Plain http(s) streams; with an image transformation the whole image is scaled first. An `instagram://` link wraps a + * web one: `target_url`, with `headers` (base64 JSON) inside it, and `bytes_to_skip`. `sgmessaging://` is message + * media: `whatsapp` with `encrypted=true` is the encrypted file on a whatsapp.net host, which the Display decrypts; + * `phone` is a content URI of the phone's own messages. Messenger's, Instagram's and plain WhatsApp media need those + * apps' own logins and are refused, as are other schemes. + */ +@Singleton +class DataFetcher @Inject constructor( + @ApplicationContext private val context: Context, + private val log: Provider, + private val requests: RequestLog, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Fetch: failed: $e") }) + private val running = ConcurrentHashMap() + private val slots = Semaphore(PARALLEL) + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val channel = message.info.channelId + if (message.info.flags.isClosed) { + running.remove(channel)?.cancel() + if (message.payload.isEmpty()) return + } + if (message.info.messageType != MT_REQUEST) return + val query = runCatching { Query.decode(message.payload) }.getOrNull() ?: return + val reply = DataXConnection.mirror(channel) + // Off the reader: a download must not hold up everything else on the link. + running[channel] = scope.launch { + try { + slots.withPermit { serve(query, reply, connection) } + } finally { + running.remove(channel) + } + } + } + + private suspend fun serve(q: Query, reply: Int, mux: DataXConnection) { + val id = requests.start(q.url, q.source, q.options()) + var sent = 0L + val scheme = q.url.substringBefore(':').lowercase() + val target = when (scheme) { + "http", "https" -> Target(q.url, emptyMap(), 0) + "instagram" -> instagramTarget(q.url) ?: run { + // Instagram's bare check, every few seconds; the Meta app turns it down the same way. + requests.finish(id, "no target_url, turned down", failed = false) + mux.sendStatus(reply, INTERNAL_ERROR) + return + } + "sgmessaging" -> { + val link = Uri.parse(q.url) + val inner = link.getQueryParameter("uri") + when (link.authority) { + "whatsapp" -> { + val host = inner?.let { runCatching { Uri.parse(it) }.getOrNull() } + if (link.getQueryParameter("encrypted") != "true" || host?.scheme != "https" || host.host?.endsWith(".whatsapp.net") != true) { + requests.finish(id, "not served: needs WhatsApp's own login", failed = true) + mux.sendStatus(reply, INTERNAL_ERROR) + return + } + Target(inner, emptyMap(), 0) + } + "phone" -> { + if (inner == null) { + requests.finish(id, "no uri in the link", failed = true) + mux.sendStatus(reply, INTERNAL_ERROR) + } else { + requests.rename(id, inner) + servePhone(Uri.parse(inner), link.getQueryParameter("mimeType").orEmpty(), id, reply, mux) + } + return + } + else -> { + requests.finish(id, "not served: needs ${link.authority}'s own login", failed = true) + mux.sendStatus(reply, INTERNAL_ERROR) + return + } + } + } + else -> { + requests.finish(id, "not served: $scheme links", failed = true) + mux.sendStatus(reply, UNKNOWN_TYPE) + return + } + } + if (target.url != q.url) requests.rename(id, target.url) + val chunk = (if (q.chunkSize > 0) q.chunkSize else DEFAULT_CHUNK).coerceAtMost(MAX_CHUNK) + val c = runCatching { open(target) }.getOrElse { + log.get().add("Fetch: ${host(q.url)} failed: ${it.message}") + requests.finish(id, "failed: ${it.message}", failed = true) + mux.sendStatus(reply, INTERNAL_ERROR) + return + } + try { + val code = c.responseCode + if (code !in 200..299) { + log.get().add("Fetch: ${host(q.url)} answered $code") + requests.finish(id, "HTTP $code", failed = true) + mux.sendStatus(reply, INTERNAL_ERROR) + return + } + if (q.scale == null) { + // From a byte on: a ranged answer says the whole length after its slash; a full one is skipped through. + val ranged = code == 206 + val total = if (ranged) c.getHeaderField("Content-Range")?.substringAfterLast('/')?.toLongOrNull() ?: -1 else c.contentLengthLong + sent = stream(c.inputStream, total, chunk, reply, mux, skip = if (ranged) 0 else target.skip) + } else { + var body = c.inputStream.use { it.readBytes() } + body = scaled(body, q.scale) ?: body + val compressed = q.compress + if (compressed) body = ByteArrayOutputStream().also { out -> DeflaterOutputStream(out).use { it.write(body) } }.toByteArray() + for (at in body.indices step chunk) { + send(mux, reply, body.copyOfRange(at, minOf(body.size, at + chunk)), body.size.toLong(), compressed) + } + sent = body.size.toLong() + } + mux.sendStatus(reply, OK) + requests.finish(id, "HTTP $code" + (if (q.scale != null) ", scaled" else "") + (if (scheme == "instagram") ", for instagram://" else ""), failed = false, bytes = sent) + } catch (error: kotlinx.coroutines.CancellationException) { + requests.finish(id, "cancelled", failed = false, bytes = sent) + throw error + } catch (error: Exception) { + log.get().add("Fetch: ${host(q.url)} broke off: ${error.message}") + requests.finish(id, "broke off: ${error.message}", failed = true, bytes = sent) + runCatching { mux.sendStatus(reply, INTERNAL_ERROR) } + } finally { + c.disconnect() + } + } + + /** A file of the phone's own messages: a video as its first frame, anything else as it is. */ + private suspend fun servePhone(uri: Uri, mime: String, id: Long, reply: Int, mux: DataXConnection) { + try { + val sent = if (mime.startsWith("video/")) { + val frame = MediaMetadataRetriever().run { + try { + setDataSource(context, uri) + getFrameAtTime(0) + } finally { + release() + } + } ?: error("no frame in the video") + val jpeg = ByteArrayOutputStream().also { frame.compress(Bitmap.CompressFormat.JPEG, DEFAULT_QUALITY, it) }.toByteArray() + for (at in jpeg.indices step DEFAULT_CHUNK) send(mux, reply, jpeg.copyOfRange(at, minOf(jpeg.size, at + DEFAULT_CHUNK)), jpeg.size.toLong(), false) + jpeg.size.toLong() + } else { + val input = context.contentResolver.openInputStream(uri) ?: error("it could not be opened") + stream(input, -1, DEFAULT_CHUNK, reply, mux) + } + mux.sendStatus(reply, OK) + requests.finish(id, if (mime.startsWith("video/")) "first frame" else "read", failed = false, bytes = sent) + } catch (error: kotlinx.coroutines.CancellationException) { + throw error + } catch (error: Exception) { + requests.finish(id, "failed: ${error.message}", failed = true) + runCatching { mux.sendStatus(reply, INTERNAL_ERROR) } + } + } + + /** The body as it comes after its first [skip] bytes, a chunk per message; how much went. */ + private suspend fun stream(input: InputStream, total: Long, chunk: Int, reply: Int, mux: DataXConnection, skip: Long = 0): Long { + var sent = 0L + input.use { + var left = skip + while (left > 0) { + val n = it.skip(left) + if (n <= 0) break + left -= n + } + val buffer = ByteArray(chunk) + while (true) { + kotlin.coroutines.coroutineContext.ensureActive() + val n = it.read(buffer) + if (n < 0) break + if (n > 0) { + send(mux, reply, buffer.copyOf(n), total, false) + sent += n + } + } + } + return sent + } + + private suspend fun send(mux: DataXConnection, reply: Int, data: ByteArray, total: Long, compressed: Boolean) { + // An unknown length goes as 0: the field is an int32, and a negative one does not encode here. + val body = Wire.fieldBytes(1, data) + Wire.fieldVarint(2, if (compressed) 1 else 0) + Wire.fieldVarint(3, total.coerceIn(0, Int.MAX_VALUE.toLong())) + mux.send(reply, body, messageType = MT_REQUEST) + } + + private fun open(url: String): HttpURLConnection = open(Target(url, emptyMap(), 0)) + + private fun open(t: Target): HttpURLConnection = (URL(t.url).openConnection() as HttpURLConnection).apply { + for ((k, v) in t.headers) setRequestProperty(k, v) + if (t.skip > 0) setRequestProperty("Range", "bytes=${t.skip}-") + connectTimeout = TIMEOUT_MS + readTimeout = TIMEOUT_MS + instanceFollowRedirects = true + } + + /** The image fitted to the asked size and format; null to send it as it came, also when that is no bigger. */ + private fun scaled(image: ByteArray, s: Scale): ByteArray? { + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(image, 0, image.size, bounds) + if (bounds.outWidth <= 0 || s.width <= 0 || s.height <= 0) return null + var sample = 1 + while (bounds.outWidth / (sample * 2) >= s.width && bounds.outHeight / (sample * 2) >= s.height) sample *= 2 + val decoded = BitmapFactory.decodeByteArray(image, 0, image.size, BitmapFactory.Options().apply { inSampleSize = sample }) ?: return null + val (w, h) = if (s.keepAspect) { + val k = minOf(s.width.toFloat() / decoded.width, s.height.toFloat() / decoded.height, 1f) + max(1, (decoded.width * k).roundToInt()) to max(1, (decoded.height * k).roundToInt()) + } else s.width to s.height + val fitted = if (w == decoded.width && h == decoded.height) decoded else Bitmap.createScaledBitmap(decoded, w, h, true) + val format = when (s.format) { + FORMAT_PNG -> Bitmap.CompressFormat.PNG + FORMAT_WEBP -> Bitmap.CompressFormat.WEBP_LOSSY + else -> Bitmap.CompressFormat.JPEG + } + val out = ByteArrayOutputStream().also { fitted.compress(format, if (s.quality in 1..100) s.quality else DEFAULT_QUALITY, it) }.toByteArray() + return out.takeIf { it.size < image.size } + } + + private fun host(url: String) = runCatching { URL(url).host }.getOrDefault("?") + + /** What to fetch, the headers to send with it, and from which byte. */ + private class Target(val url: String, val headers: Map, val skip: Long) + + private fun instagramTarget(link: String): Target? { + val outer = runCatching { Uri.parse(link) }.getOrNull() ?: return null + val inner = outer.getQueryParameter("target_url")?.takeIf { it.isNotEmpty() } ?: return null + val skip = outer.getQueryParameter("bytes_to_skip")?.toLongOrNull() ?: 0 + val innerUri = Uri.parse(inner) + val headers = innerUri.getQueryParameter("headers")?.let { encoded -> + runCatching { + val json = JSONObject(String(Base64.decode(encoded, Base64.NO_WRAP))) + json.keys().asSequence().associateWith { json.getString(it) } + }.getOrNull() + }.orEmpty() + val url = if (innerUri.getQueryParameter("headers") == null) inner else { + val rebuilt = innerUri.buildUpon().clearQuery() + for (name in innerUri.queryParameterNames) if (name != "headers") innerUri.getQueryParameter(name)?.let { rebuilt.appendQueryParameter(name, it) } + rebuilt.build().toString() + } + return Target(url, headers, skip) + } + + private class Scale(val width: Int, val height: Int, val format: Int, val keepAspect: Boolean, val quality: Int) + + /** RequestQuery: 1 url, 2 compress, 3 transformations (1 image scale), 4 chunk size, 6 who asked. */ + private class Query(val url: String, val compress: Boolean, val scale: Scale?, val chunkSize: Int, val source: Int) { + fun options(): String = listOfNotNull( + scale?.let { s -> "${s.width}x${s.height} ${listOf("jpeg", "png", "webp").getOrElse(s.format) { "format ${s.format}" }}" }, + "zlib".takeIf { compress }, + "chunk $chunkSize".takeIf { chunkSize > 0 }, + ).joinToString(", ") + + companion object { + fun decode(payload: ByteArray): Query { + val f = Wire.parse(payload) + val scale = f.repeatedBytes(3).firstNotNullOfOrNull { t -> + Wire.parse(t).takeIf { it.has(1) }?.let { Wire.parse(it.bytes(1)) }?.let { + Scale(it.varint(1).toInt(), it.varint(2).toInt(), it.varint(3).toInt(), it.varint(5) != 0L, it.varint(6).toInt()) + } + } + return Query(f.bytes(1).decodeToString(), f.varint(2) != 0L, scale, f.varint(4).toInt(), f.varint(6).toInt()) + } + } + } + + companion object { + const val SVC = 9000 + private const val MT_REQUEST = 0 + private const val OK = 0 + private const val INTERNAL_ERROR = 0xC004 + private const val UNKNOWN_TYPE = 0xD001 + private const val DEFAULT_CHUNK = 1024 + // A DataX message holds at most 16383 bytes, header included. + private const val MAX_CHUNK = 16_000 + private const val FORMAT_PNG = 1 + private const val FORMAT_WEBP = 2 + private const val DEFAULT_QUALITY = 85 + private const val PARALLEL = 4 + private const val TIMEOUT_MS = 20_000 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesPresence.kt b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesPresence.kt new file mode 100644 index 00000000..923159cf --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesPresence.kt @@ -0,0 +1,45 @@ +package com.macedonga.visor.connection + +import android.companion.AssociationInfo +import android.companion.CompanionDeviceManager +import android.companion.CompanionDeviceService +import android.companion.DevicePresenceEvent +import androidx.annotation.RequiresApi +import com.macedonga.visor.core.identity.IdentityStore +import dagger.hilt.android.AndroidEntryPoint +import javax.inject.Inject +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking + +/** + * Bound by the system when the associated glasses appear, so the connection starts even with Visor in the background. + * Nothing is done when they disappear: the reconnect ladder handles that. + */ +@AndroidEntryPoint +class GlassesPresence : CompanionDeviceService() { + + @Inject lateinit var store: IdentityStore + + @RequiresApi(36) + override fun onDevicePresenceEvent(event: DevicePresenceEvent) { + if (event.event != DevicePresenceEvent.EVENT_BLE_APPEARED && event.event != DevicePresenceEvent.EVENT_BT_CONNECTED) return + val address = getSystemService(CompanionDeviceManager::class.java).myAssociations + .firstOrNull { it.id == event.associationId }?.deviceMacAddress?.toString() + appeared(address) + } + + @RequiresApi(33) + override fun onDeviceAppeared(associationInfo: AssociationInfo) = appeared(associationInfo.deviceMacAddress?.toString()) + + // Before Android 13, the association is known by its address. + @Suppress("OVERRIDE_DEPRECATION") + override fun onDeviceAppeared(address: String) = appeared(address) + + /** Reaches the pair Visor is for when it is the one at [address]; another pair stays as it is. */ + private fun appeared(address: String?) { + if (address == null) return + val current = runBlocking(Dispatchers.IO) { store.currentSerial()?.let { serial -> store.all().firstOrNull { it.serial == serial } } } ?: return + if (!current.glassesAddress.equals(address, ignoreCase = true)) return + GlassesService.reconnect(this) + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt index 3eb893f9..89bc7e41 100644 --- a/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt +++ b/app/src/main/kotlin/com/macedonga/visor/connection/GlassesService.kt @@ -56,6 +56,12 @@ class GlassesService : Service() { @Inject lateinit var contacts: PhoneContacts + @Inject + lateinit var sharing: PhoneSharing + + @Inject + lateinit var navigation: Navigation + @Inject lateinit var kind: LinkKind @@ -90,6 +96,10 @@ class GlassesService : Service() { @Inject lateinit var outside: com.macedonga.visor.outside.Outside + // Injected so a Display's band is linked while the service runs. + @Inject + lateinit var band: com.macedonga.visor.band.BandLink + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) private var started = false @@ -126,6 +136,8 @@ class GlassesService : Service() { started = true connection.start(scope, targetFrom(intent)) contacts.start(scope) + sharing.start(scope) + navigation.start(scope) moveToLe() holdLe() com.macedonga.visor.gallery.notePlaces(this, scope, connection, media.places) @@ -155,10 +167,22 @@ class GlassesService : Service() { } } else if (intent?.action == ACTION_RECONNECT) { retry() + } else { + targetFrom(intent)?.let(::switchTo) } return START_STICKY } + /** Another pair asked for: the session with this one ends first, as the glasses hold one each. */ + private fun switchTo(target: GlassesTarget) { + if (target.serial == connection.state.value.target?.serial) return + if (live.running.value || media.importing.value) return + scope.launch { + connection.stop() + connection.start(scope, target) + } + } + /** * Asked from the tile: a pair the ladder is waiting to try again is tried now. Never while the * Live or an import has the glasses over Wi-Fi: Bluetooth is paused for them, and a second session ends theirs. @@ -169,7 +193,8 @@ class GlassesService : Service() { if (live.running.value || media.importing.value) return scope.launch { connection.stop() - connection.start(scope, state.target) + // Tried again on Bluetooth: a Wi-Fi link that dropped is not one to wait for. + connection.start(scope, state.target?.takeIf { it.link is GlassesLink.Bluetooth }) } } @@ -197,7 +222,8 @@ class GlassesService : Service() { scope.launch { var last = 0L combine(connection.state, kind.overLe, keeper.connected) { state, le, held -> - held && !le && state.link == LinkState.READY && state.device?.streamingActive != true + // A session moved to Wi-Fi for a lease is not on LE either, and stays where it is. + held && !le && state.link == LinkState.READY && state.device?.streamingActive != true && state.target?.link !is GlassesLink.Wifi } .distinctUntilChanged() .collect { move -> diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt new file mode 100644 index 00000000..a3ee79e5 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/InternetSwitch.kt @@ -0,0 +1,100 @@ +package com.macedonga.visor.connection + +import android.content.Context +import android.net.ConnectivityManager +import android.net.Network +import android.net.NetworkCapabilities +import android.net.LinkProperties +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import java.util.concurrent.atomic.AtomicBoolean +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * svc41108, internet_switch: how the phone's internet is (type 3): connected, metered, and its DNS servers. The Display + * asks on a channel of its own (type 6), but takes the state only on the phone's channel, which the phone opens when the + * Display asks and writes again whenever its network changes. The Display keeps its apps off the phone's internet until + * it hears the phone is connected. + */ +@Singleton +class InternetSwitch @Inject constructor( + @ApplicationContext private val context: Context, + private val connection: Provider, + private val log: Provider, +) : Responder { + private val watching = AtomicBoolean(false) + + /** False keeps the Display off the phone's internet: it is told the phone is offline. */ + @Volatile var sharing = true + set(value) { + field = value + push() + } + + override suspend fun respond(message: DataXMessage, mux: DataXConnection) { + if (message.info.messageType != MT_LISTEN) return + val state = phoneInternet() + log.get().add("Internet: the Display asked; the phone is ${if (state.connected) "online" else "offline"}" + if (state.metered) ", metered" else "") + mux.send(DataXConnection.mirror(message.info.channelId), encode(state), messageType = MT_COMPANION_STATE) + mux.sendOnService(SVC, MT_COMPANION_STATE, encode(state)) + watchNetwork() + } + + /** svc76, wearables_ping: a ping (type 1) is answered with PingResult {1 hasInternet} (type 2). */ + val ping = Responder { message, mux -> + if (message.info.messageType != MT_PING || message.info.flags.isClosed) return@Responder + val online = phoneInternet().connected + mux.send(DataXConnection.mirror(message.info.channelId), Wire.fieldVarint(1, if (online) 1 else 0), messageType = MT_PING_RESULT) + } + + /** From the first question on, every change of the phone's network goes to the Display while it is connected. */ + private fun watchNetwork() { + if (!watching.compareAndSet(false, true)) return + val cm = context.getSystemService(ConnectivityManager::class.java) ?: return + cm.registerDefaultNetworkCallback(object : ConnectivityManager.NetworkCallback() { + override fun onCapabilitiesChanged(network: Network, caps: NetworkCapabilities) = push() + override fun onLinkPropertiesChanged(network: Network, props: LinkProperties) = push() + override fun onLost(network: Network) = push() + }) + } + + private fun push() { + val glasses = connection.get() + if (glasses.state.value.link != LinkState.READY) return + val state = encode(phoneInternet()) + glasses.submit { it.sendOnService(SVC, MT_COMPANION_STATE, state) } + } + + private data class PhoneInternet(val connected: Boolean, val metered: Boolean, val dns: List) + + private fun phoneInternet(): PhoneInternet { + if (!sharing) return PhoneInternet(false, false, emptyList()) + val cm = context.getSystemService(ConnectivityManager::class.java) ?: return PhoneInternet(false, false, emptyList()) + val network = cm.activeNetwork + val caps = network?.let { cm.getNetworkCapabilities(it) } + val online = caps?.hasCapability(NetworkCapabilities.NET_CAPABILITY_VALIDATED) == true + val dns = network?.let { cm.getLinkProperties(it) }?.dnsServers.orEmpty().mapNotNull { it.hostAddress } + return PhoneInternet(online, cm.isActiveNetworkMetered, dns) + } + + private fun encode(s: PhoneInternet): ByteArray = + Wire.fieldVarint(1, if (s.connected) 1 else 0) + Wire.fieldVarint(2, if (s.metered) 1 else 0) + + Wire.fieldBytes(3, s.dns.fold(ByteArray(0)) { acc, a -> acc + Wire.fieldBytes(1, a.encodeToByteArray()) }) + + companion object { + const val SVC = 41108 + private const val MT_COMPANION_STATE = 3 + private const val MT_LISTEN = 6 + const val PING_SVC = 76 + private const val MT_PING = 1 + private const val MT_PING_RESULT = 2 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/IpPackets.kt b/app/src/main/kotlin/com/macedonga/visor/connection/IpPackets.kt new file mode 100644 index 00000000..a3912fd3 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/IpPackets.kt @@ -0,0 +1,175 @@ +package com.macedonga.visor.connection + +import java.net.InetAddress +import java.nio.ByteBuffer + +/** One IP packet as the Display's tunnel carries it: version 4 or 6, TCP or UDP. */ +class IpPacket( + val version: Int, + val src: ByteArray, + val dst: ByteArray, + val protocol: Int, + val transport: ByteArray, +) { + val srcAddress: InetAddress get() = InetAddress.getByAddress(src) + val dstAddress: InetAddress get() = InetAddress.getByAddress(dst) + + companion object { + const val TCP = 6 + const val UDP = 17 + + fun parse(b: ByteArray): IpPacket? { + if (b.isEmpty()) return null + return when (b[0].toInt() ushr 4) { + 4 -> { + if (b.size < 20) return null + val ihl = (b[0].toInt() and 0x0F) * 4 + val total = u16(b, 2).coerceAtMost(b.size) + if (ihl < 20 || total < ihl) return null + // Fragments are not expected inside a 1400-byte tunnel; they are dropped. + if (u16(b, 6) and 0x3FFF != 0) return null + IpPacket(4, b.copyOfRange(12, 16), b.copyOfRange(16, 20), b[9].toInt() and 0xFF, b.copyOfRange(ihl, total)) + } + 6 -> { + if (b.size < 40) return null + val length = u16(b, 4) + // Extension headers are not followed; such packets are dropped. + IpPacket(6, b.copyOfRange(8, 24), b.copyOfRange(24, 40), b[6].toInt() and 0xFF, b.copyOfRange(40, (40 + length).coerceAtMost(b.size))) + } + else -> null + } + } + + /** An IP packet around [transport], its checksums filled in. */ + fun build(version: Int, src: ByteArray, dst: ByteArray, protocol: Int, transport: ByteArray, id: Int): ByteArray { + val sum = checksum(pseudoHeader(version, src, dst, protocol, transport.size), transport) + val at = if (protocol == TCP) 16 else 6 + transport[at] = (sum ushr 8).toByte() + transport[at + 1] = sum.toByte() + return if (version == 4) { + val h = ByteBuffer.allocate(20) + h.put(0x45).put(0).putShort((20 + transport.size).toShort()).putShort(id.toShort()).putShort(0x4000) + h.put(64).put(protocol.toByte()).putShort(0).put(src).put(dst) + val header = h.array() + val hs = checksum(ByteArray(0), header) + header[10] = (hs ushr 8).toByte() + header[11] = hs.toByte() + header + transport + } else { + val h = ByteBuffer.allocate(40) + h.putInt(0x60000000).putShort(transport.size.toShort()).put(protocol.toByte()).put(64).put(src).put(dst) + h.array() + transport + } + } + + private fun pseudoHeader(version: Int, src: ByteArray, dst: ByteArray, protocol: Int, length: Int): ByteArray = + if (version == 4) { + ByteBuffer.allocate(12).put(src).put(dst).put(0).put(protocol.toByte()).putShort(length.toShort()).array() + } else { + ByteBuffer.allocate(40).put(src).put(dst).putInt(length).put(0).put(0).put(0).put(protocol.toByte()).array() + } + + /** The Internet checksum over [a] then [b], the checksum field in [b] taken as zero beforehand. */ + private fun checksum(a: ByteArray, b: ByteArray): Int { + var sum = 0L + fun add(x: ByteArray) { + var i = 0 + while (i + 1 < x.size) { + sum += ((x[i].toInt() and 0xFF) shl 8) or (x[i + 1].toInt() and 0xFF) + i += 2 + } + if (i < x.size) sum += (x[i].toInt() and 0xFF) shl 8 + } + add(a) + add(b) + while (sum ushr 16 != 0L) sum = (sum and 0xFFFF) + (sum ushr 16) + return (sum.inv() and 0xFFFF).toInt() + } + + fun u16(b: ByteArray, at: Int) = ((b[at].toInt() and 0xFF) shl 8) or (b[at + 1].toInt() and 0xFF) + + fun u32(b: ByteArray, at: Int): Long = + ((b[at].toLong() and 0xFF) shl 24) or ((b[at + 1].toLong() and 0xFF) shl 16) or ((b[at + 2].toLong() and 0xFF) shl 8) or (b[at + 3].toLong() and 0xFF) + } +} + +/** A TCP segment's header fields and its data. */ +class TcpSegment( + val srcPort: Int, + val dstPort: Int, + val seq: Long, + val ack: Long, + val flags: Int, + val window: Int, + val data: ByteArray, +) { + val syn get() = flags and SYN != 0 + val fin get() = flags and FIN != 0 + val rst get() = flags and RST != 0 + val hasAck get() = flags and ACK != 0 + + companion object { + const val FIN = 0x01 + const val SYN = 0x02 + const val RST = 0x04 + const val PSH = 0x08 + const val ACK = 0x10 + + fun parse(t: ByteArray): TcpSegment? { + if (t.size < 20) return null + val offset = ((t[12].toInt() and 0xF0) ushr 4) * 4 + if (offset < 20 || offset > t.size) return null + return TcpSegment( + IpPacket.u16(t, 0), IpPacket.u16(t, 2), IpPacket.u32(t, 4), IpPacket.u32(t, 8), + t[13].toInt() and 0x3F, IpPacket.u16(t, 14), t.copyOfRange(offset, t.size), + ) + } + + /** A segment, its checksum left for [IpPacket.build]; [mss] adds the option, for a SYN. */ + fun build(srcPort: Int, dstPort: Int, seq: Long, ack: Long, flags: Int, window: Int, data: ByteArray, mss: Int = 0): ByteArray { + val header = if (mss > 0) 24 else 20 + val b = ByteBuffer.allocate(header + data.size) + b.putShort(srcPort.toShort()).putShort(dstPort.toShort()).putInt(seq.toInt()).putInt(ack.toInt()) + b.put(((header / 4) shl 4).toByte()).put(flags.toByte()).putShort(window.toShort()).putShort(0).putShort(0) + if (mss > 0) b.put(2).put(4).putShort(mss.toShort()) + b.put(data) + return b.array() + } + } +} + +/** UDP: ports and data. */ +class UdpDatagram(val srcPort: Int, val dstPort: Int, val data: ByteArray) { + companion object { + fun parse(u: ByteArray): UdpDatagram? { + if (u.size < 8) return null + val length = IpPacket.u16(u, 4).coerceIn(8, u.size) + return UdpDatagram(IpPacket.u16(u, 0), IpPacket.u16(u, 2), u.copyOfRange(8, length)) + } + + fun build(srcPort: Int, dstPort: Int, data: ByteArray): ByteArray = + ByteBuffer.allocate(8 + data.size).putShort(srcPort.toShort()).putShort(dstPort.toShort()) + .putShort((8 + data.size).toShort()).putShort(0).put(data).array() + } +} + +/** The server name a TLS ClientHello asks for, if [hello] is one. */ +fun tlsServerName(hello: ByteArray): String? = runCatching { + if (hello.size < 43 || hello[0].toInt() != 0x16 || hello[5].toInt() != 0x01) return null + var i = 43 + i += 1 + (hello[i].toInt() and 0xFF) + i += 2 + IpPacket.u16(hello, i) + i += 1 + (hello[i].toInt() and 0xFF) + val end = i + 2 + IpPacket.u16(hello, i) + i += 2 + while (i + 4 <= end && i + 4 <= hello.size) { + val type = IpPacket.u16(hello, i) + val len = IpPacket.u16(hello, i + 2) + if (type == 0) { + val nameLen = IpPacket.u16(hello, i + 7) + return String(hello, i + 9, nameLen, Charsets.US_ASCII) + } + i += 4 + len + } + null +}.getOrNull() diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt b/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt new file mode 100644 index 00000000..d4a73ef8 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/LeaseService.kt @@ -0,0 +1,97 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * svc41107, lease_service: the glasses ask for internet (type 1, InternetLeaseRequest) and for a link (type 4, + * LinkLeaseRequest), and end a lease (type 3, no answer). Internet leases are answered as the Meta app answers them + * (2, use the glasses' own Wi-Fi); a link lease (5) on Bluetooth is granted, and one that needs Wi-Fi is granted + * when the phone can move the session to Wi-Fi ([WifiLink]), refused otherwise. + */ +@Singleton +class LeaseService @Inject constructor( + private val log: Provider, + private val wifiLink: Provider, +) : Responder { + // Leases the glasses hold, by lease id, so an expiry can be named in the log. + private val held = ConcurrentHashMap() + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed) return + val channel = DataXConnection.mirror(message.info.channelId) + val request = Wire.parse(message.payload) + val id = request.bytes(1) + val key = id.joinToString("") { "%02x".format(it) } + when (message.info.messageType) { + MT_INTERNET_REQUEST -> { + val type = INTERNET_TYPES.getOrElse(request.varint(2).toInt()) { "UNKNOWN" } + // As the Meta app does: a block is kept and not answered; any other kind is told to use the glasses' own Wi-Fi. + // The phone's internet still reaches the Display's apps through the tunnel (svc41108, svc41111). + if (type == "BLOCK") { + log.get().add("Lease: the glasses blocked internet leases") + return + } + log.get().add("Lease: the glasses asked for internet ($type); answered: use the glasses' Wi-Fi") + connection.send(channel, Wire.fieldBytes(1, id) + refusal(ERROR_USE_DEVICE_WIFI, "ERROR_CODE_USE_DEVICE_WIFI"), messageType = MT_INTERNET_RESPONSE) + } + MT_LINK_REQUEST -> { + // Field 7 is a link (2 Wi-Fi Direct, 3 SoftAP, 4 local Wi-Fi); field 6 a bandwidth (1 medium, 2 high). + val wifi = when { + request.has(7) -> request.varint(7) in 2..4 + request.has(6) -> request.varint(6) >= 1 + else -> null + } + val unavailable = if (wifi == true) wifiLink.get().refusal() else null + val reply = when { + wifi == null -> Wire.fieldBytes(1, id) + refusal(ERROR_BAD_REQUEST, "Invalid Request") + unavailable != null -> Wire.fieldBytes(1, id) + refusal(ERROR_WIFI_DIRECT_NOT_AVAILABLE, "Wi-Fi is not available for this link.") + else -> Wire.fieldBytes(1, id) + } + log.get().add( + when { + wifi == null -> "Lease: the glasses asked for a link with no requirement; refused" + unavailable != null -> "Lease: the glasses asked for Wi-Fi; refused, $unavailable" + wifi == true -> "Lease: the glasses asked for Wi-Fi; answered OK" + else -> "Lease: the glasses asked for a link on Bluetooth; answered OK" + }, + ) + if (wifi == false) held[key] = "link" + connection.send(channel, reply, messageType = MT_LINK_RESPONSE) + // Answered on the Bluetooth session first: the move to Wi-Fi ends it. + if (wifi == true && unavailable == null) { + held[key] = "Wi-Fi link" + wifiLink.get().hold(key) + } + } + MT_LEASE_EXPIRATION -> { + held.remove(key)?.let { log.get().add("Lease: a $it lease ended") } + wifiLink.get().release(key) + } + else -> log.get().add("Lease: message type ${message.info.messageType} is not handled") + } + } + + private fun refusal(code: Int, text: String): ByteArray = + Wire.fieldBytes(2, Wire.fieldVarint(1, code.toLong()) + Wire.fieldBytes(2, text.encodeToByteArray())) + + companion object { + const val SVC = 41107 + private const val MT_INTERNET_REQUEST = 1 + private const val MT_INTERNET_RESPONSE = 2 + private const val MT_LEASE_EXPIRATION = 3 + private const val MT_LINK_REQUEST = 4 + private const val MT_LINK_RESPONSE = 5 + private val INTERNET_TYPES = listOf("AUTO_BIND", "WARM", "WARM_THEN_IDLE", "BLOCK") + private const val ERROR_BAD_REQUEST = 0 + private const val ERROR_USE_DEVICE_WIFI = 4 + private const val ERROR_WIFI_DIRECT_NOT_AVAILABLE = 104 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/LinkPrompt.kt b/app/src/main/kotlin/com/macedonga/visor/connection/LinkPrompt.kt new file mode 100644 index 00000000..bac39146 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/LinkPrompt.kt @@ -0,0 +1,70 @@ +package com.macedonga.visor.connection + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.content.Context +import android.content.Intent +import com.macedonga.visor.MainActivity +import com.macedonga.visor.R +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * svc70 type 28, TriggerProviderLinking {1 provider, 2 requestId, 3 source, 4 scenario (0 connect, 1 reconnect)}: a + * Display app's "Open on phone". As the Meta app does, the phone posts a notification that opens its linking page, and + * answers type 29 {1 requestId, 2 success}. + */ +@Singleton +class LinkPrompt @Inject constructor( + @ApplicationContext private val context: Context, + private val log: Provider, +) { + suspend fun trigger(request: Wire.Fields, reply: Int, connection: DataXConnection) { + val name = PROVIDERS[request.varint(1)] + val reconnect = request.varint(4) == SCENARIO_RECONNECT + val shown = name != null && show(name, reconnect) + log.get().add("Accounts: the glasses asked to link ${name ?: "provider ${request.varint(1)}"} on the phone" + if (shown) "" else "; not shown") + val answer = Wire.fieldBytes(1, request.bytes(2)) + Wire.fieldVarint(2, if (shown) 1L else 0L) + connection.send(reply, answer, messageType = MT_RESPONSE) + } + + private fun show(name: String, reconnect: Boolean): Boolean { + val manager = context.getSystemService(NotificationManager::class.java) + if (!manager.areNotificationsEnabled()) return false + manager.createNotificationChannel( + NotificationChannel(CHANNEL, context.getString(R.string.link_channel), NotificationManager.IMPORTANCE_DEFAULT), + ) + val open = PendingIntent.getActivity( + context, 2, + Intent(context, MainActivity::class.java).putExtra(MainActivity.EXTRA_OPEN, MainActivity.OPEN_ACCOUNTS).addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP), + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, + ) + val notification = Notification.Builder(context, CHANNEL) + .setSmallIcon(R.drawable.ic_stat_visor) + .setContentTitle(context.getString(if (reconnect) R.string.link_reconnect_title else R.string.link_title, name)) + .setContentText(context.getString(R.string.link_text)) + .setContentIntent(open) + .setAutoCancel(true) + .build() + manager.notify(NOTIFICATION_ID, notification) + return true + } + + companion object { + const val MT_TRIGGER = 28 + private const val MT_RESPONSE = 29 + private const val SCENARIO_RECONNECT = 1L + private const val CHANNEL = "linking" + private const val NOTIFICATION_ID = 7 + + // CommunicateProvider, as the Meta app numbers it. + private val PROVIDERS = mapOf(1L to "Messenger", 2L to "WhatsApp", 4L to "Instagram", 6L to "Facebook", 7L to "Threads") + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/LocationShare.kt b/app/src/main/kotlin/com/macedonga/visor/connection/LocationShare.kt new file mode 100644 index 00000000..8e8bf5d1 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/LocationShare.kt @@ -0,0 +1,122 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.annotation.SuppressLint +import android.content.Context +import android.content.pm.PackageManager +import android.location.Location +import android.location.LocationListener +import android.location.LocationManager +import android.os.Looper +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import dagger.hilt.android.qualifiers.ApplicationContext +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch + +/** + * svc41102, locationservice: the Display asks for the phone's location (type 3) and gets fixes (type 4, + * LocationResult {1 status, 2 Location}) until it says stop (type 5). Its register message on connect (26) needs no + * answer. Without the location permission the answer says so. + */ +@Singleton +class LocationShare @Inject constructor( + @ApplicationContext private val context: Context, + private val requests: RequestLog, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, _ -> }) + private val listening = ConcurrentHashMap() + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val channel = message.info.channelId + if (message.info.flags.isClosed) { + listening.remove(channel)?.stop() + return + } + when (message.info.messageType) { + MT_REQUEST -> if (!listening.containsKey(channel)) { + val precise = runCatching { Wire.parse(message.payload).varint(3) }.getOrDefault(0L) == ACCURACY_BEST + listening[channel] = Subscription(connection, DataXConnection.mirror(channel), precise).also { it.start() } + } + MT_STOP -> listening.remove(channel)?.stop() + } + } + + private inner class Subscription(private val mux: DataXConnection, private val reply: Int, private val precise: Boolean) : LocationListener { + private val manager = context.getSystemService(LocationManager::class.java) + private val logId = requests.start("location://${if (precise) "precise" else "approximate"}", SOURCE_LOCATION) + private var fixes = 0L + + @SuppressLint("MissingPermission") + fun start() { + val permission = if (precise) Manifest.permission.ACCESS_FINE_LOCATION else Manifest.permission.ACCESS_COARSE_LOCATION + if (manager == null || context.checkSelfPermission(permission) != PackageManager.PERMISSION_GRANTED) { + send(Wire.fieldVarint(1, STATUS_NO_PERMISSION)) + requests.finish(logId, "no location permission", failed = true) + return + } + val provider = listOf(LocationManager.FUSED_PROVIDER, LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER, LocationManager.PASSIVE_PROVIDER) + .firstOrNull { p -> (p != LocationManager.GPS_PROVIDER || precise) && manager.allProviders.contains(p) && manager.isProviderEnabled(p) } + if (provider == null) { + send(Wire.fieldVarint(1, STATUS_OTHER_ERROR)) + requests.finish(logId, "no location provider enabled", failed = true) + return + } + try { + manager.getLastKnownLocation(provider)?.let { onLocationChanged(it) } + manager.requestLocationUpdates(provider, INTERVAL_MS, 0f, this, Looper.getMainLooper()) + } catch (error: RuntimeException) { + send(Wire.fieldVarint(1, STATUS_OTHER_ERROR)) + requests.finish(logId, "location updates refused: ${error.message}", failed = true) + } + } + + fun stop() { + manager?.removeUpdates(this) + requests.finish(logId, "stopped after $fixes fixes", failed = false) + } + + override fun onLocationChanged(location: Location) { + fixes++ + send(Wire.fieldVarint(1, STATUS_OK) + Wire.fieldBytes(2, encode(location))) + } + + private fun send(body: ByteArray) { + scope.launch { runCatching { mux.send(reply, body, messageType = MT_RESULT) } } + } + } + + /** Location: 1 latitude, 2 longitude, 3 accuracy m, 5 time ms, 8 altitude, 9 its accuracy, 10 bearing, 11 its accuracy, 12 speed, 13 its accuracy. */ + private fun encode(l: Location): ByteArray { + var b = Wire.fieldDouble(1, l.latitude) + Wire.fieldDouble(2, l.longitude) + Wire.fieldFloat(3, l.accuracy) + Wire.fieldVarint(5, l.time) + if (l.hasAltitude()) b += Wire.fieldDouble(8, l.altitude) + if (l.hasVerticalAccuracy()) b += Wire.fieldFloat(9, l.verticalAccuracyMeters) + if (l.hasBearing()) b += Wire.fieldFloat(10, l.bearing) + if (l.hasBearingAccuracy()) b += Wire.fieldFloat(11, l.bearingAccuracyDegrees) + if (l.hasSpeed()) b += Wire.fieldFloat(12, l.speed) + if (l.hasSpeedAccuracy()) b += Wire.fieldFloat(13, l.speedAccuracyMetersPerSecond) + return b + } + + companion object { + const val SVC = 41102 + private const val MT_REQUEST = 3 + private const val MT_RESULT = 4 + private const val MT_STOP = 5 + private const val ACCURACY_BEST = 0L + private const val STATUS_OK = 0L + private const val STATUS_NO_PERMISSION = 1L + // ERROR_OTHER in LocationResult: no provider can answer (notes/213, location-to-glasses.md:191). + private const val STATUS_OTHER_ERROR = 2L + private const val INTERVAL_MS = 2_000L + private const val SOURCE_LOCATION = -3 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/MapsSupport.kt b/app/src/main/kotlin/com/macedonga/visor/connection/MapsSupport.kt new file mode 100644 index 00000000..9233d1ab --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/MapsSupport.kt @@ -0,0 +1,64 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * svc11020, maps_support. The glasses ask for saved and recent places, survey and feature state, third-party routes + * and Tessa events; each request gets an empty answer with result 0, so the Display never waits on silence. Location + * signals (0) and trip feedback (1) are notes with no answer. + */ +@Singleton +class MapsSupport @Inject constructor() : Responder { + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed) return + val reply = REPLIES[message.info.messageType] ?: return + // Route details (4 -> 5) have no decoded reply, so theirs is empty. + val payload = if (message.info.messageType == REQUEST_ROUTE_DETAILS) ByteArray(0) else EMPTY_RESULT + connection.send(DataXConnection.mirror(message.info.channelId), payload, messageType = reply) + } + + companion object { + const val SVC = 11020 + private val EMPTY_RESULT = Wire.fieldVarint(1, 0) + private const val REQUEST_ROUTE_DETAILS = 4 + + /** Request type to the reply type the glasses wait for: the phone's table, reply = request + 1. */ + private val REPLIES = mapOf( + 2 to 3, + 4 to 5, + 6 to 7, + 8 to 9, + 10 to 11, + 12 to 13, + 14 to 15, + 16 to 17, + 18 to 19, + ) + } +} + +/** + * svc87 (persistent store) and svc11004 (constellation): accepted and never answered, since their schemas are native + * and undecoded. The first message of each is logged once. + */ +@Singleton +class MapsQuiet @Inject constructor(private val log: Provider) { + private val logged = ConcurrentHashMap.newKeySet() + + fun responder(svc: Int): Responder = Responder { _, _ -> + if (logged.add(svc)) log.get().add("Maps: service $svc opened, not served") + } + + companion object { + const val SVC_PERSISTENT_STORE = 87 + const val SVC_CONSTELLATION = 11004 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/Mms.kt b/app/src/main/kotlin/com/macedonga/visor/connection/Mms.kt new file mode 100644 index 00000000..57700ad7 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/Mms.kt @@ -0,0 +1,154 @@ +package com.macedonga.visor.connection + +import android.app.Activity +import android.app.PendingIntent +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.graphics.Bitmap +import android.graphics.ImageDecoder +import android.os.Build +import android.telephony.SmsManager +import androidx.core.content.ContextCompat +import androidx.core.content.FileProvider +import java.io.ByteArrayOutputStream +import java.io.File +import java.util.UUID +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.withTimeoutOrNull + +/** + * One picture, with an optional caption, as an MMS through Android's MMS service: an M-Send.req (OMA MMS 1.2) with a + * multipart/related body of SMIL, the JPEG and the text. Android files what a non-default app sends in the SMS store. + */ +object Mms { + /** Sends [image] to [to]; null when the carrier took it, else why not. */ + suspend fun send(context: Context, to: String, image: ByteArray, caption: String): String? { + val jpeg = fit(image, maxBytes(context)) ?: return "The picture could not be read" + val pdu = sendReq(to, jpeg, caption) + val dir = File(context.cacheDir, "mms").apply { mkdirs() } + val file = File(dir, "${UUID.randomUUID()}.pdu").apply { writeBytes(pdu) } + val uri = FileProvider.getUriForFile(context, "${context.packageName}.files", file) + // The MMS service reads the message from the uri in its own process. + for (pkg in listOf("com.android.phone", "com.android.mms.service")) { + runCatching { context.grantUriPermission(pkg, uri, Intent.FLAG_GRANT_READ_URI_PERMISSION) } + } + val action = "${context.packageName}.MMS_SENT.${UUID.randomUUID()}" + val result = CompletableDeferred() + val receiver = object : BroadcastReceiver() { + override fun onReceive(c: Context, intent: Intent) { + result.complete(resultCode) + } + } + ContextCompat.registerReceiver(context, receiver, IntentFilter(action), ContextCompat.RECEIVER_NOT_EXPORTED) + return try { + val sent = PendingIntent.getBroadcast(context, 0, Intent(action).setPackage(context.packageName), PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_ONE_SHOT) + val sms = if (Build.VERSION.SDK_INT >= 31) context.getSystemService(SmsManager::class.java) else @Suppress("DEPRECATION") SmsManager.getDefault() + sms.sendMultimediaMessage(context, uri, null, null, sent) + when (val code = withTimeoutOrNull(SEND_TIMEOUT_MS) { result.await() }) { + null -> "No answer from the phone's MMS service" + Activity.RESULT_OK -> null + else -> "Not sent (MMS error $code)" + } + } catch (e: Exception) { + e.message ?: e.toString() + } finally { + context.unregisterReceiver(receiver) + file.delete() + } + } + + // SmsManager's MMS config needs no permission; the carrier config would need the phone state. + private fun maxBytes(context: Context): Int { + val sms = if (Build.VERSION.SDK_INT >= 31) context.getSystemService(SmsManager::class.java) else @Suppress("DEPRECATION") SmsManager.getDefault() + val max = runCatching { sms.carrierConfigValues.getInt(SmsManager.MMS_CONFIG_MAX_MESSAGE_SIZE, 0) }.getOrDefault(0) + return (if (max > 0) max else DEFAULT_MAX_BYTES) - PDU_ROOM + } + + /** The picture as a JPEG under [maxBytes]: scaled to at most 1600 px, then lower quality until it fits. */ + private fun fit(image: ByteArray, maxBytes: Int): ByteArray? { + val bitmap = runCatching { + ImageDecoder.decodeBitmap(ImageDecoder.createSource(java.nio.ByteBuffer.wrap(image))) { decoder, info, _ -> + val side = maxOf(info.size.width, info.size.height) + if (side > MAX_SIDE) decoder.setTargetSampleSize((side + MAX_SIDE - 1) / MAX_SIDE) + decoder.allocator = ImageDecoder.ALLOCATOR_SOFTWARE + } + }.getOrNull() ?: return null + var scaled = bitmap + while (true) { + for (quality in intArrayOf(85, 70, 55, 40)) { + val out = ByteArrayOutputStream() + scaled.compress(Bitmap.CompressFormat.JPEG, quality, out) + if (out.size() <= maxBytes) return out.toByteArray() + } + if (scaled.width < 320) return null + scaled = Bitmap.createScaledBitmap(scaled, scaled.width * 3 / 4, scaled.height * 3 / 4, true) + } + } + + private fun sendReq(to: String, jpeg: ByteArray, caption: String): ByteArray { + val out = ByteArrayOutputStream() + out.write(byteArrayOf(0x8C.toByte(), 0x80.toByte())) // X-Mms-Message-Type: m-send-req + out.write(0x98) // X-Mms-Transaction-ID + out.write(text(UUID.randomUUID().toString().replace("-", ""))) + out.write(byteArrayOf(0x8D.toByte(), 0x92.toByte())) // X-Mms-MMS-Version: 1.2 + out.write(byteArrayOf(0x89.toByte(), 0x01, 0x81.toByte())) // From: insert-address-token + out.write(0x97) // To + out.write(text("${to.filter { it.isDigit() || it == '+' }}/TYPE=PLMN")) + // Content-Type, last of the headers: multipart/related, start , type application/smil. + out.write(0x84) + out.write(valueLength(byteArrayOf(0xB3.toByte()) + byteArrayOf(0x8A.toByte()) + text("") + byteArrayOf(0x89.toByte()) + text("application/smil"))) + + val parts = mutableListOf( + part(text("application/smil"), "smil.xml", "", smil(caption.isNotBlank()).encodeToByteArray()), + // image/jpeg; name=image.jpg + part(valueLength(byteArrayOf(0x9E.toByte(), 0x85.toByte()) + text("image.jpg")), "image.jpg", "", jpeg), + ) + if (caption.isNotBlank()) { + // text/plain; charset=utf-8 + parts += part(valueLength(byteArrayOf(0x83.toByte(), 0x81.toByte(), 0xEA.toByte())), "text.txt", "", caption.encodeToByteArray()) + } + out.write(uintvar(parts.size.toLong())) + parts.forEach { out.write(it) } + return out.toByteArray() + } + + /** One body part: its encoded content type, then Content-Location and Content-ID, then the data. */ + private fun part(contentType: ByteArray, location: String, id: String, data: ByteArray): ByteArray { + val headers = contentType + byteArrayOf(0x8E.toByte()) + text(location) + byteArrayOf(0xC0.toByte(), 0x22) + id.encodeToByteArray() + byteArrayOf(0) + return uintvar(headers.size.toLong()) + uintvar(data.size.toLong()) + headers + data + } + + private fun smil(withText: Boolean): String = + "" + + "" + + "" + + "" + + (if (withText) "" else "") + + "" + + /** Text-string: a leading byte at or above 0x80 is quoted, and the string ends with a zero. */ + private fun text(s: String): ByteArray { + val b = s.encodeToByteArray() + return (if (b.isNotEmpty() && (b[0].toInt() and 0xFF) >= 0x80) byteArrayOf(0x7F) else ByteArray(0)) + b + byteArrayOf(0) + } + + private fun valueLength(value: ByteArray): ByteArray = + (if (value.size <= 30) byteArrayOf(value.size.toByte()) else byteArrayOf(31) + uintvar(value.size.toLong())) + value + + private fun uintvar(v: Long): ByteArray { + var n = v + val groups = ArrayList() + do { + groups.add((n and 0x7F).toInt()) + n = n shr 7 + } while (n > 0) + return ByteArray(groups.size) { i -> (groups[groups.size - 1 - i] or if (i < groups.size - 1) 0x80 else 0).toByte() } + } + + private const val DEFAULT_MAX_BYTES = 300 * 1024 + private const val PDU_ROOM = 8 * 1024 + private const val MAX_SIDE = 1600 + private const val SEND_TIMEOUT_MS = 120_000L +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/MotionActivity.kt b/app/src/main/kotlin/com/macedonga/visor/connection/MotionActivity.kt new file mode 100644 index 00000000..36966726 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/MotionActivity.kt @@ -0,0 +1,26 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder + +/** + * svc101, motion_activity_service: the Display's driving detection subscribes (type 0) and asks (type 2) whether the + * phone sees its owner driving. Visor has no activity recognition, so it answers every message as the Meta app does + * with driving detection off: MotionActivityServiceError {1 SETTINGS_NOT_ENABLED} (type 3), read as "not driving". + */ +object MotionActivity : Responder { + const val SVC = 101 + private const val MT_SERVICE_ERROR = 3 + private const val SETTINGS_NOT_ENABLED = 2L + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed) return + connection.send( + DataXConnection.mirror(message.info.channelId), + Wire.fieldVarint(1, SETTINGS_NOT_ENABLED), + messageType = MT_SERVICE_ERROR, + ) + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/Navigation.kt b/app/src/main/kotlin/com/macedonga/visor/connection/Navigation.kt new file mode 100644 index 00000000..5f12689c --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/Navigation.kt @@ -0,0 +1,321 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.settings.hasDisplay +import java.util.UUID +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlin.math.max +import kotlin.math.roundToLong +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.launch + +/** + * svc42, smartglasses_navigation, on the Display. The phone opens the channel and sends NavInitChannelRequest (type 0, + * {1 platform, Android 1}); the glasses answer (type 1), then both sides send NavMessage (type 2) {1 msgType, and a + * oneof whose case is the field number}. The glasses ask for a route (ROUTE_REQUEST, 0) and for guidance (NAV_REQUEST, + * 2); Visor answers with ROUTE_RESPONSE (1), then NAV_UPDATE (3) each second with the phone's snapped position, a + * MANEUVER_NOTIFICATION (8) per turn, REROUTE_UPDATE (7) after a re-route, and KEEPALIVE (17) on a timer. The route + * is OpenStreetMap's (see [OsrmRouter]); its line goes out as that router's encoded polyline. The map tiles are the + * glasses' own. + */ +@Singleton +class Navigation @Inject constructor( + private val glasses: Provider, + private val position: PhonePosition, + private val router: OsrmRouter, + private val requests: RequestLog, + private val log: Provider, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default + CoroutineExceptionHandler { _, e -> log.get().add("Navigation: failed: $e") }) + + @Volatile + private var session: Session? = null + + /** Opens the channel once a Display is linked, and drops the session when the link goes. */ + fun start(owner: CoroutineScope) { + owner.launch { + glasses.get().state.map { it.link == LinkState.READY && hasDisplay(it.info?.build) }.distinctUntilChanged().collect { ready -> + if (ready) { + glasses.get().submit { it.sendOnService(SVC, MT_INIT_REQUEST, Wire.fieldVarint(1, PLATFORM_ANDROID)) } + } else { + endSession() + } + } + } + } + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed) return endSession() + if (message.info.messageType != MT_NAV_MESSAGE) return + val f = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return + // The channel is Visor's own: the glasses answer on it without the originator bit, and Visor writes it with. + val s = session ?: Session(connection, message.info.channelId xor 0x8000).also { session = it } + when (f.varint(1).toInt()) { + MSG_ROUTE_REQUEST -> scope.launch { routeRequest(f, s) } + MSG_NAV_REQUEST -> startGuidance(s) + MSG_STOP_NAVIGATION -> stopGuidance(s, "stopped") + MSG_APP_START -> keepAlive(s) + MSG_APP_EXIT, MSG_APP_DIED -> endSession() + } + } + + private suspend fun routeRequest(f: Wire.Fields, s: Session) { + val req = Wire.parse(f.bytes(2)) + val requestId = req.bytes(2) + val place = Wire.parse(req.bytes(1)) + val geo = Wire.parse(place.bytes(4)) + val title = place.bytes(1).decodeToString().ifBlank { "place" } + val to = if (geo.has(1)) LatLng(geo.double(1), geo.double(2)) else null + val logId = requests.start("route://$title", SOURCE_MAPS, "walking") + if (s.guiding) return answerRoute(s, requestId, null, "Error: Navigation in progress.", logId) + val from = position.current() + if (to == null || from == null) return answerRoute(s, requestId, null, "No phone location or destination", logId) + val route = try { + router.route(from, to) + } catch (e: Exception) { + return answerRoute(s, requestId, null, "No route found", logId, e.message) + } + s.route = route + s.destination = to + s.title = title + s.spot = Spot(route.line[0], 0, 0.0) + answerRoute(s, requestId, route, null, logId) + } + + private suspend fun answerRoute(s: Session, requestId: ByteArray, route: Route?, error: String?, logId: Long, detail: String? = null) { + val body = if (route != null) { + Wire.fieldVarint(1, RESULT_SUCCESS) + Wire.fieldVarint(2, SOURCE_GLASS) + Wire.fieldBytes(3, routeDetails(route)) + Wire.fieldBytes(6, requestId) + } else { + Wire.fieldVarint(1, RESULT_ERROR) + Wire.fieldVarint(2, SOURCE_MWA) + Wire.fieldBytes(4, (error ?: "").encodeToByteArray()) + Wire.fieldBytes(6, requestId) + } + s.mux.send(s.reply, nav(MSG_ROUTE_RESPONSE, CASE_ROUTE_RESPONSE, body), MT_NAV_MESSAGE) + if (route != null) { + requests.finish(logId, "${route.steps.size} steps, ${(route.metres / 100).roundToLong() / 10.0} km", failed = false) + log.get().add("Navigation: route to ${s.title}, ${route.steps.size} maneuvers") + } else { + requests.finish(logId, detail ?: error.orEmpty(), failed = true) + log.get().add("Navigation: no route: ${error.orEmpty()}") + } + } + + private suspend fun startGuidance(s: Session) { + val route = s.route ?: return log.get().add("Navigation: guidance asked with no route") + if (s.guiding) return + s.guiding = true + s.notified = -1 + s.offTicks = 0 + s.logId = requests.start("guide://${s.title}", SOURCE_MAPS, "walking") + position.use() + s.spot = Spot(route.line[0], 0, 0.0) + sendUpdate(s, route, STATE_STARTED, NAVIGATOR_ON_ROUTE, 0) + s.ticker = scope.launch { + while (true) { + delay(TICK_MS) + if (!tick(s)) break + } + } + } + + /** One second of guidance. Returns false once guidance has ended. */ + private suspend fun tick(s: Session): Boolean { + if (!s.guiding) return false + val route = s.route ?: return false + val here = position.current() + if (here == null) { + sendUpdate(s, route, STATE_IN_PROGRESS, NAVIGATOR_BAD_GPS, max(s.notified, 0)) + return true + } + val snap = route.snap(here) + val offRoute = snap.off > OFF_ROUTE_M + s.offTicks = if (offRoute) s.offTicks + 1 else 0 + if (s.offTicks >= REROUTE_TICKS) { + reroute(s, here) + return s.guiding + } + s.spot = Spot(snap.at, snap.index, snap.along) + if (route.metres - snap.along < ARRIVE_M) { + sendUpdate(s, route, STATE_ARRIVED, NAVIGATOR_STOPPED, route.steps.lastIndex) + finish(s, "arrived") + return false + } + val steps = route.steps + val next = steps.indexOfFirst { snap.along < it.along + PASSED_M }.let { if (it < 0) steps.lastIndex else it } + sendUpdate(s, route, STATE_IN_PROGRESS, if (offRoute) NAVIGATOR_OFF_ROUTE else NAVIGATOR_ON_ROUTE, next) + if (next != s.notified) { + s.notified = next + s.mux.send(s.reply, nav(MSG_MANEUVER_NOTIFICATION, CASE_MANEUVER, notification(steps[next])), MT_NAV_MESSAGE) + } + return true + } + + private suspend fun reroute(s: Session, here: LatLng) { + val to = s.destination ?: return + s.offTicks = 0 + sendUpdate(s, s.route ?: return, STATE_IN_PROGRESS, NAVIGATOR_REROUTING, max(s.notified, 0)) + val fresh = try { + router.route(here, to) + } catch (e: Exception) { + log.get().add("Navigation: reroute failed: ${e.message}") + s.mux.send(s.reply, nav(MSG_REROUTE_UPDATE, CASE_REROUTE, Wire.fieldVarint(1, RESULT_ERROR)), MT_NAV_MESSAGE) + return + } + s.route = fresh + s.spot = Spot(fresh.line[0], 0, 0.0) + s.notified = -1 + s.mux.send(s.reply, nav(MSG_REROUTE_UPDATE, CASE_REROUTE, Wire.fieldVarint(1, RESULT_SUCCESS) + Wire.fieldBytes(2, routeDetails(fresh))), MT_NAV_MESSAGE) + log.get().add("Navigation: rerouted, ${fresh.steps.size} maneuvers") + } + + private suspend fun stopGuidance(s: Session, outcome: String) { + if (!s.guiding) return + s.ticker?.cancel() + s.ticker = null + s.route?.let { sendUpdate(s, it, STATE_STOPPED, NAVIGATOR_STOPPED, max(s.notified, 0)) } + finish(s, outcome) + } + + /** Ends guidance's bookkeeping. Only called while guidance runs. */ + private fun finish(s: Session, outcome: String) { + s.guiding = false + position.release() + requests.finish(s.logId, outcome, failed = false) + log.get().add("Navigation: $outcome") + } + + private fun keepAlive(s: Session) { + s.keepAlive?.cancel() + s.keepAlive = scope.launch { + while (true) { + s.mux.send(s.reply, nav(MSG_KEEPALIVE), MT_NAV_MESSAGE) + delay(KEEPALIVE_MS) + } + } + } + + private suspend fun endSession() { + val s = session ?: return + session = null + s.keepAlive?.cancel() + runCatching { stopGuidance(s, "ended by the link") } + } + + /** NAV_UPDATE for the spot, with the next maneuver's distance and time, as the glasses' guidance needs them. */ + private suspend fun sendUpdate(s: Session, route: Route, state: Long, navigator: Long, maneuver: Int) { + val spot = s.spot + val remaining = max(0.0, route.metres - spot.along) + val next = route.steps.getOrNull(maneuver) + val toNext = max(0.0, (next?.along ?: route.metres) - spot.along) + val perMetre = route.secondsPerMetre + val body = Wire.fieldVarint(1, state) + + Wire.fieldBytes(2, Wire.fieldDouble(1, spot.at.lat) + Wire.fieldDouble(2, spot.at.lng)) + + Wire.fieldVarint(3, spot.index.toLong()) + + Wire.fieldVarint(4, maneuver.toLong()) + + Wire.fieldVarint(5, (remaining * perMetre).roundToLong()) + + Wire.fieldVarint(6, remaining.roundToLong()) + + Wire.fieldVarint(7, (toNext * perMetre).roundToLong()) + + Wire.fieldVarint(8, toNext.roundToLong()) + + Wire.fieldBytes(10, (next?.text ?: "").encodeToByteArray()) + + Wire.fieldVarint(13, navigator) + s.mux.send(s.reply, nav(MSG_NAV_UPDATE, CASE_NAV_UPDATE, body), MT_NAV_MESSAGE) + } + + private fun routeDetails(r: Route): ByteArray { + var b = Wire.fieldVarint(1, ROUTE_ID) + Wire.fieldVarint(2, r.seconds.roundToLong()) + Wire.fieldVarint(3, r.metres.roundToLong()) + for (step in r.steps) b += Wire.fieldBytes(4, maneuver(step)) + return b + Wire.fieldBytes(5, r.encoded.encodeToByteArray()) + } + + private fun maneuver(step: Step): ByteArray = + Wire.fieldVarint(1, step.direction.toLong()) + + Wire.fieldBytes(2, step.street.encodeToByteArray()) + + Wire.fieldVarint(3, step.metres.toLong()) + + Wire.fieldVarint(4, step.seconds.toLong()) + + Wire.fieldBytes(5, step.text.encodeToByteArray()) + + private fun notification(step: Step): ByteArray = + Wire.fieldBytes(1, maneuver(step)) + Wire.fieldBytes(2, "HNNAV-${UUID.randomUUID().toString().take(8)}".encodeToByteArray()) + + private fun nav(type: Int, case: Int = 0, body: ByteArray = ByteArray(0)): ByteArray = + Wire.fieldVarint(1, type.toLong()) + if (case == 0) ByteArray(0) else Wire.fieldBytes(case, body) + + private class Spot(val at: LatLng, val index: Int, val along: Double) + + /** The channel's state: where replies go, the route and the guidance running on it. */ + private class Session(val mux: DataXConnection, val reply: Int) { + @Volatile var route: Route? = null + @Volatile var destination: LatLng? = null + @Volatile var title = "" + @Volatile var guiding = false + @Volatile var logId = 0L + @Volatile var spot = Spot(LatLng(0.0, 0.0), 0, 0.0) + @Volatile var notified = -1 + @Volatile var offTicks = 0 + var ticker: Job? = null + var keepAlive: Job? = null + } + + companion object { + const val SVC = 42 + private const val SOURCE_MAPS = -6 + private const val MT_INIT_REQUEST = 0 + private const val MT_NAV_MESSAGE = 2 + private const val PLATFORM_ANDROID = 1L + + private const val MSG_ROUTE_REQUEST = 0 + private const val MSG_ROUTE_RESPONSE = 1 + private const val MSG_NAV_REQUEST = 2 + private const val MSG_NAV_UPDATE = 3 + private const val MSG_STOP_NAVIGATION = 6 + private const val MSG_REROUTE_UPDATE = 7 + private const val MSG_MANEUVER_NOTIFICATION = 8 + private const val MSG_APP_START = 13 + private const val MSG_APP_EXIT = 14 + private const val MSG_APP_DIED = 15 + private const val MSG_KEEPALIVE = 17 + + private const val CASE_ROUTE_RESPONSE = 3 + private const val CASE_NAV_UPDATE = 5 + private const val CASE_REROUTE = 7 + private const val CASE_MANEUVER = 8 + + private const val RESULT_SUCCESS = 0L + private const val RESULT_ERROR = 1L + private const val SOURCE_GLASS = 0L + private const val SOURCE_MWA = 1L + private const val ROUTE_ID = 1L + + private const val STATE_STARTED = 0L + private const val STATE_IN_PROGRESS = 1L + private const val STATE_STOPPED = 2L + private const val STATE_ARRIVED = 3L + + private const val NAVIGATOR_ON_ROUTE = 1L + private const val NAVIGATOR_OFF_ROUTE = 2L + private const val NAVIGATOR_BAD_GPS = 3L + private const val NAVIGATOR_REROUTING = 4L + private const val NAVIGATOR_STOPPED = 5L + + /** The Meta app's keepalive period was not found; this is a guess. */ + private const val KEEPALIVE_MS = 5_000L + private const val TICK_MS = 1_000L + private const val OFF_ROUTE_M = 50.0 + private const val REROUTE_TICKS = 10 + private const val PASSED_M = 15.0 + private const val ARRIVE_M = 25.0 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/Panels.kt b/app/src/main/kotlin/com/macedonga/visor/connection/Panels.kt new file mode 100644 index 00000000..63eb86f4 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/Panels.kt @@ -0,0 +1,170 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.annotation.SuppressLint +import android.content.Context +import android.content.pm.PackageManager +import android.location.LocationManager +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.login.MetaLogin +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.ByteArrayOutputStream +import java.util.Locale +import java.util.TimeZone +import java.util.zip.GZIPOutputStream +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import org.json.JSONArray +import org.json.JSONObject + +/** + * svc9002, panels: the Display's home asks which panels the phone offers (type 1, answered 2) and opens one (type 0, + * answered with a UICommand on 0). The one panel is the Briefing, the Glanceables (weather, reminders and the rest): + * Meta's XFBGlanceablesQuery run with the user's login, its preview and each card's Bloks payload gzipped into a + * BloksPayload, the cards' list beside it. A cache prewarm (3) is answered 4. + */ +@Singleton +class Panels @Inject constructor( + @ApplicationContext private val context: Context, + private val login: Provider, + private val requests: RequestLog, + private val log: Provider, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Panels: failed: $e") }) + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val reply = DataXConnection.mirror(message.info.channelId) + when (message.info.messageType) { + MT_GET_ENABLED -> connection.send(reply, Wire.fieldBytes(1, briefingInfo()), messageType = MT_ENABLED) + MT_OPEN -> { + val host = runCatching { Host.decode(Wire.parse(message.payload)) }.getOrNull() ?: return + scope.launch { open(host, reply, connection) } + } + MT_PREWARM -> connection.send(reply, Wire.fieldVarint(1, 0), messageType = MT_PREWARMED) + } + } + + /** PanelInfo: 1 id, 2 name, 3 ttl s, 4 renderer (Bloks), 5 data provider (DataX), 6 cache policy (enabled). */ + private fun briefingInfo(): ByteArray = + Wire.fieldVarint(1, BRIEFING.toLong()) + Wire.fieldBytes(2, "Briefing".encodeToByteArray()) + Wire.fieldVarint(3, TTL_S) + + Wire.fieldVarint(4, RENDERER_BLOKS) + Wire.fieldVarint(5, PROVIDER_DATAX) + Wire.fieldVarint(6, CACHE_ENABLED) + + private suspend fun open(host: Host, reply: Int, mux: DataXConnection) { + val id = requests.start("bloks://glanceables", SOURCE_META) + val started = System.nanoTime() + try { + val g = fetch(host) + val hashes = mutableMapOf() + val payloads = g.optJSONArray("l1_payload_list") ?: JSONArray() + var bloks = ByteArray(0) + for (i in 0 until payloads.length()) { + val p = payloads.getJSONObject(i) + val cardId = p.optString("id") + val hash = p.optString("content_hash") + hashes[cardId] = hash + bloks += Wire.fieldBytes(1, Wire.fieldBytes(1, cardId.encodeToByteArray()) + Wire.fieldBytes(2, p.optString("payload").encodeToByteArray()) + Wire.fieldBytes(3, hash.encodeToByteArray())) + } + bloks += Wire.fieldBytes(2, g.optString("preview").encodeToByteArray()) + val gz = ByteArrayOutputStream().also { out -> GZIPOutputStream(out).use { it.write(bloks) } }.toByteArray() + + var data = Wire.fieldBytes(1, gz) + val cards = g.optJSONArray("all_glanceables") ?: JSONArray() + for (i in 0 until cards.length()) { + val c = cards.getJSONObject(i) + val name = c.optString("name") + data += Wire.fieldBytes( + 2, + Wire.fieldBytes(1, c.optString("id").encodeToByteArray()) + Wire.fieldBytes(2, name.encodeToByteArray()) + + Wire.fieldBytes(3, "Shows ${name.lowercase()} information".encodeToByteArray()) + Wire.fieldVarint(4, if (c.optBoolean("enabled")) 1 else 0) + + Wire.fieldBytes(6, hashes[c.optString("id")].orEmpty().encodeToByteArray()) + Wire.fieldBytes(7, c.optString("icon_uri").encodeToByteArray()), + ) + } + data += Wire.fieldVarint(3, g.optLong("fresh_ttl")) + Wire.fieldVarint(4, g.optLong("ttl")) + val ms = (System.nanoTime() - started) / 1_000_000f + val stats = Wire.fieldVarint(1, OS_ANDROID) + Wire.fieldFloat(3, ms) + val command = Wire.fieldVarint(1, COMMAND_CREATE) + Wire.fieldVarint(2, WIDGET_DATAPAYLOAD) + Wire.fieldBytes(3, stats) + Wire.fieldBytes(10, data) + mux.send(reply, command, messageType = MT_OPEN) + requests.finish(id, "from Meta, ${payloads.length()} cards", failed = false, bytes = gz.size.toLong()) + } catch (error: Exception) { + requests.finish(id, "failed: ${error.message}", failed = true) + val details = Wire.fieldBytes(1, (error.message ?: "failed").encodeToByteArray()) + runCatching { mux.send(reply, Wire.fieldVarint(1, COMMAND_ERROR) + Wire.fieldBytes(11, details), messageType = MT_OPEN) } + } + } + + private fun fetch(host: Host): JSONObject { + val vars = JSONObject() + .put("bloks_version_id", host.versioningId) + .put("device_type", if (host.device == DEVICE_WRIST) "WRIST" else "GREATWHITE") + .put("design_system_version", host.designSystem) + .put("timezone", TimeZone.getDefault().id) + .put("request_id", host.requestId) + .put("assistant_locale", Locale.getDefault().toString()) + .put("display_locale", Locale.getDefault().toString()) + here()?.let { (lat, lon) -> vars.put("latitude", lat).put("longitude", lon) } + if (host.persist) { + vars.put("enabled_glanceables", JSONArray(host.cards.filter { it.enabled }.map { JSONObject().put("id", it.id).put("name", it.name).put("cached_hash", it.hash) })) + } + val answer = login.get().graphql("XFBGlanceablesQuery", GLANCEABLES_DOC, vars.toString()) + val json = JSONObject(answer) + json.optJSONArray("errors")?.let { error(it.optJSONObject(0)?.optString("message") ?: "Meta refused it") } + val data = json.optJSONObject("data") ?: error(json.optJSONObject("error")?.optString("message") ?: answer.take(300)) + return data.getJSONObject("xfb_glanceables") + } + + /** The phone's last known place, for the weather; none without the permission. */ + @SuppressLint("MissingPermission") + private fun here(): Pair? { + if (context.checkSelfPermission(Manifest.permission.ACCESS_COARSE_LOCATION) != PackageManager.PERMISSION_GRANTED) return null + val lm = context.getSystemService(LocationManager::class.java) ?: return null + val best = lm.getProviders(true).mapNotNull { runCatching { lm.getLastKnownLocation(it) }.getOrNull() }.maxByOrNull { it.time } ?: return null + // Two decimals, as the Display itself sends them: a kilometre is plenty for the weather. + return Math.round(best.latitude * 100) / 100.0 to Math.round(best.longitude * 100) / 100.0 + } + + private class Card(val id: String, val name: String, val enabled: Boolean, val hash: String) + + /** OpenPanel 2 HostInfo: 1 device, 4 Bloks version, 5 design system, 7 panel configs, 8 persist them, 9 request id. */ + private class Host(val device: Long, val versioningId: String, val designSystem: String, val cards: List, val persist: Boolean, val requestId: String) { + companion object { + fun decode(open: Wire.Fields): Host { + val h = Wire.parse(open.bytes(2)) + val cards = h.repeatedBytes(7).map { Wire.parse(it) }.filter { it.varint(5) == BRIEFING.toLong() }.map { + Card(it.bytes(1).decodeToString(), it.bytes(2).decodeToString(), it.varint(4) != 0L, it.bytes(6).decodeToString()) + } + return Host(h.varint(1), h.bytes(4).decodeToString(), h.bytes(5).decodeToString(), cards, h.varint(8) != 0L, h.bytes(9).decodeToString()) + } + } + } + + companion object { + const val SVC = 9002 + private const val MT_OPEN = 0 + private const val MT_GET_ENABLED = 1 + private const val MT_ENABLED = 2 + private const val MT_PREWARM = 3 + private const val MT_PREWARMED = 4 + private const val BRIEFING = 30 + private const val TTL_S = 3600L + private const val RENDERER_BLOKS = 2L + private const val PROVIDER_DATAX = 1L + private const val CACHE_ENABLED = 1L + private const val COMMAND_CREATE = 0L + private const val COMMAND_ERROR = 3L + private const val WIDGET_DATAPAYLOAD = 5L + private const val OS_ANDROID = 1L + private const val DEVICE_WRIST = 2L + private const val GLANCEABLES_DOC = "168198258811666417495687582669" + private const val SOURCE_META = -2 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneCallControl.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneCallControl.kt new file mode 100644 index 00000000..779879b6 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneCallControl.kt @@ -0,0 +1,223 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.app.ActivityManager +import android.bluetooth.BluetoothManager +import android.content.ContentUris +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.net.ConnectivityManager +import android.net.NetworkCapabilities +import android.net.Uri +import android.os.Bundle +import android.provider.ContactsContract +import android.telecom.TelecomManager +import android.telephony.TelephonyManager +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.phone.companionAssociated +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * svc70 call control, beside the texts: OUTGOING_CALL_REQUEST (4) by OUTGOING_CALL_RESPONSE (10) {1 request id, + * 2 result, 3 error detail}; CALL_STATE_REQUEST (3) by CALL_STATE_RESPONSE (9) {1 call states}; ANSWER_INCOMING_CALL_REQUEST + * (38) by its response (39) {1 request id, 2 success}; COMMS_HEALTH_REQUEST (59) by COMMS_HEALTH_RESPONSE (60). The other + * listeners get the ack they wait for: 6 by 15, 13 by 14, 22 by 36. Cellular calls (provider 3) go through Telecom, and + * WhatsApp calls open the contact's WhatsApp call row; any other provider is refused. + */ +@Singleton +class PhoneCallControl @Inject constructor( + @ApplicationContext private val context: Context, + private val requests: RequestLog, + private val log: Provider, +) { + suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed) return + val reply = DataXConnection.mirror(message.info.channelId) + val request = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return + when (message.info.messageType) { + MT_CALL_STATE_REQUEST -> callState(reply, connection) + MT_OUTGOING_CALL_REQUEST -> outgoing(request, reply, connection) + MT_ANSWER_INCOMING_CALL_REQUEST -> answer(request, reply, connection) + MT_COMMS_HEALTH_REQUEST -> commsHealth(request, reply, connection) + MT_CALL_ENGINE_SUBSCRIPTION -> send("calls://engine/subscribe", reply, connection, MT_CALL_ENGINE_SUBSCRIPTION_ACK, ByteArray(0), "call engine subscription acknowledged") + MT_OUTGOING_ERROR_LISTENER -> send("calls://errors/subscribe", reply, connection, MT_OUTGOING_ERROR_LISTENER_ACK, ByteArray(0), "call error listener acknowledged") + MT_CALLING_SVC_SUBSCRIBE -> send("calls://svc/subscribe", reply, connection, MT_CALLING_SVC_ACK, ByteArray(0), "calling service state acknowledged") + } + } + + /** CallState on the phone's own line: ringing in as RINGING_INBOUND, a call under way as ACTIVE, else none. */ + @Suppress("DEPRECATION") + private suspend fun callState(reply: Int, mux: DataXConnection) { + if (!allowed(Manifest.permission.READ_PHONE_STATE)) { + return send(STATE_URL, reply, mux, MT_CALL_STATE_RESPONSE, ByteArray(0), "phone state not allowed", failed = true) + } + val state = runCatching { context.getSystemService(TelephonyManager::class.java)?.callState }.getOrNull() + val entry = when (state) { + TelephonyManager.CALL_STATE_RINGING -> STATE_RINGING_INBOUND + TelephonyManager.CALL_STATE_OFFHOOK -> STATE_ACTIVE + else -> null + } + // CallState {1 call id, 2 provider, 3 state}, in GetCallStatesResponse {1}. + val body = if (entry == null) ByteArray(0) else + Wire.fieldBytes(1, Wire.fieldBytes(1, CELLULAR_CALL_ID.encodeToByteArray()) + Wire.fieldVarint(2, NATIVE) + Wire.fieldVarint(3, entry)) + send(STATE_URL, reply, mux, MT_CALL_STATE_RESPONSE, body, if (entry == null) "no call" else "call state $entry") + } + + /** OutgoingCallRequest {1 provider, 2 callee, 7 request id}. */ + private suspend fun outgoing(request: Wire.Fields, reply: Int, mux: DataXConnection) { + val provider = request.varint(1) + val callee = request.bytes(2).decodeToString() + val (result, error) = when (provider) { + NATIVE -> placeCall(callee) + WHATSAPP_PROVIDER -> startWhatsApp(callee) + else -> RESULT_UNKNOWN to "Calls on this provider are not supported" + } + val body = Wire.fieldBytes(1, request.bytes(7)) + Wire.fieldVarint(2, result) + + (error?.let { Wire.fieldBytes(3, it.encodeToByteArray()) } ?: ByteArray(0)) + send("calls://outgoing/$provider", reply, mux, MT_OUTGOING_CALL_RESPONSE, body, error ?: "call started", failed = result != RESULT_OK) + } + + /** Telecom takes the call; the call itself rides HFP to the glasses. */ + private fun placeCall(number: String): Pair { + if (!allowed(Manifest.permission.CALL_PHONE)) return ERROR_PERMISSION to "Call permission is not allowed in Visor" + if (number.isBlank()) return RESULT_UNKNOWN to "No number to call" + return try { + requireNotNull(context.getSystemService(TelecomManager::class.java)).placeCall(Uri.fromParts("tel", number, null), Bundle()) + RESULT_OK to null + } catch (e: Exception) { + RESULT_UNKNOWN to (e.message ?: e.toString()) + } + } + + /** Android has no call API for WhatsApp, so this opens the contact's WhatsApp call row, which starts the call. */ + private fun startWhatsApp(number: String): Pair { + if (!allowed(Manifest.permission.READ_CONTACTS)) return ERROR_PERMISSION to "Contacts permission is not allowed in Visor" + if (!startable()) return RESULT_UNKNOWN to "Visor cannot open WhatsApp from the background" + val row = whatsAppRow(number) ?: return RESULT_UNKNOWN to "No WhatsApp call for this number" + val intent = Intent(Intent.ACTION_VIEW, ContentUris.withAppendedId(ContactsContract.Data.CONTENT_URI, row)).setPackage(WHATSAPP_PACKAGE) + return try { + context.startActivity(intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)) + RESULT_OK to null + } catch (e: Exception) { + RESULT_UNKNOWN to (e.message ?: e.toString()) + } + } + + /** The data row holding the WhatsApp call of the contact that owns [number]. */ + private fun whatsAppRow(number: String): Long? { + if (number.isBlank()) return null + val contact = context.contentResolver.query( + Uri.withAppendedPath(ContactsContract.PhoneLookup.CONTENT_FILTER_URI, Uri.encode(number)), + arrayOf(ContactsContract.PhoneLookup._ID), null, null, null, + )?.use { if (it.moveToFirst()) it.getLong(0) else null } ?: return null + return context.contentResolver.query( + ContactsContract.Data.CONTENT_URI, + arrayOf(ContactsContract.Data._ID), + "${ContactsContract.Data.CONTACT_ID} = ? AND ${ContactsContract.Data.MIMETYPE} = ?", + arrayOf(contact.toString(), WHATSAPP_CALL), + null, + )?.use { if (it.moveToFirst()) it.getLong(0) else null } + } + + /** AnswerIncomingCallRequest {1 provider, 2 request id, 3 call id}: the ringing cellular call is taken. */ + @Suppress("DEPRECATION") + private suspend fun answer(request: Wire.Fields, reply: Int, mux: DataXConnection) { + val provider = request.varint(1) + val error = when { + provider != NATIVE -> "Answering on this provider is not supported" + !allowed(Manifest.permission.ANSWER_PHONE_CALLS) || !allowed(Manifest.permission.READ_PHONE_STATE) -> + "Call permission is not allowed in Visor" + !ringing() -> "No ringing call" + else -> try { + requireNotNull(context.getSystemService(TelecomManager::class.java)).acceptRingingCall() + null + } catch (e: Exception) { + e.message ?: e.toString() + } + } + val body = Wire.fieldBytes(1, request.bytes(2)) + Wire.fieldVarint(2, if (error == null) RESULT_OK else 0L) + send("calls://answer/$provider", reply, mux, MT_ANSWER_INCOMING_CALL_RESPONSE, body, error ?: "call answered", failed = error != null) + } + + /** CommsHealthResponse {1 request id, 2 has state, 4 network, 5 Bluetooth, 6 network time, 7 state time}. */ + private suspend fun commsHealth(request: Wire.Fields, reply: Int, mux: DataXConnection) { + val now = System.currentTimeMillis() + val network = if (networkUp()) SIGNAL_PASS else SIGNAL_FAIL + val bluetooth = if (bluetoothOn()) SIGNAL_PASS else SIGNAL_FAIL + val body = Wire.fieldBytes(1, request.bytes(1)) + Wire.fieldVarint(2, 1) + Wire.fieldVarint(4, network) + + Wire.fieldVarint(5, bluetooth) + Wire.fieldVarint(6, now) + Wire.fieldVarint(7, now) + val outcome = "network ${signal(network)}, bluetooth ${signal(bluetooth)}" + send("calls://health", reply, mux, MT_COMMS_HEALTH_RESPONSE, body, outcome, failed = network == SIGNAL_FAIL || bluetooth == SIGNAL_FAIL) + } + + @Suppress("DEPRECATION") + private fun ringing(): Boolean = runCatching { + context.getSystemService(TelephonyManager::class.java)?.callState == TelephonyManager.CALL_STATE_RINGING + }.getOrDefault(false) + + private fun networkUp(): Boolean { + val manager = context.getSystemService(ConnectivityManager::class.java) ?: return false + val caps = manager.getNetworkCapabilities(manager.activeNetwork) ?: return false + return caps.hasCapability(NetworkCapabilities.NET_CAPABILITY_VALIDATED) + } + + private fun bluetoothOn(): Boolean = context.getSystemService(BluetoothManager::class.java)?.adapter?.isEnabled == true + + /** Android lets an app start an activity from the screen or as the glasses' companion, as PhoneMedia does. */ + private fun startable(): Boolean = onScreen() || companionAssociated(context) + + private fun onScreen(): Boolean = ActivityManager.RunningAppProcessInfo() + .also(ActivityManager::getMyMemoryState).importance == ActivityManager.RunningAppProcessInfo.IMPORTANCE_FOREGROUND + + private fun allowed(permission: String) = context.checkSelfPermission(permission) == PackageManager.PERMISSION_GRANTED + + private suspend fun send(url: String, reply: Int, mux: DataXConnection, type: Int, body: ByteArray, outcome: String, failed: Boolean = false) { + val id = requests.start(url, SOURCE_CALLS) + // The glasses may have closed the channel by now; the answer is theirs to miss. + runCatching { mux.send(reply, body, messageType = type) } + requests.finish(id, outcome, failed, bytes = body.size.toLong()) + log.get().add("Calls: $outcome") + } + + private fun signal(value: Long) = if (value == SIGNAL_PASS) "pass" else "fail" + + companion object { + /** The types this class answers. PhoneMessages hands them here before its own checks. */ + val TYPES = setOf(3, 4, 6, 13, 22, 38, 59) + private const val MT_CALL_STATE_REQUEST = 3 + private const val MT_OUTGOING_CALL_REQUEST = 4 + private const val MT_CALL_ENGINE_SUBSCRIPTION = 6 + private const val MT_CALL_STATE_RESPONSE = 9 + private const val MT_OUTGOING_CALL_RESPONSE = 10 + private const val MT_OUTGOING_ERROR_LISTENER = 13 + private const val MT_OUTGOING_ERROR_LISTENER_ACK = 14 + private const val MT_CALL_ENGINE_SUBSCRIPTION_ACK = 15 + private const val MT_ANSWER_INCOMING_CALL_REQUEST = 38 + private const val MT_ANSWER_INCOMING_CALL_RESPONSE = 39 + private const val MT_CALLING_SVC_SUBSCRIBE = 22 + private const val MT_CALLING_SVC_ACK = 36 + private const val MT_COMMS_HEALTH_REQUEST = 59 + private const val MT_COMMS_HEALTH_RESPONSE = 60 + private const val RESULT_OK = 1L + private const val ERROR_PERMISSION = 4L + private const val RESULT_UNKNOWN = 12L + private const val WHATSAPP_PACKAGE = "com.whatsapp" + private const val WHATSAPP_CALL = "vnd.android.cursor.item/vnd.com.whatsapp.voip.call" + private const val NATIVE = 3L + private const val WHATSAPP_PROVIDER = 2L + private const val STATE_RINGING_INBOUND = 1L + private const val STATE_ACTIVE = 3L + private const val CELLULAR_CALL_ID = "cellular" + private const val STATE_URL = "calls://state" + private const val SIGNAL_PASS = 1L + private const val SIGNAL_FAIL = 2L + private const val SOURCE_CALLS = -5 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneCalls.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneCalls.kt new file mode 100644 index 00000000..2798ff44 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneCalls.kt @@ -0,0 +1,152 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.provider.CallLog +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import org.json.JSONArray +import org.json.JSONObject + +/** + * The phone's recent calls, asked for twice by the Display: + * + * - svc11006, phone_book_sync, by its telephony service, on connect, after a call and now and then: type 1 {1 request + * id}, answered with PhoneBookSyncResponse {1 request id, 2 call log, 3 error, 4 phone OS} (type 2). Told the phone is + * Android, it keeps the call log from the answer; an iPhone it would read over PBAP instead. The call log is JSON, a + * list of {"cType": "CDEntryAdd", "id", "entry": {id, cachedName, phoneNumber, type, dateEpochMilliseconds, + * durationSeconds}} with the type by name (INCOMING, OUTGOING, MISSED, ...). + * - svc70 LoadCallLogRequest {1 provider, 2 request id, 3 count, 4 request type} (type 11), by the Phone app once per + * provider, answered with LoadCallLogResponse {1 request id, 2 CallLog} (type 12). The phone's own calls (provider 3) + * come from the call log; the other providers get an empty list, as the Meta app gives when it has no handler. + */ +@Singleton +class PhoneCalls @Inject constructor( + @ApplicationContext private val context: Context, + private val requests: RequestLog, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, _ -> }) + + private data class Call(val id: Long, val name: String?, val number: String?, val type: Int, val date: Long, val duration: Long, val video: Boolean) + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.messageType != MT_SYNC_REQUEST || message.info.flags.isClosed) return + val requestId = runCatching { Wire.parse(message.payload).bytes(1) }.getOrDefault(ByteArray(0)) + val reply = DataXConnection.mirror(message.info.channelId) + scope.launch { + val id = requests.start("calls://sync", SOURCE_CALLS) + val calls = if (allowed()) recent(MAX_CALLS) else null + val json = JSONArray() + calls?.forEach { c -> + val entry = JSONObject() + .put("id", c.id) + .put("cachedName", c.name ?: JSONObject.NULL) + .put("phoneNumber", c.number ?: JSONObject.NULL) + .put("type", typeName(c.type)) + .put("dateEpochMilliseconds", c.date) + .put("durationSeconds", c.duration) + json.put(JSONObject().put("cType", "CDEntryAdd").put("id", c.date.toString()).put("entry", entry)) + } + val body = Wire.fieldBytes(1, requestId) + + Wire.fieldBytes(2, (if (calls != null) json.toString() else "").encodeToByteArray()) + + Wire.fieldVarint(3, if (calls != null) SUCCESS else NO_PERMISSION) + + Wire.fieldVarint(4, OS_ANDROID) + connection.send(reply, body, messageType = MT_SYNC_RESPONSE) + if (calls != null) requests.finish(id, "${calls.size} calls", failed = false, bytes = body.size.toLong()) + else requests.finish(id, "call history not allowed", failed = true) + } + } + + /** svc70 type 11, from [PhoneMessages]: [reply] is the request's mirror channel. */ + fun loadCallLog(request: Wire.Fields, reply: Int, mux: DataXConnection) { + scope.launch { + val provider = request.varint(1) + val id = requests.start("calls://recent/$provider", SOURCE_CALLS) + val count = request.varint(3).toInt().takeIf { it > 0 }?.coerceAtMost(MAX_CALLS) ?: DEFAULT_RECENT + val calls = if (provider == NATIVE && allowed()) recent(count) else emptyList() + var body = Wire.fieldBytes(1, request.bytes(2)) + for (c in calls) { + // CallLog: 1 name, 3 time, 5 type (1 incoming, 2 outgoing, 3 missed, 0 other), 7 number, 11 video. + if (c.type == CallLog.Calls.VOICEMAIL_TYPE) continue + body += Wire.fieldBytes( + 2, + (if (!c.name.isNullOrEmpty()) Wire.fieldBytes(1, c.name.encodeToByteArray()) else ByteArray(0)) + + Wire.fieldVarint(3, c.date) + Wire.fieldVarint(5, wireType(c.type)) + + (if (!c.number.isNullOrEmpty()) Wire.fieldBytes(7, c.number.encodeToByteArray()) else ByteArray(0)) + + (if (c.video) Wire.fieldVarint(11, 1) else ByteArray(0)), + ) + } + mux.send(reply, body, messageType = MT_LOAD_CALL_LOG_RESPONSE) + val outcome = when { + provider != NATIVE -> "not the phone's, none" + !allowed() -> "call history not allowed" + else -> "${calls.size} calls" + } + requests.finish(id, outcome, failed = provider == NATIVE && !allowed(), bytes = body.size.toLong()) + } + } + + private fun allowed() = context.checkSelfPermission(Manifest.permission.READ_CALL_LOG) == PackageManager.PERMISSION_GRANTED + + private fun recent(count: Int): List { + val calls = ArrayList() + val columns = arrayOf( + CallLog.Calls._ID, CallLog.Calls.CACHED_NAME, CallLog.Calls.NUMBER, CallLog.Calls.TYPE, CallLog.Calls.DATE, + CallLog.Calls.DURATION, CallLog.Calls.FEATURES, + ) + val uri = CallLog.Calls.CONTENT_URI.buildUpon().appendQueryParameter(CallLog.Calls.LIMIT_PARAM_KEY, count.toString()).build() + context.contentResolver.query(uri, columns, null, null, "${CallLog.Calls.DATE} DESC")?.use { c -> + while (c.moveToNext() && calls.size < count) { + calls += Call( + c.getLong(0), c.getString(1), c.getString(2), c.getInt(3), c.getLong(4), c.getLong(5), + c.getInt(6) and CallLog.Calls.FEATURES_VIDEO != 0, + ) + } + } + return calls + } + + private fun typeName(type: Int): String = when (type) { + CallLog.Calls.INCOMING_TYPE -> "INCOMING" + CallLog.Calls.OUTGOING_TYPE -> "OUTGOING" + CallLog.Calls.MISSED_TYPE -> "MISSED" + CallLog.Calls.VOICEMAIL_TYPE -> "VOICEMAIL" + CallLog.Calls.REJECTED_TYPE -> "REJECTED" + CallLog.Calls.BLOCKED_TYPE -> "BLOCKED" + CallLog.Calls.ANSWERED_EXTERNALLY_TYPE -> "ANSWERED_EXTERNALLY" + else -> "UNKNOWN" + } + + // A rejected call shows as missed, as the Meta app sends it. + private fun wireType(type: Int): Long = when (type) { + CallLog.Calls.INCOMING_TYPE -> 1 + CallLog.Calls.OUTGOING_TYPE -> 2 + CallLog.Calls.MISSED_TYPE, CallLog.Calls.REJECTED_TYPE -> 3 + else -> 0 + } + + companion object { + const val SVC = 11006 + private const val MT_SYNC_REQUEST = 1 + private const val MT_SYNC_RESPONSE = 2 + private const val MT_LOAD_CALL_LOG_RESPONSE = 12 + private const val SUCCESS = 0L + private const val NO_PERMISSION = 1L + private const val OS_ANDROID = 1L + private const val NATIVE = 3L + private const val MAX_CALLS = 100 + private const val DEFAULT_RECENT = 15 + private const val SOURCE_CALLS = -5 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneHttpRelay.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneHttpRelay.kt new file mode 100644 index 00000000..960b7218 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneHttpRelay.kt @@ -0,0 +1,149 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.log.EventLog +import java.io.ByteArrayOutputStream +import java.net.HttpURLConnection +import java.net.URL +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch + +/** + * svc41111, internet_proxy_remote, HTTP relay: the Display sends an HttpRequest (type 8) on a channel of its own, and the + * phone answers on that channel with an HttpResponse (type 11), the body in chunks (12), then an empty end (13). The reply + * types follow the Display's reader of HttpResponse (GreatwhiteSystemUIRelease, X/C0OQ.java). The request type 8 and a + * streamed request body (chunks until 13) are not in the research; they are guessed from that reader. + */ +@Singleton +class PhoneHttpRelay @Inject constructor( + private val requests: RequestLog, + private val log: Provider, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Internet: a request failed: $e") }) + // Requests whose body the Display streams after them, by channel. + private val streaming = ConcurrentHashMap() + + fun handle(message: DataXMessage, connection: DataXConnection) { + val channel = message.info.channelId + when (message.info.messageType) { + MT_REQUEST -> { + val request = runCatching { Request.decode(message.payload) }.getOrElse { + scope.launch { answer(channel, connection, error = true) } + return + } + if (request.streamed) streaming[channel] = Streamed(request, connection, ByteArrayOutputStream()) + else scope.launch { serve(channel, request, null, connection) } + } + MT_BODY -> streaming[channel]?.body?.write(message.payload) + MT_END -> streaming.remove(channel)?.let { scope.launch { serve(channel, it.request, it.body.toByteArray(), it.mux) } } + } + } + + fun forget(channel: Int) { + streaming.remove(channel) + } + + private suspend fun serve(channel: Int, request: Request, body: ByteArray?, mux: DataXConnection) { + val id = requests.start(request.url, SOURCE_TUNNEL, request.method) + var answered = false + try { + val url = URL(request.url) + require(url.protocol == "http" || url.protocol == "https") { "not an http link" } + val conn = (url.openConnection() as HttpURLConnection).apply { + requestMethod = request.method + instanceFollowRedirects = !request.disableRedirects + connectTimeout = CONNECT_MS + readTimeout = READ_MS + for ((name, value) in request.headers) setRequestProperty(name, value) + if (body != null) { + doOutput = true + outputStream.use { it.write(body) } + } + } + val code = conn.responseCode + mux.send(DataXConnection.mirror(channel), response(code, conn.responseMessage.orEmpty(), conn), messageType = MT_RESPONSE) + answered = true + var sent = 0L + (if (code >= 400) conn.errorStream else conn.inputStream)?.use { input -> + val buffer = ByteArray(CHUNK) + while (true) { + val n = input.read(buffer) + if (n < 0) break + mux.send(DataXConnection.mirror(channel), buffer.copyOf(n), messageType = MT_BODY) + sent += n + } + } + mux.send(DataXConnection.mirror(channel), ByteArray(0), messageType = MT_END) + requests.finish(id, "HTTP $code", failed = code >= 400, bytes = sent) + } catch (e: Exception) { + requests.finish(id, "failed: ${e.message}", failed = true) + answer(channel, mux, error = !answered) + } + } + + /** The reply that ends a request the phone could not serve: an error and the end, so the Display stops waiting. */ + private suspend fun answer(channel: Int, mux: DataXConnection, error: Boolean) { + val reply = DataXConnection.mirror(channel) + runCatching { + if (error) mux.send(reply, Wire.fieldVarint(4, ERROR_FAILED), messageType = MT_RESPONSE) + mux.send(reply, ByteArray(0), messageType = MT_END) + } + } + + private fun response(code: Int, message: String, conn: HttpURLConnection): ByteArray { + var out = Wire.fieldVarint(1, code.toLong()) + Wire.fieldBytes(2, message.toByteArray()) + for ((name, values) in conn.headerFields) { + if (name == null) continue + for (value in values) out += Wire.fieldBytes(3, Wire.fieldBytes(1, name.toByteArray()) + Wire.fieldBytes(2, value.toByteArray())) + } + return out + } + + private class Request( + val url: String, + val method: String, + val headers: List>, + val streamed: Boolean, + val disableRedirects: Boolean, + ) { + companion object { + // The request's uuid (field 1) is not needed: each request has its own channel. + fun decode(payload: ByteArray): Request { + val f = Wire.parse(payload) + return Request( + url = f.bytes(2).decodeToString(), + method = f.bytes(7).decodeToString().ifEmpty { "GET" }, + headers = f.repeatedBytes(5).map { h -> Wire.parse(h).let { it.bytes(1).decodeToString() to it.bytes(2).decodeToString() } }, + streamed = f.varint(6) != 0L, + disableRedirects = f.varint(8) != 0L, + ) + } + } + } + + private class Streamed(val request: Request, val mux: DataXConnection, val body: ByteArrayOutputStream) + + companion object { + private const val MT_REQUEST = 8 + private const val MT_RESPONSE = 11 + private const val MT_BODY = 12 + private const val MT_END = 13 + // HttpResponse.error for a request the phone could not make. The Display's error codes are not in the research. + private const val ERROR_FAILED = 1L + // The chunk size the data fetcher uses by default, which the Display already reads. + private const val CHUNK = 1024 + private const val CONNECT_MS = 10_000 + private const val READ_MS = 30_000 + // The request log's source for traffic of the tunnel rather than of the data fetcher. + private const val SOURCE_TUNNEL = -1 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt new file mode 100644 index 00000000..2065f15a --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneInternet.kt @@ -0,0 +1,404 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import java.io.InputStream +import java.io.OutputStream +import java.net.DatagramPacket +import java.net.DatagramSocket +import java.net.InetSocketAddress +import java.net.Socket +import java.net.SocketTimeoutException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlin.random.Random +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.withTimeoutOrNull + +/** + * svc41111, internet_proxy_remote: the Display's apps reach the internet through the phone. The Display's tunnel sends + * each IP packet as one message (type 1), framed `1c fb`, its length, its ethertype and two spare bytes; replies go + * back the same way on the same channel. The phone ends the connections itself: each TCP connection from the Display + * becomes a socket of the phone's, each UDP flow a datagram socket, so the Display's traffic is the phone's own. + * Other types go to [PhoneHttpRelay]: a single HTTP request (type 8) answered on its own channel. + */ +@Singleton +class PhoneInternet @Inject constructor( + private val requests: RequestLog, + private val log: Provider, + private val relay: PhoneHttpRelay, +) : Responder { + // A connection that fails ends alone; nothing in here may take the app down. + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Internet: a connection failed: $e") }) + private val sessions = ConcurrentHashMap() + // Tunnel frames waiting for the link. Control frames of the session share its one send lock, so they wait + // behind at most this many of these. + private val tunnelSlots = Semaphore(TUNNEL_FRAMES) + // The connection the tunnels run on; a new one ends the old tunnels. + private var owner: DataXConnection? = null + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val info = message.info + val channel = info.channelId + if (connection !== owner) { + // A tunnel of an earlier connection would write on this one under the old keys, and keep retransmitting. + owner = connection + sessions.values.forEach { it.close() } + sessions.clear() + } + if (info.flags.isClosed && message.payload.isEmpty()) { + sessions.remove(channel)?.close() + relay.forget(channel) + return + } + when (info.messageType) { + MT_START -> { + sessions.remove(channel)?.close() + sessions[channel] = Session(connection, DataXConnection.mirror(channel)) + log.get().add("Internet: the Display's apps are using the phone's internet") + } + MT_FRAME -> sessions.getOrPut(channel) { Session(connection, DataXConnection.mirror(channel)) }.inbound(message.payload) + else -> relay.handle(message, connection) + } + } + + /** One tunnel from the Display: its TCP and UDP flows, and the way back. */ + private inner class Session(private val mux: DataXConnection, private val reply: Int) { + private val tcp = ConcurrentHashMap() + private val udp = ConcurrentHashMap() + private val ids = AtomicInteger(Random.nextInt(0xFFFF)) + + fun inbound(frame: ByteArray) { + if (frame.size < ICEE_HEADER) return + val head = ByteBuffer.wrap(frame).order(ByteOrder.LITTLE_ENDIAN) + if (head.getShort(0).toInt() and 0xFFFF != ICEE_MAGIC) return + val length = head.getShort(2).toInt() and 0xFFFF + val packet = IpPacket.parse(frame.copyOfRange(ICEE_HEADER, (ICEE_HEADER + length).coerceAtMost(frame.size))) ?: return + when (packet.protocol) { + IpPacket.TCP -> { + val seg = TcpSegment.parse(packet.transport) ?: return + val key = key(packet, seg.srcPort, seg.dstPort) + val flow = tcp[key] ?: if (seg.syn && !seg.hasAck) TcpFlow(this, packet, seg).also { tcp[key] = it } else { + // A segment for a connection the phone does not know: tell the Display it is gone. + if (!seg.rst) resetFor(packet, seg) + return + } + flow.inbox.trySend(seg) + } + IpPacket.UDP -> { + val d = UdpDatagram.parse(packet.transport) ?: return + val key = key(packet, d.srcPort, d.dstPort) + // A flow the phone cannot reach (IPv6 on a phone without it) is dropped; the Display falls back. + val flow = udp[key] ?: runCatching { UdpFlow(this, packet, d.srcPort, d.dstPort, key) }.getOrNull()?.also { udp[key] = it } ?: return + flow.send(d.data) + } + } + } + + fun forget(key: String) { + tcp.remove(key) + udp.remove(key) + } + + /** An IP packet to the Display, framed for its tunnel. */ + suspend fun deliver(version: Int, src: ByteArray, dst: ByteArray, protocol: Int, transport: ByteArray) { + val ip = IpPacket.build(version, src, dst, protocol, transport, ids.incrementAndGet()) + val frame = ByteBuffer.allocate(ICEE_HEADER + ip.size).order(ByteOrder.LITTLE_ENDIAN) + .putShort(ICEE_MAGIC.toShort()).putShort(ip.size.toShort()) + .order(ByteOrder.BIG_ENDIAN).putShort(if (version == 4) ETHER_IPV4 else ETHER_IPV6).putShort(0) + .put(ip).array() + tunnelSlots.withPermit { mux.send(reply, frame, messageType = MT_FRAME) } + } + + private fun resetFor(p: IpPacket, seg: TcpSegment) { + val ack = (seg.seq + seg.data.size + (if (seg.syn) 1 else 0) + (if (seg.fin) 1 else 0)) and MASK + val rst = TcpSegment.build(seg.dstPort, seg.srcPort, if (seg.hasAck) seg.ack else 0, ack, TcpSegment.RST or TcpSegment.ACK, 0, ByteArray(0)) + scope.launch { runCatching { deliver(p.version, p.dst, p.src, IpPacket.TCP, rst) } } + } + + fun close() { + tcp.values.forEach { it.close() } + udp.values.forEach { it.close() } + tcp.clear() + udp.clear() + } + } + + /** One TCP connection from the Display, carried on a socket of the phone's. */ + private inner class TcpFlow(private val s: Session, p: IpPacket, syn: TcpSegment) { + val inbox = Channel(Channel.UNLIMITED) + private val version = p.version + private val display = p.src + private val remote = p.dst + private val remoteAddress = p.dstAddress + private val displayPort = syn.srcPort + private val remotePort = syn.dstPort + private val key = key(p, syn.srcPort, syn.dstPort) + private val mss = if (version == 4) MSS_V4 else MSS_V6 + private val lock = Mutex() + private val progress = MutableStateFlow(0L) + private val logId = requests.start("tcp://${p.dstAddress.hostAddress}:$remotePort", SOURCE_TUNNEL) + + private var socket: Socket? = null + private var output: OutputStream? = null + private val iss = Random.nextLong(0, MASK) + private var sndUna = iss + private var sndNxt = iss + private var rcvNxt = (syn.seq + 1) and MASK + private var peerWindow = syn.window + private var pending = ByteArray(0) + private var established = false + private var peerFin = false + private var finSent = false + private var named = false + @Volatile private var finished = false + private var bytes = 0L + private val job: Job = scope.launch { run() } + + private suspend fun run() { + val connected = runCatching { + runInterruptible { Socket().apply { connect(InetSocketAddress(remoteAddress, remotePort), CONNECT_MS); tcpNoDelay = true } } + }.getOrElse { + send(TcpSegment.RST or TcpSegment.ACK, 0, ByteArray(0)) + return finish("unreachable", failed = true) + } + socket = connected + output = connected.getOutputStream() + lock.withLock { synAck() } + while (true) { + val next = withTimeoutOrNull(IDLE_MS) { inbox.receiveCatching() } ?: return finish("idle") + // Closed: the tunnel or the link went, and close() has already let go of everything. + val seg = next.getOrNull() ?: return + if (seg.rst) return finish("reset by the Display") + if (seg.syn) { + lock.withLock { synAck() } + continue + } + if (seg.hasAck) onAck(seg) + if (seg.data.isNotEmpty() || seg.fin) onData(seg) + if (peerFin && finSent && sndUna == sndNxt) return finish("closed") + } + } + + private suspend fun synAck() { + sndNxt = (iss + 1) and MASK + send(TcpSegment.SYN or TcpSegment.ACK, iss, ByteArray(0), mss) + } + + private suspend fun onAck(seg: TcpSegment) { + lock.withLock { + val acked = diff(seg.ack, sndUna) + if (acked > 0 && diff(seg.ack, sndNxt) <= 0) { + val dataAcked = acked.coerceAtMost(pending.size.toLong()).toInt() + pending = pending.copyOfRange(dataAcked, pending.size) + sndUna = seg.ack + } + peerWindow = seg.window + if (!established && diff(seg.ack, iss) > 0) { + established = true + scope.launch { pump() } + } + } + progress.value++ + } + + private suspend fun onData(seg: TcpSegment) { + if (seg.seq != rcvNxt) { + // Out of order or a repeat: the last good point again, and the Display sends it anew. + lock.withLock { send(TcpSegment.ACK, sndNxt, ByteArray(0)) } + return + } + if (seg.data.isNotEmpty()) { + if (!named) { + named = true + tlsServerName(seg.data)?.let { requests.rename(logId, "tls://$it:$remotePort") } + } + val ok = runCatching { runInterruptible { output?.write(seg.data); output?.flush() } }.isSuccess + if (!ok) { + lock.withLock { send(TcpSegment.RST or TcpSegment.ACK, sndNxt, ByteArray(0)) } + finish("the server went away", failed = true) + return + } + bytes += seg.data.size + } + lock.withLock { + rcvNxt = (rcvNxt + seg.data.size) and MASK + if (seg.fin && !peerFin) { + peerFin = true + rcvNxt = (rcvNxt + 1) and MASK + runCatching { socket?.shutdownOutput() } + } + send(TcpSegment.ACK, sndNxt, ByteArray(0)) + } + } + + /** The server's bytes to the Display, as much as its window lets through, again when it goes unanswered. */ + private suspend fun pump() { + val input: InputStream = socket?.getInputStream() ?: return + val buffer = ByteArray(mss) + try { + while (scope.isActive) { + val n = runInterruptible { input.read(buffer) } + if (n < 0) break + if (n == 0) continue + waitForWindow(n) + lock.withLock { + val data = buffer.copyOf(n) + send(TcpSegment.ACK or TcpSegment.PSH, sndNxt, data) + pending += data + sndNxt = (sndNxt + n) and MASK + } + bytes += n + } + } catch (_: Exception) { + } + waitForWindow(0) + lock.withLock { + if (!finSent) { + finSent = true + send(TcpSegment.FIN or TcpSegment.ACK, sndNxt, ByteArray(0)) + sndNxt = (sndNxt + 1) and MASK + } + } + } + + private suspend fun waitForWindow(n: Int) { + while (true) { + val (inFlight, window, last) = lock.withLock { Triple(pending.size, peerWindow, progress.value) } + if (inFlight + n <= window.coerceAtLeast(mss) && (n > 0 || inFlight == 0)) return + if (withTimeoutOrNull(RETRANSMIT_MS) { progress.first { it != last } } == null) retransmit() + if (!job.isActive) return + } + } + + /** + * Resend the oldest unacked segment only. Resending the whole window each tick queued up to 48 frames per flow + * every 1.5 s behind a link that was already not keeping up, and held this flow's lock while they waited. + */ + private suspend fun retransmit() { + val (seq, data) = lock.withLock { sndUna to pending.copyOf(minOf(pending.size, mss)) } + if (data.isNotEmpty()) send(TcpSegment.ACK or TcpSegment.PSH, seq, data) + } + + private suspend fun send(flags: Int, seq: Long, data: ByteArray, mssOption: Int = 0) { + val t = TcpSegment.build(remotePort, displayPort, seq, rcvNxt, flags, WINDOW, data, mssOption) + runCatching { s.deliver(version, remote, display, IpPacket.TCP, t) } + } + + private fun finish(outcome: String, failed: Boolean = false) { + if (!finished) requests.finish(logId, outcome, failed, bytes) + finished = true + close() + } + + fun close() { + if (!finished) requests.finish(logId, "the tunnel ended", failed = false, bytes = bytes) + finished = true + s.forget(key) + runCatching { socket?.close() } + job.cancel() + inbox.close() + } + } + + /** One UDP flow from the Display: a datagram socket, kept while it is used. */ + private inner class UdpFlow(private val s: Session, p: IpPacket, private val displayPort: Int, private val remotePort: Int, private val key: String) { + private val version = p.version + private val display = p.src + private val remote = p.dst + private val socket = DatagramSocket().apply { + connect(InetSocketAddress(p.dstAddress, remotePort)) + soTimeout = POLL_MS + } + private val logId = requests.start("udp://${p.dstAddress.hostAddress}:$remotePort", SOURCE_TUNNEL) + @Volatile private var lastUsed = System.currentTimeMillis() + private var bytes = 0L + private val job = scope.launch { receive() } + + fun send(data: ByteArray) { + lastUsed = System.currentTimeMillis() + bytes += data.size + scope.launch { runCatching { socket.send(DatagramPacket(data, data.size)) } } + } + + private suspend fun receive() { + val buffer = ByteArray(UDP_MAX) + while (scope.isActive && System.currentTimeMillis() - lastUsed < UDP_IDLE_MS) { + val packet = DatagramPacket(buffer, buffer.size) + try { + runInterruptible { socket.receive(packet) } + } catch (_: SocketTimeoutException) { + continue + } catch (_: Exception) { + break + } + lastUsed = System.currentTimeMillis() + bytes += packet.length + val u = UdpDatagram.build(remotePort, displayPort, buffer.copyOf(packet.length)) + runCatching { s.deliver(version, remote, display, IpPacket.UDP, u) } + } + requests.finish(logId, "done", failed = false, bytes = bytes) + close() + } + + fun close() { + s.forget(key) + socket.close() + job.cancel() + } + } + + private fun key(p: IpPacket, srcPort: Int, dstPort: Int) = "${p.protocol}/${p.srcAddress.hostAddress}:$srcPort>${p.dstAddress.hostAddress}:$dstPort" + + /** a - b in sequence space, as a signed 32-bit distance. */ + private fun diff(a: Long, b: Long): Long { + val d = (a - b) and MASK + return if (d >= 0x80000000L) d - 0x100000000L else d + } + + companion object { + const val SVC = 41111 + private const val MT_FRAME = 1 + private const val MT_START = 3 + private const val ICEE_HEADER = 8 + private const val ICEE_MAGIC = 0xFB1C + private const val ETHER_IPV4: Short = 0x0800 + private const val ETHER_IPV6: Short = 0x86DD.toShort() + private const val MASK = 0xFFFFFFFFL + // The Display's tunnel has an MTU of 1400. + private const val MSS_V4 = 1360 + private const val MSS_V6 = 1340 + private const val TUNNEL_FRAMES = 8 + private const val WINDOW = 65535 + private const val CONNECT_MS = 10_000 + private const val IDLE_MS = 600_000L + private const val RETRANSMIT_MS = 1_500L + private const val POLL_MS = 1_000 + private const val UDP_MAX = 2048 + private const val UDP_IDLE_MS = 60_000L + // The request log's "source" for traffic of the tunnel rather than of the data fetcher. + private const val SOURCE_TUNNEL = -1 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMedia.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMedia.kt new file mode 100644 index 00000000..c11eba1c --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMedia.kt @@ -0,0 +1,257 @@ +package com.macedonga.visor.connection + +import android.app.ActivityManager +import android.content.ComponentName +import android.content.Context +import android.content.Intent +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.media.MediaMetadata +import android.media.Rating +import android.media.session.MediaController +import android.media.session.MediaSessionManager +import android.media.session.PlaybackState +import android.net.Uri +import android.os.Handler +import android.os.Looper +import android.provider.Settings +import androidx.core.app.NotificationManagerCompat +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.notify.MirrorListener +import com.macedonga.visor.phone.companionAssociated +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.ByteArrayOutputStream +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +/** + * svc9012, partner_ecosystem, for what the phone plays. The Display subscribes (type 5) on connect and takes, on that + * channel, MediaPlayerInfo {1 player, 2 content type, 3 liked, 4 package} (type 100) and the cover art as a JPEG in + * 1024-byte pieces (101, the last one 102; an empty 102 clears it), again whenever the playing session changes. With an + * Android phone the Display's music screen takes the cover only from here; title and artist come over AVRCP. + * Playlists are answered empty (9, 13). Like (15) sets the heart rating or the app's like action. Launch (1) starts an + * app from the foreground or as a companion only. Other requests are logged once and not answered. + */ +@Singleton +class PhoneMedia @Inject constructor(@ApplicationContext private val context: Context, private val log: Provider) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main + CoroutineExceptionHandler { _, _ -> }) + private val sending = Mutex() + private val main = Handler(Looper.getMainLooper()) + private val unserved = mutableSetOf() + + @Volatile private var subscriber: Pair? = null + private var sessions: MediaSessionManager? = null + @Volatile private var controller: MediaController? = null + private var sentInfo: ByteArray? = null + private var sentArt: Bitmap? = null + + private val onSessions = MediaSessionManager.OnActiveSessionsChangedListener { follow(it.orEmpty()) } + private val onChange = object : MediaController.Callback() { + override fun onMetadataChanged(metadata: MediaMetadata?) = push() + override fun onPlaybackStateChanged(state: PlaybackState?) = sessions?.let { follow(active(it)) } ?: Unit + override fun onSessionDestroyed() = sessions?.let { follow(active(it)) } ?: Unit + } + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + val reply = DataXConnection.mirror(message.info.channelId) + if (message.info.flags.isClosed) { + if (subscriber?.second == reply) subscriber = null + return + } + when (message.info.messageType) { + MT_SUBSCRIBE -> scope.launch { + subscriber = connection to reply + sentInfo = null + sentArt = null + watch() + push() + runCatching { sending.withLock { connection.send(reply, EMPTY, messageType = MT_SPOTIFY_PLAYLISTS_RESPONSE) } } + } + MT_INFO_REQUEST -> { + val (player, name) = player(controller) + connection.send(reply, Wire.fieldVarint(1, player) + Wire.fieldBytes(2, name.encodeToByteArray()), messageType = MT_INFO_RESPONSE) + } + MT_TRY_LAUNCH -> { + val f = fields(message) + val pkg = f?.bytes(2)?.decodeToString().orEmpty() + val ok = pkg.isNotEmpty() && launch(pkg) + if (!ok) log.get().add("Partner app $pkg: not started from the background") + connection.send(reply, Wire.fieldVarint(1, if (ok) 1 else 0) + Wire.fieldBytes(4, f?.bytes(3) ?: EMPTY), messageType = MT_TRY_LAUNCH) + } + MT_LIKE -> scope.launch { setLiked(fields(message)?.varint(1) == 1L) } + MT_NOTIFICATION_ACCESS -> scope.launch { + if (!startFromHere(Intent(Settings.ACTION_NOTIFICATION_LISTENER_SETTINGS))) log.get().add("Notification access: Visor may not open settings from the background") + } + MT_SPOTIFY_PLAYLISTS_REQUEST -> connection.send(reply, EMPTY, messageType = MT_SPOTIFY_PLAYLISTS_RESPONSE) + MT_PLAYLIST_REQUEST -> { + val partner = fields(message)?.bytes(1) ?: EMPTY + connection.send(reply, Wire.fieldBytes(3, partner), messageType = MT_PLAYLIST_RESPONSE) + } + else -> if (unserved.add(message.info.messageType)) log.get().add("Partner ecosystem: request ${message.info.messageType} not served") + } + } + + private fun fields(message: DataXMessage) = runCatching { Wire.parse(message.payload) }.getOrNull() + + /** From the first subscription on, follows the playing session; without notification access there is none. */ + private fun watch() { + if (sessions != null) return + val manager = context.getSystemService(MediaSessionManager::class.java) ?: return + runCatching { + manager.addOnActiveSessionsChangedListener(onSessions, listener(), main) + sessions = manager + follow(active(manager)) + } + } + + private fun listener() = ComponentName(context, MirrorListener::class.java) + + private fun active(manager: MediaSessionManager): List = + runCatching { manager.getActiveSessions(listener()) }.getOrDefault(emptyList()) + + // The playing session, or else the first one Android lists. + private fun follow(controllers: List) { + val next = controllers.firstOrNull { it.playbackState?.state == PlaybackState.STATE_PLAYING } ?: controllers.firstOrNull() + if (next?.sessionToken != controller?.sessionToken) { + controller?.unregisterCallback(onChange) + controller = next + next?.registerCallback(onChange, main) + } + push() + } + + // Sends what changed since the last push: the player, the cover, or both. + private fun push() { + val session = controller + scope.launch(Dispatchers.Default) { + sending.withLock { + val (mux, reply) = subscriber ?: return@withLock + val (player, name) = player(session) + val info = Wire.fieldVarint(1, player) + Wire.fieldVarint(2, CONTENT_UNKNOWN) + Wire.fieldVarint(3, liked(session)) + + Wire.fieldBytes(4, name.encodeToByteArray()) + val art = session?.metadata?.let { art(it) } + val newInfo = !info.contentEquals(sentInfo) + val newArt = !(art === sentArt || (art != null && sentArt?.let { art.sameAs(it) } == true)) + runCatching { + if (newInfo) mux.send(reply, info, messageType = MT_PLAYER_INFO) + if (newArt) sendArt(mux, reply, art) + sentInfo = info + sentArt = art + }.onFailure { if (subscriber?.first === mux) subscriber = null } + } + } + } + + private suspend fun sendArt(mux: DataXConnection, reply: Int, art: Bitmap?) { + if (art == null) return mux.send(reply, ByteArray(0), messageType = MT_ART_LAST) + val jpeg = ByteArrayOutputStream().also { fit(art).compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, it) }.toByteArray() + for (at in jpeg.indices step CHUNK) { + val end = minOf(at + CHUNK, jpeg.size) + mux.send(reply, Wire.fieldBytes(1, jpeg.copyOfRange(at, end)), messageType = if (end == jpeg.size) MT_ART_LAST else MT_ART_PIECE) + } + } + + // At most 300 x 300, as the Meta app sends it. + private fun fit(art: Bitmap): Bitmap { + if (art.width <= MAX_SIDE && art.height <= MAX_SIDE) return art + val scale = MAX_SIDE.toFloat() / maxOf(art.width, art.height) + return Bitmap.createScaledBitmap(art, (art.width * scale).toInt().coerceAtLeast(1), (art.height * scale).toInt().coerceAtLeast(1), true) + } + + private fun art(metadata: MediaMetadata): Bitmap? { + for (key in listOf(MediaMetadata.METADATA_KEY_ALBUM_ART, MediaMetadata.METADATA_KEY_ART, MediaMetadata.METADATA_KEY_DISPLAY_ICON)) { + metadata.getBitmap(key)?.let { return it } + } + for (key in listOf(MediaMetadata.METADATA_KEY_ALBUM_ART_URI, MediaMetadata.METADATA_KEY_ART_URI, MediaMetadata.METADATA_KEY_DISPLAY_ICON_URI)) { + val uri = metadata.getString(key)?.takeIf { it.isNotBlank() } ?: continue + runCatching { context.contentResolver.openInputStream(Uri.parse(uri))?.use { BitmapFactory.decodeStream(it) } }.getOrNull()?.let { return it } + } + return null + } + + // MediaPlayer: the partner apps the Display knows by name, any other player as 0 with its package. + private fun player(session: MediaController?): Pair { + if (context.packageName !in NotificationManagerCompat.getEnabledListenerPackages(context)) return NO_ACCESS to "" + val pkg = session?.packageName ?: return 0L to "" + return (PLAYERS[pkg] ?: 0L) to pkg + } + + // The heart rating of the track, when the session keeps one. + private fun liked(session: MediaController?): Long { + val rating = session?.metadata?.getRating(MediaMetadata.METADATA_KEY_USER_RATING) ?: return LIKED_UNKNOWN + if (rating.ratingStyle != Rating.RATING_HEART || !rating.isRated) return LIKED_UNKNOWN + return if (rating.hasHeart()) LIKED_YES else LIKED_NO + } + + // Heart rating when the session takes it, else its like action; the glasses get no reply, only the new state. + private fun setLiked(wanted: Boolean) { + val session = controller ?: return + val action = session.playbackState?.customActions?.firstOrNull { a -> + val words = a.action + " " + a.name + LIKE_WORDS.any { words.contains(it, ignoreCase = true) } && !NOT_LIKE.containsMatchIn(words) + } + when { + session.ratingType == Rating.RATING_HEART -> session.transportControls.setRating(Rating.newHeartRating(wanted)) + action == null -> log.get().add("Like: ${session.packageName} has no like control") + liked(session) != (if (wanted) LIKED_YES else LIKED_NO) -> session.transportControls.sendCustomAction(action.action, null) + } + push() + } + + // Starts the partner app as the Meta app would: only from the foreground or as a companion, else it stays put. + private fun launch(pkg: String): Boolean { + val intent = context.packageManager.getLaunchIntentForPackage(pkg) ?: return false + return startFromHere(intent) + } + + private fun startFromHere(intent: Intent): Boolean = + (onScreen() || companionAssociated(context)) && runCatching { context.startActivity(intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)) }.isSuccess + + private fun onScreen(): Boolean = ActivityManager.RunningAppProcessInfo() + .also(ActivityManager::getMyMemoryState).importance == ActivityManager.RunningAppProcessInfo.IMPORTANCE_FOREGROUND + + companion object { + const val SVC = 9012 + private const val MT_TRY_LAUNCH = 1 + private const val MT_INFO_REQUEST = 3 + private const val MT_INFO_RESPONSE = 4 + private const val MT_SUBSCRIBE = 5 + private const val MT_SPOTIFY_PLAYLISTS_REQUEST = 8 + private const val MT_SPOTIFY_PLAYLISTS_RESPONSE = 9 + private const val MT_NOTIFICATION_ACCESS = 11 + private const val MT_PLAYLIST_REQUEST = 12 + private const val MT_PLAYLIST_RESPONSE = 13 + private const val MT_LIKE = 15 + private const val MT_PLAYER_INFO = 100 + private const val MT_ART_PIECE = 101 + private const val MT_ART_LAST = 102 + private const val CONTENT_UNKNOWN = 0L + private const val LIKED_UNKNOWN = 0L + private const val LIKED_YES = 1L + private const val LIKED_NO = 2L + private const val NO_ACCESS = 7L + private const val CHUNK = 1024 + private const val MAX_SIDE = 300 + private const val JPEG_QUALITY = 70 + private val EMPTY = ByteArray(0) + private val LIKE_WORDS = listOf("like", "thumbs", "heart", "favorite", "favourite") + // Thumbs down and dislike sit beside the like action in many players. + private val NOT_LIKE = Regex("dislike|unlike|down|remove", RegexOption.IGNORE_CASE) + private val PLAYERS = mapOf( + "com.spotify.music" to 1L, "com.apple.android.music" to 2L, "com.amazon.mp3" to 3L, + "com.audible.application" to 4L, "com.clearchannel.iheartradio.controller" to 5L, "com.calm.android" to 6L, + ) + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt new file mode 100644 index 00000000..6143f752 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneMessages.kt @@ -0,0 +1,422 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.app.Activity +import android.app.PendingIntent +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.content.pm.PackageManager +import android.net.Uri +import android.os.Build +import android.provider.ContactsContract.PhoneLookup +import android.provider.Telephony +import android.telephony.PhoneNumberUtils +import android.telephony.SmsManager +import android.telephony.TelephonyManager +import androidx.core.content.ContextCompat +import com.macedonga.visor.R +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.notify.RecentChats +import com.macedonga.visor.notify.TextReplies +import dagger.hilt.android.qualifiers.ApplicationContext +import java.nio.ByteBuffer +import java.util.Locale +import java.util.UUID +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull + +/** + * svc70, smartglasses_messaging, for the phone's own texts (provider 3, native), as the Meta app serves them from + * Android's SMS store: the glasses load the threads (type 1) and one thread's messages (2), SMS and MMS together, and + * send a text (8). A conversation whose notification is up answers through the Messages app's reply, as RCS when it + * is one; any other goes as SMS, answered once the radio has sent it. Each request comes on a channel of its own and + * is answered with its own type; a reply over 1000 bytes goes as type-32 chunks when the request allows them. Texts + * that arrive reach the glasses as native notifications, from the SMS app's own notifications (see MirrorListener). + * The Phone app's recent calls (type 11) come on this service too; [PhoneCalls] answers them, and the call control + * (placing, answering) is [PhoneCallControl]'s. + * + * Google Messages files RCS chats as MMS marked restricted, which Android shows only to the default SMS app. Those chats + * come from what the SMS app's notifications showed ([RecentChats], this process only), listed and answered alongside. + */ +@Singleton +class PhoneMessages @Inject constructor( + @ApplicationContext private val context: Context, + private val requests: RequestLog, + private val log: Provider, + private val replies: TextReplies, + private val calls: PhoneCalls, + private val control: PhoneCallControl, + private val recent: RecentChats, + private val linkPrompt: LinkPrompt, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Messages: failed: $e") }) + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + // Call control has types with empty payloads, so it is checked before the text's. + if (message.info.messageType in PhoneCallControl.TYPES) return control.respond(message, connection) + if (message.payload.isEmpty()) return + val request = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return + val type = message.info.messageType + val reply = DataXConnection.mirror(message.info.channelId) + // The Phone app's recent calls, asked once per provider, each waiting for its answer. + if (type == MT_LOAD_CALL_LOG) return calls.loadCallLog(request, reply, connection) + if (type == LinkPrompt.MT_TRIGGER) return linkPrompt.trigger(request, reply, connection) + if (type !in SERVED) return log.get().add("Messages: type $type not served") + // The phone's texts, and replies to a chat notification (Messenger, WhatsApp, Instagram) through its reply action. + val provider = request.varint(1) + val chat = type == MT_OUTGOING && provider in CHAT + if (provider != NATIVE && !chat) return log.get().add("Messages: provider $provider not served") + scope.launch { + when (type) { + MT_LOAD_THREADS -> threads(request, reply, connection) + MT_LOAD_MESSAGES -> messages(request, reply, connection) + MT_OUTGOING -> send(request, reply, connection, chat) + } + } + } + + /** LoadThreadsRequest {2 request id, 3 count, 5 chunked}: the newest conversations, SMS, MMS and RCS alike. */ + private suspend fun threads(request: Wire.Fields, reply: Int, mux: DataXConnection) { + val id = requests.start("sms://threads", SOURCE_SMS) + val requestId = request.bytes(2) + if (!allowed(Manifest.permission.READ_SMS)) { + answer(mux, reply, MT_LOAD_THREADS, Wire.fieldBytes(1, requestId) + refused(), request.varint(5) != 0L) + return requests.finish(id, "SMS not allowed", failed = true) + } + val count = request.varint(3).toInt().takeIf { it > 0 } ?: DEFAULT_THREADS + val stored = conversations(count, null) + // The remembered chats the store does not hold, by thread id or by a number it has. + val listed = stored.map { t -> + Listed(t.id.toString(), t.addresses, t.date, t.snippet.ifBlank { messagesOf(t.id, 1).firstOrNull()?.text.orEmpty() }, "", t.addresses.size > 1) + } + recent.chats().filter { c -> + stored.none { s -> s.id.toString() == c.id || s.addresses.any { a -> c.numbers.any { RecentChats.sameNumber(a, it) } } } + }.map { c -> + Listed(c.id, c.numbers, c.messages.firstOrNull()?.time ?: 0L, c.messages.firstOrNull()?.text.orEmpty(), c.title, c.group) + } + val threads = listed.sortedByDescending { it.date }.take(count) + val names = HashMap() + var body = Wire.fieldBytes(1, requestId) + for (t in threads) { + val name = if (t.group && t.title.isNotBlank()) t.title else (t.numbers.joinToString(", ") { a -> names.getOrPut(a) { nameOf(a) } }.ifEmpty { t.title }) + // SgThread: 1 id, 2 name, 3 type (1 group), 4 latest message, 5 its time, 6 Participant {1 id}. + body += Wire.fieldBytes( + 2, + Wire.fieldBytes(1, t.id.encodeToByteArray()) + Wire.fieldBytes(2, name.encodeToByteArray()) + + (if (t.group) Wire.fieldVarint(3, THREAD_GROUP) else ByteArray(0)) + + Wire.fieldBytes(4, t.snippet.encodeToByteArray()) + Wire.fieldVarint(5, t.date) + + t.numbers.fold(ByteArray(0)) { acc, a -> acc + Wire.fieldBytes(6, Wire.fieldBytes(1, a.encodeToByteArray())) }, + ) + } + body += Wire.fieldVarint(3, threads.maxOfOrNull { it.date } ?: 0) + answer(mux, reply, MT_LOAD_THREADS, body, request.varint(5) != 0L) + requests.finish(id, "${threads.size} threads", failed = false, bytes = body.size.toLong()) + } + + /** LoadMessagesRequest {2 request id, 3 count, 4 thread id, 8 chunked, 9 phone numbers}: oldest first. */ + private suspend fun messages(request: Wire.Fields, reply: Int, mux: DataXConnection) { + val requestId = request.bytes(2) + val numbers = request.repeatedBytes(9).map { it.decodeToString() }.filter { it.isNotBlank() } + val chunked = request.varint(8) != 0L + val key = request.bytes(4).decodeToString() + val id = requests.start("sms://thread/$key", SOURCE_SMS) + if (!allowed(Manifest.permission.READ_SMS)) { + answer(mux, reply, MT_LOAD_MESSAGES, Wire.fieldBytes(1, requestId) + refused(), chunked) + return requests.finish(id, "SMS not allowed", failed = true) + } + val thread = key.toLongOrNull() + ?: numbers.takeIf { it.isNotEmpty() }?.let { runCatching { Telephony.Threads.getOrCreateThreadId(context, it.toSet()) }.getOrNull() } + val count = request.varint(3).toInt().takeIf { it > 0 }?.coerceAtMost(MAX_MESSAGES) ?: DEFAULT_MESSAGES + // A store thread that came back empty, or a chat the store does not hold, is answered from what its notifications showed. + val rows = (thread?.let { messagesOf(it, count) }.orEmpty()).ifEmpty { recentMessages(key, thread, numbers, count) } + var body = Wire.fieldBytes(1, requestId) + for (m in rows.asReversed()) { + // SgMessage: 1 id, 2 text, 3 time, 4 the address (the other side's, in or out), 7 attachment, 19 box, 20 service. + body += Wire.fieldBytes( + 2, + Wire.fieldBytes(1, m.id.encodeToByteArray()) + Wire.fieldBytes(2, m.text.encodeToByteArray()) + + Wire.fieldVarint(3, m.date) + Wire.fieldBytes(4, m.address.encodeToByteArray()) + + (if (m.attachment) Wire.fieldVarint(7, 1) else ByteArray(0)) + + Wire.fieldBytes(19, folder(m.box).encodeToByteArray()) + Wire.fieldVarint(20, m.service), + ) + } + body += Wire.fieldVarint(5, RESPONSE_FILE_BUS) + Wire.fieldBytes(7, (thread?.toString() ?: key.ifEmpty { "0" }).encodeToByteArray()) + answer(mux, reply, MT_LOAD_MESSAGES, body, chunked) + requests.finish(id, "${rows.size} messages", failed = false, bytes = body.size.toLong()) + } + + /** A chat from the notifications, newest first, in the same shape as the store's messages. */ + private fun recentMessages(key: String, thread: Long?, numbers: List, count: Int): List { + val chat = recent.find(key) ?: thread?.let { recent.find(it.toString()) } ?: recent.withNumber(numbers) ?: return emptyList() + return chat.messages.take(count).map { m -> + val other = if (m.outgoing) chat.numbers.firstOrNull().orEmpty() else m.number + val box = if (m.outgoing) Telephony.Sms.MESSAGE_TYPE_SENT else Telephony.Sms.MESSAGE_TYPE_INBOX + Message("recent://${chat.id}/${m.time}", m.text, m.time, other, box, SERVICE_SMS, false) + } + } + + /** OutgoingMessageRequest {2 request id, 3 thread id, 4 text, 7 participants}: OutgoingMessageResponse {1, 2 status, 3 error}. */ + private suspend fun send(request: Wire.Fields, reply: Int, mux: DataXConnection, chat: Boolean) { + val requestId = request.bytes(2) + val text = request.bytes(4).decodeToString() + val thread = request.bytes(3).decodeToString() + val to = request.repeatedBytes(7).map { it.decodeToString() }.filter { it.isNotBlank() }.ifEmpty { addressesOf(thread) } + // A conversation with its notification up answers through the Messages app, as RCS when it is one. A chat only does that. + val viaApp = text.isNotBlank() && (chat || replies.has(thread)) + val id = requests.start("${if (viaApp) "messages" else "sms"}://send/${to.joinToString(",").ifEmpty { thread }}", SOURCE_SMS) + val error = when { + viaApp -> replies.reply(context, thread, text) + chat -> if (text.isBlank()) "Empty message" else "No reply in that conversation" + !allowed(Manifest.permission.SEND_SMS) -> "SMS not allowed in Visor" + to.isEmpty() -> "No recipient" + to.size > 1 -> "Group texts are not supported" + text.isBlank() -> "Empty message" + else -> deliver(to.single(), text) + } + val body = Wire.fieldBytes(1, requestId) + Wire.fieldVarint(2, if (error == null) OUT_SUCCESS else OUT_FAILURE) + + (error?.let { Wire.fieldBytes(3, it.encodeToByteArray()) } ?: ByteArray(0)) + // The glasses may have closed the channel by now; the status is then theirs to miss. + runCatching { mux.send(reply, body, messageType = MT_OUTGOING) } + requests.finish(id, error ?: "sent", failed = error != null, bytes = text.length.toLong()) + } + + /** Sends one text and waits for the radio's word on every part: null when all went, else why not. */ + private suspend fun deliver(to: String, text: String): String? { + val sms = if (Build.VERSION.SDK_INT >= 31) context.getSystemService(SmsManager::class.java) else @Suppress("DEPRECATION") SmsManager.getDefault() + val parts = sms.divideMessage(text) + val action = "${context.packageName}.SMS_SENT.${UUID.randomUUID()}" + val results = Channel(parts.size) + val receiver = object : BroadcastReceiver() { + override fun onReceive(c: Context, intent: Intent) { + results.trySend(resultCode) + } + } + ContextCompat.registerReceiver(context, receiver, IntentFilter(action), ContextCompat.RECEIVER_NOT_EXPORTED) + try { + val sent = ArrayList( + parts.indices.map { + PendingIntent.getBroadcast(context, it, Intent(action).setPackage(context.packageName), PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_ONE_SHOT) + }, + ) + // Android files a text a non-default app sends in the SMS store itself. + if (parts.size == 1) sms.sendTextMessage(to, null, text, sent[0], null) else sms.sendMultipartTextMessage(to, null, parts, sent, null) + val codes = withTimeoutOrNull(SEND_TIMEOUT_MS) { List(parts.size) { results.receive() } } ?: return "No answer from the phone's radio" + return when (codes.firstOrNull { it != Activity.RESULT_OK }) { + null -> null + SmsManager.RESULT_ERROR_RADIO_OFF -> "The phone's radio is off" + SmsManager.RESULT_ERROR_NO_SERVICE -> "No service" + else -> "Not sent (error ${codes.first { it != Activity.RESULT_OK }})" + } + } catch (e: Exception) { + return e.message ?: e.toString() + } finally { + context.unregisterReceiver(receiver) + } + } + + private suspend fun answer(mux: DataXConnection, reply: Int, type: Int, body: ByteArray, chunked: Boolean) { + if (!chunked || body.size <= CHUNK) return mux.send(reply, body, messageType = type) + // The Meta app's chunk: the reply's type, 36 zero bytes, the index, 1000, the count, then up to 1000 bytes. + val total = (body.size + CHUNK - 1) / CHUNK + for (i in 0 until total) { + val head = ByteBuffer.allocate(CHUNK_HEAD).putInt(type).put(ByteArray(36)).putInt(i).putInt(CHUNK).putInt(total).array() + mux.send(reply, head + body.copyOfRange(i * CHUNK, minOf((i + 1) * CHUNK, body.size)), messageType = MT_CHUNK) + } + } + + /** The numbers of a thread's other side. */ + fun addressesOf(thread: String): List { + val stored = thread.toLongOrNull()?.takeIf { allowed(Manifest.permission.READ_SMS) }?.let { conversations(1, it).firstOrNull()?.addresses } + return stored ?: recent.find(thread)?.numbers.orEmpty() + } + + private class Conversation(val id: Long, val date: Long, val addresses: List, val snippet: String) + + /** A thread for the list: from the store, or a remembered chat (its id is the notification's thread). */ + private class Listed(val id: String, val numbers: List, val date: Long, val snippet: String, val title: String, val group: Boolean) + + /** Conversations newest first, from Android's thread list, which holds SMS, MMS and the RCS chats Messages files as MMS. */ + private fun canonical(): Map { + val canonical = HashMap() + context.contentResolver.query(CANONICAL_ADDRESSES, arrayOf("_id", "address"), null, null, null)?.use { c -> + while (c.moveToNext()) canonical[c.getLong(0)] = c.getString(1).orEmpty() + } + return canonical + } + + /** For the debug receiver: the newest rows of the thread list as Visor reads them, ids and counts only. */ + fun describeThreads(count: Int): String { + val canonical = canonical() + val columns = arrayOf(Telephony.Threads._ID, Telephony.Threads.DATE, Telephony.Threads.RECIPIENT_IDS, Telephony.Threads.SNIPPET, Telephony.Threads.MESSAGE_COUNT) + fun rows(uri: Uri, where: String? = null) = context.contentResolver.query(uri, arrayOf("_id"), where, null, null)?.use { it.count } ?: -1 + val lines = mutableListOf( + "canonical addresses: ${canonical.size}, sms ${rows(Telephony.Sms.CONTENT_URI)}, mms ${rows(Telephony.Mms.CONTENT_URI)}", + "thread 489: sms ${rows(Telephony.Sms.CONTENT_URI, "thread_id=489")}, mms ${rows(Telephony.Mms.CONTENT_URI, "thread_id=489")}; " + + "thread 509: sms ${rows(Telephony.Sms.CONTENT_URI, "thread_id=509")}; listed ${rows(CONVERSATIONS, "_id IN (489,509)")}", + ) + context.contentResolver.query(CONVERSATIONS, columns, null, null, "${Telephony.Threads.DATE} DESC")?.use { c -> + lines += "rows: ${c.count}" + while (c.moveToNext() && lines.size < count + 2) { + val ids = c.getString(2).orEmpty() + val known = ids.split(' ').count { canonical[it.toLongOrNull()]?.isNotBlank() == true } + lines += "${c.getLong(0)} date=${c.getLong(1)} messages=${c.getInt(4)} recipients=[$ids] resolved=$known" + } + } ?: lines.add("no cursor") + return lines.joinToString("\n") + } + + private fun conversations(count: Int, only: Long?): List { + val canonical = canonical() + val out = mutableListOf() + val where = only?.let { "${Telephony.Threads._ID} = $it" } + val columns = arrayOf(Telephony.Threads._ID, Telephony.Threads.DATE, Telephony.Threads.RECIPIENT_IDS, Telephony.Threads.SNIPPET, Telephony.Threads.MESSAGE_COUNT) + context.contentResolver.query(CONVERSATIONS, columns, where, null, "${Telephony.Threads.DATE} DESC")?.use { c -> + while (c.moveToNext() && out.size < count) { + if (c.getInt(4) == 0) continue + val addresses = c.getString(2).orEmpty().split(' ').mapNotNull { canonical[it.toLongOrNull()]?.takeIf(String::isNotBlank) } + if (addresses.isEmpty()) continue + out += Conversation(c.getLong(0), c.getLong(1), addresses, c.getString(3).orEmpty()) + } + } + return out + } + + private class Message(val id: String, val text: String, val date: Long, val address: String, val box: Int, val service: Long, val attachment: Boolean) + + /** A thread's newest messages, SMS and MMS together, newest first. */ + private fun messagesOf(thread: Long, count: Int): List { + val out = mutableListOf() + read("${Telephony.Sms.THREAD_ID} = ?", arrayOf(thread.toString())) { + out += Message("content://sms/${it.id}", it.body, it.date, it.address, it.box, SERVICE_SMS, false) + out.size < count + } + val mms = mutableListOf() + val columns = arrayOf(Telephony.Mms._ID, Telephony.Mms.DATE, Telephony.Mms.MESSAGE_BOX) + context.contentResolver.query(Telephony.Mms.CONTENT_URI, columns, "${Telephony.Mms.THREAD_ID} = ?", arrayOf(thread.toString()), "${Telephony.Mms.DATE} DESC")?.use { c -> + while (c.moveToNext() && mms.size < count) { + val id = c.getLong(0) + val box = c.getInt(2) + val (text, kind) = mmsText(id) + // An MMS keeps its date in seconds. + mms += Message("content://mms/$id", text.ifBlank { kind.orEmpty() }, c.getLong(1) * 1000, mmsAddress(id, box), box, SERVICE_MMS, kind != null) + } + } + return (out + mms).sortedByDescending { it.date }.take(count) + } + + /** An MMS's text parts, and a word for its picture or video when it has one. */ + private fun mmsText(id: Long): Pair { + val text = StringBuilder() + var kind: String? = null + context.contentResolver.query(MMS_PARTS, arrayOf(Telephony.Mms.Part.CONTENT_TYPE, Telephony.Mms.Part.TEXT), "${Telephony.Mms.Part.MSG_ID} = ?", arrayOf(id.toString()), null)?.use { c -> + while (c.moveToNext()) { + val type = c.getString(0).orEmpty() + when { + type == "text/plain" -> c.getString(1)?.let { if (text.isNotEmpty()) text.append('\n'); text.append(it) } + type.startsWith("image/") -> kind = kind ?: context.getString(R.string.texts_photo) + type.startsWith("video/") -> kind = kind ?: context.getString(R.string.texts_video) + type != "application/smil" -> kind = kind ?: context.getString(R.string.texts_attachment) + } + } + } + return text.toString() to kind + } + + /** The other side of an MMS: who sent one that came in, who one sent went to. */ + private fun mmsAddress(id: Long, box: Int): String { + val wanted = if (box == Telephony.Mms.MESSAGE_BOX_INBOX) PDU_FROM else PDU_TO + context.contentResolver.query(Uri.parse("content://mms/$id/addr"), arrayOf(Telephony.Mms.Addr.ADDRESS, Telephony.Mms.Addr.TYPE), null, null, null)?.use { c -> + while (c.moveToNext()) if (c.getInt(1) == wanted) c.getString(0)?.takeIf { it.isNotBlank() && it != INSERT_ADDRESS }?.let { return it } + } + return "" + } + + private class Sms(val id: Long, val thread: Long, val address: String, val body: String, val date: Long, val box: Int) + + /** SMS rows newest first, handed to [more] until it says stop. */ + private fun read(selection: String?, args: Array?, more: (Sms) -> Boolean) { + val columns = arrayOf(Telephony.Sms._ID, Telephony.Sms.THREAD_ID, Telephony.Sms.ADDRESS, Telephony.Sms.BODY, Telephony.Sms.DATE, Telephony.Sms.TYPE) + context.contentResolver.query(Telephony.Sms.CONTENT_URI, columns, selection, args, "${Telephony.Sms.DATE} DESC")?.use { c -> + while (c.moveToNext()) { + val address = c.getString(2).orEmpty() + // A draft with no number yet is not a conversation. + if (address.isBlank()) continue + if (!more(Sms(c.getLong(0), c.getLong(1), address, c.getString(3).orEmpty(), c.getLong(4), c.getInt(5)))) break + } + } + } + + /** The contact's name for a number, else the number as the phone writes it. */ + private fun nameOf(number: String): String { + if (allowed(Manifest.permission.READ_CONTACTS)) { + val uri = Uri.withAppendedPath(PhoneLookup.CONTENT_FILTER_URI, Uri.encode(number)) + runCatching { + context.contentResolver.query(uri, arrayOf(PhoneLookup.DISPLAY_NAME), null, null, null)?.use { if (it.moveToFirst()) it.getString(0) else null } + }.getOrNull()?.takeIf { it.isNotBlank() }?.let { return it } + } + val country = context.getSystemService(TelephonyManager::class.java)?.simCountryIso?.uppercase(Locale.ROOT)?.ifEmpty { null } ?: Locale.getDefault().country + return PhoneNumberUtils.formatNumber(number, country) ?: number + } + + private fun allowed(permission: String) = context.checkSelfPermission(permission) == PackageManager.PERMISSION_GRANTED + + /** SgError {1 title, 2 message} in a reply's field 6. */ + private fun refused(): ByteArray = + Wire.fieldBytes(6, Wire.fieldBytes(1, "SMS not allowed".encodeToByteArray()) + Wire.fieldBytes(2, "Allow SMS for Visor on the phone.".encodeToByteArray())) + + companion object { + const val SVC = 70 + private const val MT_LOAD_THREADS = 1 + private const val MT_LOAD_MESSAGES = 2 + private const val MT_OUTGOING = 8 + private const val MT_LOAD_CALL_LOG = 11 + private const val MT_CHUNK = 32 + private val SERVED = setOf(MT_LOAD_THREADS, MT_LOAD_MESSAGES, MT_OUTGOING) + private const val NATIVE = 3L + private val CHAT = setOf(1L, 2L, 4L) + private const val DEFAULT_THREADS = 10 + private const val DEFAULT_MESSAGES = 25 + private const val MAX_MESSAGES = 100 + private const val SERVICE_SMS = 1L + private const val SERVICE_MMS = 2L + private const val THREAD_GROUP = 1L + private const val PDU_FROM = 137 + private const val PDU_TO = 151 + private const val INSERT_ADDRESS = "insert-address-token" + private val CONVERSATIONS: Uri = Telephony.Threads.CONTENT_URI.buildUpon().appendQueryParameter("simple", "true").build() + private val CANONICAL_ADDRESSES: Uri = Uri.parse("content://mms-sms/canonical-addresses") + private val MMS_PARTS: Uri = Uri.parse("content://mms/part") + private const val RESPONSE_FILE_BUS = 2L + private const val OUT_SUCCESS = 2L + private const val OUT_FAILURE = 3L + private const val SEND_TIMEOUT_MS = 30_000L + private const val CHUNK = 1000 + private const val CHUNK_HEAD = 52 + private const val SOURCE_SMS = -4 + + // Telephony's box numbers as the Meta app names them. Lowercase: the glasses mark a message as sent only for + // "sent", and any other one of ours shows as still sending. + private fun folder(box: Int) = when (box) { + Telephony.Sms.MESSAGE_TYPE_INBOX -> "inbox" + Telephony.Sms.MESSAGE_TYPE_SENT -> "sent" + Telephony.Sms.MESSAGE_TYPE_DRAFT -> "draft" + Telephony.Sms.MESSAGE_TYPE_OUTBOX -> "outbox" + Telephony.Sms.MESSAGE_TYPE_FAILED -> "failed" + Telephony.Sms.MESSAGE_TYPE_QUEUED -> "queued" + else -> "inbox" + } + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhonePosition.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhonePosition.kt new file mode 100644 index 00000000..714ad492 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhonePosition.kt @@ -0,0 +1,84 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.annotation.SuppressLint +import android.content.Context +import android.content.pm.PackageManager +import android.location.Location +import android.location.LocationListener +import android.location.LocationManager +import android.os.Build +import android.os.CancellationSignal +import android.os.Looper +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Singleton +import kotlin.coroutines.resume +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withTimeoutOrNull + +/** + * The phone's position for navigation and place search, from the provider [LocationShare] uses. Updates run only + * while guidance does; otherwise the system's last known fix answers. + */ +@Singleton +class PhonePosition @Inject constructor(@ApplicationContext private val context: Context) { + private val manager = context.getSystemService(LocationManager::class.java) + private val listener = LocationListener { last = it } + + @Volatile + private var last: Location? = null + private var users = 0 + + /** Asks for updates while guidance runs. Each call is paired with [release]. */ + @Synchronized + @SuppressLint("MissingPermission") + fun use() { + if (users++ == 0 && manager != null && granted()) { + manager.requestLocationUpdates(provider(manager), INTERVAL_MS, 0f, listener, Looper.getMainLooper()) + } + } + + @Synchronized + fun release() { + if (users > 0 && --users == 0) manager?.removeUpdates(listener) + } + + /** The latest fix while updates run, else the system's last known one. Null without the permission. */ + @SuppressLint("MissingPermission") + fun current(): LatLng? { + if (manager == null || !granted()) return null + val fix = if (users > 0) last else null + return (fix ?: manager.getLastKnownLocation(provider(manager)))?.let { LatLng(it.latitude, it.longitude) } + } + + /** A fix taken now, for when none is known yet; null without one in [timeoutMs]. */ + @SuppressLint("MissingPermission") + suspend fun fresh(timeoutMs: Long = FRESH_MS): LatLng? { + if (manager == null || !granted() || Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return null + return withTimeoutOrNull(timeoutMs) { + suspendCancellableCoroutine { cont -> + val cancel = CancellationSignal() + cont.invokeOnCancellation { cancel.cancel() } + manager.getCurrentLocation(provider(manager), cancel, context.mainExecutor) { fix -> + cont.resume(fix?.let { LatLng(it.latitude, it.longitude) }) + } + } + } + } + + private fun granted(): Boolean = + listOf(Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_COARSE_LOCATION) + .any { context.checkSelfPermission(it) == PackageManager.PERMISSION_GRANTED } + + private fun provider(m: LocationManager): String = when { + m.allProviders.contains(LocationManager.FUSED_PROVIDER) -> LocationManager.FUSED_PROVIDER + m.isProviderEnabled(LocationManager.GPS_PROVIDER) -> LocationManager.GPS_PROVIDER + else -> LocationManager.NETWORK_PROVIDER + } + + private companion object { + const val INTERVAL_MS = 1_000L + const val FRESH_MS = 5_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PhoneSharing.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneSharing.kt new file mode 100644 index 00000000..14b4daab --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PhoneSharing.kt @@ -0,0 +1,178 @@ +package com.macedonga.visor.connection + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.media.AirshipReceiver +import com.macedonga.visor.core.media.ExtraFiles +import com.macedonga.visor.core.media.NAMESPACE_SHARING +import com.macedonga.visor.core.media.Offer +import com.macedonga.visor.core.media.Received +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.settings.hasDisplay +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.File +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull + +/** + * svc26, smartglasses_sharing: the glasses share a capture through the phone. Visor opens the channel on connect with + * an empty enumerate request (3), as the Meta app does; the glasses' sharing service refuses to start without it. A + * photo comes as an offer (7, or 11 for the last capture) that Visor accepts (8), then over airship "Sharing" under the + * offer's uuid, then a media intent (5) naming where it goes. A share to the phone's own texts (provider 3) goes out + * as an MMS and is answered (6) with the result; other destinations are the Meta app's and are turned down. The + * assistant's photo comes the same way and still reaches the assistant. + */ +@Singleton +class PhoneSharing @Inject constructor( + @ApplicationContext private val context: Context, + private val connection: Provider, + private val airship: AirshipReceiver, + private val messages: PhoneMessages, + private val kind: LinkKind, + private val requests: RequestLog, + private val log: Provider, +) : Responder, ExtraFiles { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Sharing: failed: $e") }) + private val files = ConcurrentHashMap>() + + fun start(scope: CoroutineScope) { + scope.launch { + connection.get().state.map { it.link == LinkState.READY }.distinctUntilChanged().collect { ready -> + // Only the Display's sharing is used; other glasses keep their own. + if (!ready || !hasDisplay(connection.get().state.value.info?.build)) return@collect + if (this@PhoneSharing !in airship.extras) airship.extras.add(0, this@PhoneSharing) + connection.get().submit { it.sendOnService(SVC, MT_ENUMERATE_REQUEST, ByteArray(0)) } + } + } + } + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.payload.isEmpty() && message.info.flags.isClosed) return + val type = message.info.messageType + // The channel is Visor's own: the glasses answer on it without the originator bit, and Visor writes it with. + val reply = message.info.channelId xor 0x8000 + val f = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return + log.get().add("Sharing: type $type, ${message.payload.size} bytes, fields ${f.fieldNumbers()}") + when (type) { + MT_MEDIA_OFFER, MT_MEDIA_OFFER_LAST -> { + val uuid = f.bytes(1) + files.getOrPut(uuidText(uuid)) { CompletableDeferred() } + connection.send(reply, Wire.fieldBytes(1, uuid) + Wire.fieldVarint(2, ACCEPT), messageType = MT_MEDIA_OFFER_RESPONSE) + } + // The phone keeps the link it has: the glasses are told which one that is. + MT_LINK_SWITCH -> connection.send(reply, Wire.fieldVarint(1, if (kind.overLe.value) LINK_BLE else LINK_BTC), messageType = MT_LINK_UPDATE) + MT_MEDIA_INTENT -> scope.launch { intent(f, reply, connection) } + } + } + + override fun wants(offer: Offer): Boolean = offer.namespace == NAMESPACE_SHARING && files.containsKey(offer.uuid) + + // The receiver deletes its file afterwards, so the photo is copied; the assistant may want it too. + override fun take(received: Received) { + val kept = File(File(context.cacheDir, "share").apply { mkdirs() }, received.offer.uuid) + received.file.copyTo(kept, overwrite = true) + files[received.offer.uuid]?.complete(kept) + airship.extras.filter { it !== this && it.wants(received.offer) }.forEach { runCatching { it.take(received) } } + } + + /** MediaIntent {1 uuid, 2 media uuid, 4 ShareIntent, 8 AssistantOffloadIntent}: answered with MediaIntentResponse. */ + private suspend fun intent(f: Wire.Fields, reply: Int, mux: DataXConnection) { + val uuid = f.bytes(1) + // A share of a photo just offered names no media uuid: the intent's own uuid is the offer's. + val media = uuidText(if (f.has(2)) f.bytes(2) else uuid) + try { + when { + f.has(INTENT_SHARE) -> { + val (result, error) = try { + share(Wire.parse(f.bytes(INTENT_SHARE)), media) + } catch (e: Exception) { + SHARE_ERROR_APP to (e.message ?: e.toString()) + } + val response = (error?.let { Wire.fieldBytes(1, it.encodeToByteArray()) } ?: ByteArray(0)) + Wire.fieldVarint(2, result) + + (if (result == SHARE_SUCCESS) Wire.fieldVarint(3, System.currentTimeMillis()) else ByteArray(0)) + mux.send(reply, Wire.fieldBytes(1, uuid) + Wire.fieldBytes(RESPONSE_SHARE, response), messageType = MT_MEDIA_INTENT_RESPONSE) + } + // The assistant has the photo from the airship transfer. + f.has(INTENT_ASSISTANT_OFFLOAD) -> + mux.send(reply, Wire.fieldBytes(1, uuid) + Wire.fieldBytes(RESPONSE_ASSISTANT_OFFLOAD, Wire.fieldVarint(1, OFFLOAD_SUCCESS)), messageType = MT_MEDIA_INTENT_RESPONSE) + else -> log.get().add("Sharing: intent ${f.fieldNumbers()} not served") + } + } finally { + files.remove(media)?.let { d -> if (d.isCompleted) runCatching { d.getCompleted().delete() } } + } + } + + /** ShareIntent {2 ShareDestination {1 provider, 3 thread, 6 numbers}, 5 text, 8 share id}: the result and why not. */ + private suspend fun share(share: Wire.Fields, media: String): Pair { + val destination = Wire.parse(share.bytes(2)) + val id = requests.start("mms://share/${share.bytes(8).decodeToString()}", SOURCE_SMS) + val provider = destination.varint(1) + val outcome: Pair = if (provider != PROVIDER_NATIVE) { + SHARE_ERROR_APP to "Only the phone's texts can be sent through Visor" + } else { + val numbers = destination.repeatedBytes(6).map { Wire.parse(it).bytes(1).decodeToString() }.filter { it.isNotBlank() } + val to = numbers.ifEmpty { messages.addressesOf(destination.bytes(3).decodeToString()) } + val file = files[media]?.let { withTimeoutOrNull(FILE_WAIT_MS) { it.await() } } + when { + context.checkSelfPermission(Manifest.permission.SEND_SMS) != PackageManager.PERMISSION_GRANTED -> SHARE_ERROR_NATIVE_DISABLED to "SMS not allowed in Visor" + to.isEmpty() -> SHARE_ERROR_APP to "No recipient" + to.size > 1 -> SHARE_ERROR_APP to "Group messages are not supported" + file == null -> SHARE_ERROR_MEDIA_PROCESSING to "The picture did not arrive" + else -> Mms.send(context, to.single(), file.readBytes(), share.bytes(5).decodeToString())?.let { SHARE_ERROR_APP to it } ?: (SHARE_SUCCESS to null) + } + } + requests.finish(id, outcome.second ?: "sent as MMS", failed = outcome.second != null) + return outcome + } + + private fun uuidText(raw: ByteArray): String { + val h = raw.joinToString("") { "%02x".format(it) } + if (raw.size != 16) return h + return "${h.substring(0, 8)}-${h.substring(8, 12)}-${h.substring(12, 16)}-${h.substring(16, 20)}-${h.substring(20)}" + } + + companion object { + const val SVC = 26 + private const val MT_ENUMERATE_REQUEST = 3 + private const val MT_MEDIA_INTENT = 5 + private const val MT_MEDIA_INTENT_RESPONSE = 6 + private const val MT_MEDIA_OFFER = 7 + private const val MT_MEDIA_OFFER_RESPONSE = 8 + private const val MT_MEDIA_OFFER_LAST = 11 + private const val MT_LINK_SWITCH = 12 + private const val MT_LINK_UPDATE = 13 + private const val ACCEPT = 0L + private const val LINK_BLE = 1L + private const val LINK_BTC = 2L + private const val INTENT_SHARE = 4 + private const val INTENT_ASSISTANT_OFFLOAD = 8 + private const val RESPONSE_SHARE = 2 + private const val RESPONSE_ASSISTANT_OFFLOAD = 5 + private const val OFFLOAD_SUCCESS = 1L + private const val PROVIDER_NATIVE = 3L + private const val SHARE_SUCCESS = 1L + private const val SHARE_ERROR_MEDIA_PROCESSING = 2L + private const val SHARE_ERROR_APP = 5L + private const val SHARE_ERROR_NATIVE_DISABLED = 8L + private const val FILE_WAIT_MS = 90_000L + private const val SOURCE_SMS = -4 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/PlaceSearch.kt b/app/src/main/kotlin/com/macedonga/visor/connection/PlaceSearch.kt new file mode 100644 index 00000000..ec8cfea1 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/PlaceSearch.kt @@ -0,0 +1,236 @@ +package com.macedonga.visor.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import java.net.URLEncoder +import java.util.Locale +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlin.math.cos +import kotlin.math.roundToLong +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import org.json.JSONArray +import org.json.JSONObject + +/** + * svc55072, smartglasses_place_search: the glasses search for a place (PlaceSearchRequest, types 0 and 1 {1 query, + * 2 radius in metres, 3 search type, 4 requestId, 5 prewarm}) and get PlaceSearchResponse (type 2) with places from + * OpenStreetMap's Nominatim within the radius around the phone, nearest first. Nominatim reads a kind of place + * ("cafe", "park") as one, and also finds names and addresses. As the Meta app does, a prewarm is answered at once + * with no places, and its search fills a five-minute cache for the real request. Media requests (3) get an empty + * PlaceMediaResponse (4). + */ +@Singleton +class PlaceSearch @Inject constructor( + private val position: PhonePosition, + private val requests: RequestLog, + private val log: Provider, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO + CoroutineExceptionHandler { _, e -> log.get().add("Place search: failed: $e") }) + // Nominatim allows one request a second, and the glasses send their prewarms together. + private val nominatim = Mutex() + private var lastAsked = 0L + private val cache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry) = size > CACHE_SIZE + } + + private class Cached(val at: Long, val places: List) + + private class Place( + val id: String, + val name: String, + val at: LatLng, + val street: String, + val city: String, + val postcode: String, + val country: String, + val category: String, + val phone: String, + ) + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.info.flags.isClosed) return + val reply = DataXConnection.mirror(message.info.channelId) + when (message.info.messageType) { + MT_SEARCH, MT_SEARCH_REQUEST -> { + val req = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return + val query = req.bytes(1).decodeToString().trim() + val radius = req.float(2).takeIf { it > 0f }?.toDouble() ?: DEFAULT_RADIUS_M + if (req.varint(5) != 0L) { + connection.send(reply, Wire.fieldVarint(1, RESULT_SUCCESS), messageType = MT_SEARCH_RESPONSE) + scope.launch { places(query, radius, here()) } + return + } + scope.launch { connection.send(reply, answer(query, radius), messageType = MT_SEARCH_RESPONSE) } + } + MT_MEDIA_REQUEST -> connection.send(reply, ByteArray(0), messageType = MT_MEDIA_RESPONSE) + } + } + + private suspend fun here(): LatLng? = position.current() ?: position.fresh() + + private suspend fun answer(query: String, radius: Double): ByteArray { + val logId = requests.start("place://$query", SOURCE_MAPS) + val here = here() + val found = try { + places(query, radius, here) + } catch (e: IOException) { + requests.finish(logId, e.message ?: "search failed", failed = true) + log.get().add("Place search: \"$query\" failed: ${e.message}") + return error(e.message ?: "Search failed", ERROR_NETWORK) + } catch (e: Exception) { + requests.finish(logId, e.message ?: "search failed", failed = true) + log.get().add("Place search: \"$query\" failed: $e") + return error(e.message ?: "Search failed", ERROR_INTERNAL) + } + requests.finish(logId, "${found.size} places", failed = false) + log.get().add("Place search: \"$query\", ${found.size} places" + if (here == null) ", without the phone's position" else "") + var body = Wire.fieldVarint(1, RESULT_SUCCESS) + for (p in found) body += Wire.fieldBytes(2, encode(p, here)) + return body + } + + /** From the cache when the same search near the same spot ran in the last five minutes. */ + private suspend fun places(query: String, radius: Double, here: LatLng?): List { + if (query.isEmpty()) return emptyList() + val key = query.lowercase(Locale.ROOT) + "@" + (here?.let { "%.3f,%.3f".format(Locale.ROOT, it.lat, it.lng) } ?: "") + val now = System.currentTimeMillis() + synchronized(cache) { cache[key]?.takeIf { now - it.at < CACHE_MS }?.let { return it.places } } + val found = if (here == null) { + search(query, null, bounded = false) + } else { + // A name farther than the radius is still found, the nearest first. + val box = box(here, radius) + search(query, box, bounded = true).ifEmpty { search(query, box, bounded = false) }.sortedBy { haversine(here, it.at) } + } + synchronized(cache) { cache[key] = Cached(now, found) } + return found + } + + /** The viewbox Nominatim takes: west, north, east, south. */ + private fun box(here: LatLng, radius: Double): String { + val r = radius.coerceIn(MIN_RADIUS_M, MAX_RADIUS_M) + val dLat = r / METRES_PER_DEGREE + val dLng = r / (METRES_PER_DEGREE * cos(Math.toRadians(here.lat)).coerceAtLeast(MIN_COS)) + return listOf(here.lng - dLng, here.lat + dLat, here.lng + dLng, here.lat - dLat).joinToString(",") { "%.5f".format(Locale.ROOT, it) } + } + + private suspend fun search(query: String, viewbox: String?, bounded: Boolean): List = nominatim.withLock { + val wait = lastAsked + SPACING_MS - System.currentTimeMillis() + if (wait > 0) delay(wait) + val url = StringBuilder(NOMINATIM) + .append("?format=jsonv2&addressdetails=1&extratags=1&limit=").append(MAX_RESULTS) + .append("&accept-language=").append(Locale.getDefault().toLanguageTag()) + .append("&q=").append(URLEncoder.encode(query, "UTF-8")) + if (viewbox != null) url.append("&viewbox=").append(viewbox).append(if (bounded) "&bounded=1" else "") + try { + val results = JSONArray(fetch(url.toString())) + (0 until results.length()).mapNotNull { place(results.getJSONObject(it)) } + } finally { + lastAsked = System.currentTimeMillis() + } + } + + private fun place(o: JSONObject): Place? { + val a = o.optJSONObject("address") ?: JSONObject() + val extra = o.optJSONObject("extratags") ?: JSONObject() + val street = listOf(a.optString("road"), a.optString("house_number")).filter { it.isNotBlank() }.joinToString(" ") + val name = o.optString("name").ifBlank { street }.ifBlank { o.optString("display_name").substringBefore(',') }.ifBlank { return null } + val lat = o.optString("lat").toDoubleOrNull() ?: return null + val lng = o.optString("lon").toDoubleOrNull() ?: return null + return Place( + id = "osm:${o.optString("osm_type")}/${o.optLong("osm_id")}", + name = name, + at = LatLng(lat, lng), + street = street, + city = CITY_KEYS.firstNotNullOfOrNull { k -> a.optString(k).takeIf { it.isNotBlank() } }.orEmpty(), + postcode = a.optString("postcode"), + country = a.optString("country"), + category = o.optString("type").takeIf { o.optString("category") in KIND_KEYS }?.let(::label).orEmpty(), + phone = extra.optString("phone").ifBlank { extra.optString("contact:phone") }, + ) + } + + private suspend fun fetch(url: String): String = withContext(Dispatchers.IO) { + val conn = URL(url).openConnection() as HttpURLConnection + try { + // Nominatim's policy asks for a user agent that names the app. + conn.setRequestProperty("User-Agent", USER_AGENT) + conn.connectTimeout = TIMEOUT_MS + conn.readTimeout = TIMEOUT_MS + if (conn.responseCode != 200) throw IOException("${conn.url.host} answered ${conn.responseCode}") + conn.inputStream.bufferedReader().use { it.readText() } + } finally { + conn.disconnect() + } + } + + /** PlaceSearchResult {1 id, 2 title, 3 Address, 4 GeoCoordinate, 8 category, 13 walking, 14 phone}. */ + private fun encode(p: Place, here: LatLng?): ByteArray { + val address = listOf(1 to p.street, 2 to p.city, 4 to p.country, 5 to p.postcode) + .filter { it.second.isNotBlank() } + .fold(ByteArray(0)) { b, (field, value) -> b + Wire.fieldBytes(field, value.encodeToByteArray()) } + var b = Wire.fieldBytes(1, p.id.encodeToByteArray()) + + Wire.fieldBytes(2, p.name.encodeToByteArray()) + + Wire.fieldBytes(3, address) + + Wire.fieldBytes(4, Wire.fieldDouble(1, p.at.lat) + Wire.fieldDouble(2, p.at.lng)) + if (p.category.isNotEmpty()) b += Wire.fieldBytes(8, p.category.encodeToByteArray()) + if (here != null) { + val metres = haversine(here, p.at) + b += Wire.fieldBytes(13, Wire.fieldVarint(1, metres.roundToLong()) + Wire.fieldVarint(2, (metres / WALK_MPS).roundToLong())) + } + if (p.phone.isNotEmpty()) b += Wire.fieldBytes(14, p.phone.encodeToByteArray()) + return b + } + + private fun error(text: String, code: Long): ByteArray = + Wire.fieldVarint(1, RESULT_ERROR) + Wire.fieldBytes(3, text.encodeToByteArray()) + Wire.fieldVarint(4, code) + + companion object { + const val SVC = 55072 + private const val SOURCE_MAPS = -6 + private const val MT_SEARCH = 0 + private const val MT_SEARCH_REQUEST = 1 + private const val MT_SEARCH_RESPONSE = 2 + private const val MT_MEDIA_REQUEST = 3 + private const val MT_MEDIA_RESPONSE = 4 + private const val RESULT_SUCCESS = 0L + private const val RESULT_ERROR = 1L + private const val ERROR_INTERNAL = 0L + private const val ERROR_NETWORK = 1L + private const val NOMINATIM = "https://nominatim.openstreetmap.org/search" + private const val USER_AGENT = "Visor (https://github.com/macedonga/visor)" + private const val SPACING_MS = 1_100L + private const val DEFAULT_RADIUS_M = 1_500.0 + private const val MIN_RADIUS_M = 300.0 + private const val MAX_RADIUS_M = 20_000.0 + private const val METRES_PER_DEGREE = 111_320.0 + private const val MIN_COS = 0.01 + private const val MAX_RESULTS = 10 + private const val CACHE_MS = 5 * 60_000L + private const val CACHE_SIZE = 10 + private const val TIMEOUT_MS = 12_000 + private const val WALK_MPS = 1.3 + + // The kinds of place whose type is worth showing as the place's category. + private val KIND_KEYS = setOf("amenity", "shop", "tourism", "leisure", "historic") + private val CITY_KEYS = listOf("city", "town", "village", "suburb") + + private fun label(value: String): String = value.replace('_', ' ').replaceFirstChar { it.uppercase() } + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt b/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt new file mode 100644 index 00000000..c6453eea --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/RequestLog.kt @@ -0,0 +1,70 @@ +package com.macedonga.visor.connection + +import java.util.concurrent.atomic.AtomicLong +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update + +/** One request the glasses had the phone make: [outcome] stays null while it runs. */ +data class GlassesRequest( + val id: Long, + val at: Long, + val url: String, + val source: Int, + /** What was asked besides the link: image size and format, compression. */ + val options: String = "", + val outcome: String? = null, + val failed: Boolean = false, + val bytes: Long = 0, + val millis: Long = 0, + /** How many times in a row the same request ended the same way. */ + val times: Int = 1, +) + +/** The requests the glasses made through the phone, newest last, in memory only: their links can carry tokens. */ +@Singleton +class RequestLog @Inject constructor() { + private val ids = AtomicLong() + private val _items = MutableStateFlow>(emptyList()) + val items: StateFlow> = _items.asStateFlow() + + fun start(url: String, source: Int, options: String = ""): Long { + val id = ids.incrementAndGet() + _items.update { (it + GlassesRequest(id, System.currentTimeMillis(), url, source, options)).takeLast(MAX) } + return id + } + + fun finish(id: Long, outcome: String, failed: Boolean, bytes: Long = 0) { + _items.update { list -> + val done = list.map { if (it.id == id) it.copy(outcome = outcome, failed = failed, bytes = bytes, millis = System.currentTimeMillis() - it.at) else it } + // The same request ending the same way as the one before it is counted on that one, not listed again. + val at = done.indexOfFirst { it.id == id } + val earlier = done.subList(0, maxOf(at, 0)).lastOrNull { it.url == done[at].url } + if (at > 0 && earlier != null && earlier.outcome == outcome && earlier.failed == failed) { + done.mapNotNull { e -> + when (e.id) { + id -> null + earlier.id -> e.copy(times = e.times + 1, at = done[at].at) + else -> e + } + } + } else done + } + } + + /** A better name for what an entry reached, such as the server a TLS connection asked for. */ + fun rename(id: Long, url: String) { + _items.update { list -> list.map { if (it.id == id) it.copy(url = url) else it } } + } + + fun clear() { + _items.value = emptyList() + } + + private companion object { + const val MAX = 500 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/Routing.kt b/app/src/main/kotlin/com/macedonga/visor/connection/Routing.kt new file mode 100644 index 00000000..6c0970ae --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/Routing.kt @@ -0,0 +1,227 @@ +package com.macedonga.visor.connection + +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import javax.inject.Inject +import javax.inject.Singleton +import kotlin.math.asin +import kotlin.math.cos +import kotlin.math.hypot +import kotlin.math.roundToInt +import kotlin.math.sin +import kotlin.math.sqrt +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.json.JSONObject + +data class LatLng(val lat: Double, val lng: Double) + +/** The Display's maneuver directions (the phone's D42 enum). */ +object Dir { + const val DEPART = 0 + const val ARRIVE = 1 + const val SHARP_LEFT = 2 + const val LEFT = 3 + const val SLIGHT_LEFT = 4 + const val CONTINUE = 5 + const val SLIGHT_RIGHT = 6 + const val RIGHT = 7 + const val SHARP_RIGHT = 8 + const val LEFT_FORK = 9 + const val MIDDLE_FORK = 10 + const val RIGHT_FORK = 11 + const val U_TURN = 12 + const val ARRIVE_LEFT = 13 + const val ARRIVE_RIGHT = 14 +} + +/** One maneuver. [metres] and [seconds] are the leg that leads into it; [along] is where it starts, in metres. */ +class Step(val direction: Int, val street: String, val metres: Int, val seconds: Int, val along: Double, val text: String) + +/** A snapped point on the line: where it is, the index of its segment, and metres from the start. */ +class Snap(val at: LatLng, val index: Int, val along: Double, val off: Double) + +/** A route from OSRM. [encoded] is the polyline as OSRM gives it (precision 5), passed on unchanged. */ +class Route(val encoded: String, val line: List, val steps: List, val metres: Double, val seconds: Double) { + /** Metres from the start to each point of [line], scaled to the route total OSRM gives. */ + private val along = DoubleArray(line.size).also { a -> + for (i in 1 until line.size) a[i] = a[i - 1] + haversine(line[i - 1], line[i]) + val total = a.lastOrNull() ?: 0.0 + if (total > 0) for (i in a.indices) a[i] *= metres / total + } + + val secondsPerMetre: Double get() = if (metres > 0) seconds / metres else 0.0 + + /** The point of the line nearest [p], on a flat map around [p], which is exact enough at route scale. */ + fun snap(p: LatLng): Snap { + val kx = cos(Math.toRadians(p.lat)) * METRES_PER_DEGREE + var best = Snap(line[0], 0, 0.0, Double.MAX_VALUE) + for (i in 0 until line.size - 1) { + val a = line[i] + val b = line[i + 1] + val ax = (a.lng - p.lng) * kx + val ay = (a.lat - p.lat) * METRES_PER_DEGREE + val dx = (b.lng - p.lng) * kx - ax + val dy = (b.lat - p.lat) * METRES_PER_DEGREE - ay + val len2 = dx * dx + dy * dy + val t = if (len2 == 0.0) 0.0 else (-(ax * dx + ay * dy) / len2).coerceIn(0.0, 1.0) + val off = hypot(ax + t * dx, ay + t * dy) + if (off < best.off) { + best = Snap( + LatLng(a.lat + t * (b.lat - a.lat), a.lng + t * (b.lng - a.lng)), + i, + along[i] + t * (along[i + 1] - along[i]), + off, + ) + } + } + return best + } +} + +enum class Mode(val path: String, val profile: String) { + WALK("foot", "foot"), + BIKE("bike", "bike"), + CAR("car", "driving"), +} + +/** Routes from the OpenStreetMap community's public OSRM server. */ +@Singleton +class OsrmRouter @Inject constructor() { + /** Throws when the network fails or the server finds no route. */ + suspend fun route(from: LatLng, to: LatLng, mode: Mode = Mode.WALK): Route = withContext(Dispatchers.IO) { + val url = URL( + "https://routing.openstreetmap.de/routed-${mode.path}/route/v1/${mode.profile}/" + + "${from.lng},${from.lat};${to.lng},${to.lat}?steps=true&overview=full&geometries=polyline", + ) + val conn = url.openConnection() as HttpURLConnection + try { + conn.setRequestProperty("User-Agent", "Visor") + conn.connectTimeout = TIMEOUT_MS + conn.readTimeout = TIMEOUT_MS + if (conn.responseCode != 200) throw IOException("routing server answered ${conn.responseCode}") + parse(conn.inputStream.bufferedReader().use { it.readText() }) + } finally { + conn.disconnect() + } + } + + private fun parse(body: String): Route { + val json = JSONObject(body) + if (json.optString("code") != "Ok") throw IOException(json.optString("code", "no route")) + val r = json.getJSONArray("routes").getJSONObject(0) + val steps = r.getJSONArray("legs").getJSONObject(0).getJSONArray("steps") + val out = ArrayList() + var along = 0.0 + var legMetres = 0.0 + var legSeconds = 0.0 + for (i in 0 until steps.length()) { + val s = steps.getJSONObject(i) + val m = s.getJSONObject("maneuver") + val name = s.optString("name") + val dir = direction(m.optString("type"), m.optString("modifier")) + out += Step(dir, name, legMetres.roundToInt(), legSeconds.roundToInt(), along, words(dir, name)) + legMetres = s.getDouble("distance") + legSeconds = s.getDouble("duration") + along += legMetres + } + val encoded = r.getString("geometry") + return Route(encoded, decodePolyline(encoded), out, r.getDouble("distance"), r.getDouble("duration")) + } + + private fun direction(type: String, modifier: String): Int { + val side = when (modifier) { + "sharp left" -> Dir.SHARP_LEFT + "left" -> Dir.LEFT + "slight left" -> Dir.SLIGHT_LEFT + "slight right" -> Dir.SLIGHT_RIGHT + "right" -> Dir.RIGHT + "sharp right" -> Dir.SHARP_RIGHT + "uturn" -> Dir.U_TURN + else -> Dir.CONTINUE + } + val leftish = modifier.contains("left") + val rightish = modifier.contains("right") + return when (type) { + "depart" -> Dir.DEPART + "arrive" -> when { + leftish -> Dir.ARRIVE_LEFT + rightish -> Dir.ARRIVE_RIGHT + else -> Dir.ARRIVE + } + "fork", "end of road" -> when { + leftish -> Dir.LEFT_FORK + rightish -> Dir.RIGHT_FORK + else -> if (type == "fork") Dir.MIDDLE_FORK else side + } + "merge", "roundabout", "rotary", "roundabout turn", "exit roundabout", "exit rotary" -> Dir.MIDDLE_FORK + "on ramp", "off ramp" -> when { + leftish -> Dir.LEFT_FORK + rightish -> Dir.RIGHT_FORK + else -> Dir.CONTINUE + } + else -> side + } + } + + private fun words(dir: Int, street: String): String { + val onto = if (street.isBlank()) "" else " onto $street" + return when (dir) { + Dir.DEPART -> if (street.isBlank()) "Start" else "Start on $street" + Dir.ARRIVE -> "You have arrived" + Dir.ARRIVE_LEFT -> "You have arrived, on the left" + Dir.ARRIVE_RIGHT -> "You have arrived, on the right" + Dir.SHARP_LEFT -> "Turn sharp left$onto" + Dir.LEFT -> "Turn left$onto" + Dir.SLIGHT_LEFT -> "Bear left$onto" + Dir.SLIGHT_RIGHT -> "Bear right$onto" + Dir.RIGHT -> "Turn right$onto" + Dir.SHARP_RIGHT -> "Turn sharp right$onto" + Dir.U_TURN -> "Make a U-turn" + Dir.LEFT_FORK -> "Keep left$onto" + Dir.RIGHT_FORK -> "Keep right$onto" + Dir.MIDDLE_FORK -> "Go through the junction$onto" + else -> if (street.isBlank()) "Continue straight" else "Continue on $street" + } + } + + private companion object { + const val TIMEOUT_MS = 15_000 + } +} + +/** Google's encoded polyline, precision 5. */ +fun decodePolyline(s: String): List { + val out = ArrayList() + var i = 0 + var lat = 0 + var lng = 0 + fun value(): Int { + var result = 0 + var shift = 0 + var b: Int + do { + b = s[i++].code - 63 + result = result or ((b and 0x1f) shl shift) + shift += 5 + } while (b >= 0x20) + return if ((result and 1) != 0) (result shr 1).inv() else result shr 1 + } + while (i < s.length) { + lat += value() + lng += value() + out += LatLng(lat / 1e5, lng / 1e5) + } + return out +} + +fun haversine(a: LatLng, b: LatLng): Double { + val dLat = Math.toRadians(b.lat - a.lat) + val dLng = Math.toRadians(b.lng - a.lng) + val h = sin(dLat / 2) * sin(dLat / 2) + cos(Math.toRadians(a.lat)) * cos(Math.toRadians(b.lat)) * sin(dLng / 2) * sin(dLng / 2) + return 2 * EARTH_RADIUS_M * asin(sqrt(h)) +} + +private const val EARTH_RADIUS_M = 6_371_000.0 +private const val METRES_PER_DEGREE = 111_320.0 diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt b/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt new file mode 100644 index 00000000..555e19ae --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/WebApps.kt @@ -0,0 +1,262 @@ +package com.macedonga.visor.connection + +import android.net.Uri +import android.util.Base64 +import android.webkit.CookieManager +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.core.settings.GlassesSettings +import com.macedonga.visor.core.settings.SettingsState +import com.macedonga.visor.core.settings.ProtoField +import com.macedonga.visor.core.settings.encodeProto +import com.macedonga.visor.core.settings.parseProto +import com.macedonga.visor.log.EventLog +import java.util.UUID +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.withTimeoutOrNull + +/** A web app on the glasses: its partner_app_settings key, title, page and icon. */ +data class WebApp(val key: String, val name: String, val url: String, val icon: String = "") + +enum class AddResult { ADDED, EXISTS, NOT_HTTPS, FULL, NOT_CONNECTED, UNREADABLE } + +/** + * svc55074, the browser's phone service: the Display's web-app manager (type 15: CREATE 1, DELETE 2, LIST 3, UPDATE 4, + * OPEN 5, DEV MODE STATUS 6, answered with 16), cookies (11, answered with 9) and a BLE scan (17, answered with 20 and + * no devices). The list is the glasses' own partner_app_settings, one dev_mode_web_app_ entry each, so the web apps + * added with the Meta app are in it; Visor changes only the entry it adds, removes or renames. [open] sends OPEN. + */ +@Singleton +class WebApps @Inject constructor( + private val connection: Provider, + private val settings: Provider, + private val log: Provider, +) : Responder { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + val apps: StateFlow> by lazy { + settings.get().state.map { appsIn(inForce(it)) }.stateIn(scope, SharingStarted.Eagerly, appsIn(inForce(settings.get().state.value))) + } + private val lock = Any() + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + if (message.payload.isEmpty() && message.info.flags.isClosed) return + val reply = DataXConnection.mirror(message.info.channelId) + val f = runCatching { Wire.parse(message.payload) }.getOrNull() ?: return + when (message.info.messageType) { + MT_MANAGE -> connection.send(reply, manage(f), messageType = MT_MANAGE_RESPONSE) + MT_GET_COOKIES -> connection.send(reply, cookies(f), messageType = MT_SET_COOKIES) + // Visor scans for nothing: the page gets an answer with no devices. + MT_BLE_SCAN_START -> connection.send(reply, Wire.fieldBytes(1, f.bytes(1)), messageType = MT_BLE_SCAN_RESPONSE) + else -> log.get().add("Web apps: type ${message.info.messageType} not served") + } + } + + /** Add a page, from the Display's CREATE or the Add sheet, with Meta's [manifest] (base64) for a catalogue app. */ + fun add(name: String, url: String, manifest: String? = null): AddResult { + val uri = Uri.parse(url) + if (uri.scheme != "https" || uri.host.isNullOrEmpty()) return AddResult.NOT_HTTPS + synchronized(lock) { + val fields = current() ?: return if (ready()) AddResult.UNREADABLE else AddResult.NOT_CONNECTED + val list = apps(fields) + if (list.any { it.url == url }) return AddResult.EXISTS + if (list.size >= MAX_APPS) return AddResult.FULL + val keys = fields.mapNotNull(::entryKey).toSet() + val key = generateSequence(0) { it + 1 }.map { "$PREFIX$it" }.first { it !in keys } + // DisplayConfig {1 item {1 title}, 2 url, 3 signedWebAppManifest}: the manifest goes in as the catalogue sent it. + val config = listOfNotNull( + message(1, listOf(text(1, name.ifEmpty { uri.host.orEmpty() }))), + text(2, url), + manifest?.let { runCatching { Base64.decode(it, Base64.DEFAULT) }.getOrNull() }?.let { ProtoField(3, 2, bytes = it) }, + ) + if (!write(fields + message(DISPLAY_CONFIGS, listOf(text(1, key), message(2, config))))) return AddResult.NOT_CONNECTED + } + log.get().add("Web apps: added ${uri.host}") + return AddResult.ADDED + } + + fun remove(key: String): Boolean = synchronized(lock) { + val fields = current() ?: return false + val rest = fields.filterNot { it.number == DISPLAY_CONFIGS && entryKey(it) == key } + rest.size != fields.size && write(rest) + } + + private fun rename(key: String, name: String): Boolean = synchronized(lock) { + val fields = current() ?: return false + if (name.isEmpty() || fields.none { entryKey(it) == key }) return false + write(fields.map { f -> if (entryKey(f) == key) renamed(f, name) else f }) + } + + private fun ready() = connection.get().state.value.link == LinkState.READY && settings.get().state.value.readAt != null + + /** The setting as it is now, read into fields; null when it cannot be changed safely (not connected, unreadable). */ + private fun current(): List? { + if (!ready()) return null + val value = inForce(settings.get().state.value) + if (value.isEmpty()) return emptyList() + return runCatching { Base64.decode(value, Base64.NO_WRAP) }.getOrNull()?.let(::parseProto) + } + + /** The value written last if the glasses have not confirmed it yet, else what they hold. */ + private fun inForce(state: SettingsState): String = state.pending[SETTING] ?: state.stored[SETTING].orEmpty() + + private fun write(fields: List): Boolean { + val s = settings.get() + val sent = s.write(SETTING, Base64.encodeToString(encodeProto(fields), Base64.NO_WRAP)) && + (s.value(DEVELOPER_MODE) == "true" || s.write(DEVELOPER_MODE, "true")) + if (!sent) log.get().add("Web apps: the list did not go to the glasses") + return sent + } + + /** Open [app] on the glasses: a lease on the browser's service (svc56, feature 19), then OPEN on svc55074. */ + suspend fun open(app: WebApp): Boolean { + val glasses = connection.get() + if (glasses.state.value.link != LinkState.READY) return false + coroutineScope { + val bound = async(start = CoroutineStart.UNDISPATCHED) { + withTimeoutOrNull(BOUND_MS) { + glasses.serviceData.first { + it.serviceId == SVC_LEASE && it.messageType == MT_LIFECYCLE && + runCatching { Wire.parse(it.payload).varint(2) }.getOrNull() == STATE_BOUND + } + } + } + glasses.submit { it.sendOnService(SVC_LEASE, MT_LEASE, Wire.fieldVarint(1, FEATURE_WEB_APP)) } + if (bound.await() == null) log.get().add("Web apps: the browser's service did not report bound; sending OPEN anyway") + } + delay(REGISTER_MS) + val request = Wire.fieldBytes(1, UUID.randomUUID().toString().encodeToByteArray()) + + Wire.fieldVarint(2, ACTION_OPEN) + + Wire.fieldBytes(3, app.name.encodeToByteArray()) + + Wire.fieldBytes(4, app.url.encodeToByteArray()) + glasses.submit { mux -> mux.send(mux.openChannel(SVC), request, messageType = MT_MANAGE, serviceId = SVC) } + log.get().add("Web apps: OPEN sent for ${app.name}") + return true + } + + private fun manage(f: Wire.Fields): ByteArray { + val id = f.bytes(1) + fun answer(error: String? = null, apps: List = emptyList()): ByteArray = + Wire.fieldBytes(1, id) + + Wire.fieldVarint(2, if (error == null) 1L else 0L) + + (error?.let { Wire.fieldBytes(3, it.encodeToByteArray()) } ?: ByteArray(0)) + + apps.fold(ByteArray(0)) { out, app -> out + Wire.fieldBytes(4, info(app)) } + val key = f.bytes(5).decodeToString() + val name = f.bytes(3).decodeToString() + return when (f.varint(2)) { + ACTION_CREATE -> when (add(name, f.bytes(4).decodeToString())) { + AddResult.ADDED, AddResult.EXISTS -> answer() + AddResult.NOT_HTTPS -> answer("Only https pages can be web apps") + AddResult.FULL -> answer("Up to $MAX_APPS web apps") + AddResult.NOT_CONNECTED, AddResult.UNREADABLE -> answer("The web apps could not be read") + } + ACTION_DELETE -> if (remove(key)) answer() else answer("No such web app") + ACTION_LIST -> answer(apps = apps.value) + ACTION_UPDATE -> if (rename(key, name)) answer() else answer("No such web app") + // The glasses open pages through the phone, not the other way round. + ACTION_OPEN -> answer("Web apps open from the phone") + // Developer mode goes on with the first web app Visor adds. + ACTION_DEV_MODE_STATUS -> answer() + else -> answer("Unknown action") + } + } + + /** Get cookies: the phone's own jar for the page's address, as cookies for its host. */ + private fun cookies(f: Wire.Fields): ByteArray { + val url = f.bytes(2).decodeToString() + val host = Uri.parse(url).host.orEmpty() + val jar = runCatching { CookieManager.getInstance().getCookie(url) }.getOrNull().orEmpty() + val cookies = jar.split(";").map { it.trim().split("=", limit = 2) }.filter { it.size == 2 }.fold(ByteArray(0)) { out, (name, value) -> + out + Wire.fieldBytes( + 3, + Wire.fieldBytes(1, name.encodeToByteArray()) + Wire.fieldBytes(2, value.encodeToByteArray()) + + Wire.fieldBytes(3, host.encodeToByteArray()) + Wire.fieldBytes(4, PATH_ROOT), + ) + } + return Wire.fieldBytes(1, f.bytes(1)) + Wire.fieldBytes(2, url.encodeToByteArray()) + cookies + } + + /** WebappAppInfo {1 partnerKey, 2 appName, 3 appUrl, 4 iconUrl}. */ + private fun info(app: WebApp): ByteArray = + Wire.fieldBytes(1, app.key.encodeToByteArray()) + Wire.fieldBytes(2, app.name.encodeToByteArray()) + + Wire.fieldBytes(3, app.url.encodeToByteArray()) + Wire.fieldBytes(4, app.icon.encodeToByteArray()) + + /** PartnerAppSettings {3 displayConfigs: map entry {1 key, 2 DisplayConfig {1 Item {1 title, 2 icon}, 2 url}}}. */ + private fun appsIn(value: String): List = + value.takeIf { it.isNotEmpty() }?.let { runCatching { Base64.decode(it, Base64.NO_WRAP) }.getOrNull() }?.let(::parseProto)?.let(::apps).orEmpty() + + private fun apps(fields: List): List = fields.filter { it.number == DISPLAY_CONFIGS }.mapNotNull { entry -> + val e = parseProto(entry.bytes) ?: return@mapNotNull null + val key = e.firstOrNull { it.number == 1 }?.bytes?.decodeToString() ?: return@mapNotNull null + val config = e.firstOrNull { it.number == 2 }?.bytes?.let(::parseProto) ?: return@mapNotNull null + val url = config.firstOrNull { it.number == 2 }?.bytes?.decodeToString()?.takeIf { it.startsWith("http") } ?: return@mapNotNull null + val item = config.firstOrNull { it.number == 1 }?.bytes?.let(::parseProto).orEmpty() + val name = item.firstOrNull { it.number == 1 }?.bytes?.decodeToString().orEmpty() + WebApp(key, name.ifEmpty { Uri.parse(url).host.orEmpty() }, url, item.firstOrNull { it.number == 2 }?.bytes?.decodeToString().orEmpty()) + } + + /** [entry] with its item's title set to [name]; every other field kept as it was. */ + private fun renamed(entry: ProtoField, name: String): ProtoField { + val e = parseProto(entry.bytes) ?: return entry + return message(DISPLAY_CONFIGS, e.map { f -> + val config = if (f.number == 2) parseProto(f.bytes) else null + if (config == null) f else message(2, config.map { c -> + val item = if (c.number == 1) parseProto(c.bytes) else null + if (item == null) c else message(1, listOf(text(1, name)) + item.filterNot { it.number == 1 }) + }) + }) + } + + private fun entryKey(entry: ProtoField): String? = + entry.takeIf { it.number == DISPLAY_CONFIGS }?.let { parseProto(it.bytes) }?.firstOrNull { it.number == 1 }?.bytes?.decodeToString() + + private fun text(number: Int, value: String) = ProtoField(number, 2, bytes = value.encodeToByteArray()) + + private fun message(number: Int, fields: List) = ProtoField(number, 2, bytes = encodeProto(fields)) + + companion object { + const val SVC = 55074 + private const val SVC_LEASE = 56 + private const val MT_LEASE = 1 + private const val MT_LIFECYCLE = 3 + private const val STATE_BOUND = 1L + private const val FEATURE_WEB_APP = 19L + private const val MT_MANAGE = 15 + private const val MT_MANAGE_RESPONSE = 16 + private const val MT_GET_COOKIES = 11 + private const val MT_SET_COOKIES = 9 + private const val MT_BLE_SCAN_START = 17 + private const val MT_BLE_SCAN_RESPONSE = 20 + private const val ACTION_CREATE = 1L + private const val ACTION_DELETE = 2L + private const val ACTION_LIST = 3L + private const val ACTION_UPDATE = 4L + private const val ACTION_OPEN = 5L + private const val ACTION_DEV_MODE_STATUS = 6L + private const val PREFIX = "dev_mode_web_app_" + private const val SETTING = "partner_app_settings" + private const val DEVELOPER_MODE = "developer_mode" + private const val DISPLAY_CONFIGS = 3 + private const val MAX_APPS = 20 + private const val BOUND_MS = 3_000L + private const val REGISTER_MS = 1_500L + private val PATH_ROOT = "/".encodeToByteArray() + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/connection/WifiLink.kt b/app/src/main/kotlin/com/macedonga/visor/connection/WifiLink.kt new file mode 100644 index 00000000..d729ddd2 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/connection/WifiLink.kt @@ -0,0 +1,185 @@ +package com.macedonga.visor.connection + +import android.content.Context +import android.net.wifi.WifiManager +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.connection.GlassesLink +import com.macedonga.visor.core.connection.GlassesTarget +import com.macedonga.visor.core.connection.GlassesWifi +import com.macedonga.visor.core.media.BAND_2GHZ +import com.macedonga.visor.core.media.BW_20MHZ +import com.macedonga.visor.core.media.BW_UNSPECIFIED +import com.macedonga.visor.core.media.JoinResult +import com.macedonga.visor.core.media.LiveStream +import com.macedonga.visor.core.media.MT_JOIN_ACCESS_POINT_RESULT +import com.macedonga.visor.core.media.MediaImport +import com.macedonga.visor.core.media.PhoneGroup +import com.macedonga.visor.core.media.SVC_ATC +import com.macedonga.visor.core.media.WifiUnavailable +import com.macedonga.visor.core.media.joinAccessPoint +import com.macedonga.visor.core.media.parseJoinResult +import com.macedonga.visor.core.media.wifiDirectAllowed +import com.macedonga.visor.core.media.wifiIsOn +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.onSubscription +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withTimeoutOrNull + +/** + * The Wi-Fi link the Display asks for with a link lease (svc41107), for WhatsApp, Instagram and other heavy traffic. + * As the Meta app's link manager does, the one session moves to Wi-Fi while a lease holds it and comes back after: on + * the network the phone is on when the glasses have it saved (they stay on it, with their own internet), else on a + * Wi-Fi Direct group of the phone's. Back on Bluetooth when no lease has held Wi-Fi for [IDLE_MS], or when the Wi-Fi + * session fails. + */ +@Singleton +class WifiLink @Inject constructor( + @ApplicationContext private val context: Context, + private val connection: GlassesConnection, + private val networks: GlassesWifi, + private val live: Provider, + private val media: Provider, + private val log: Provider, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default + CoroutineExceptionHandler { _, e -> say("failed: $e") }) + private val lock = Mutex() + private val leases = ConcurrentHashMap.newKeySet() + private var group: PhoneGroup? = null + private var idle: Job? = null + + /** Why the phone cannot give a Wi-Fi link now, or null when it can. */ + fun refusal(): String? = when { + !wifiIsOn(context) -> "the phone's Wi-Fi is off" + connection.state.value.link != LinkState.READY -> "the glasses are not connected" + live.get().running.value -> "the Live view is using Wi-Fi" + media.get().importing.value -> "a media import is using Wi-Fi" + else -> null + } + + /** A granted lease that needs Wi-Fi: the session moves there unless it is already. */ + fun hold(id: String) { + leases += id + idle?.cancel() + scope.launch { lock.withLock { if (!connection.onWifi && leases.isNotEmpty()) moveToWifi() } } + } + + /** A lease ended: with none left, back to Bluetooth after [IDLE_MS]. */ + fun release(id: String) { + if (!leases.remove(id) || leases.isNotEmpty()) return + idle?.cancel() + idle = scope.launch { + delay(IDLE_MS) + lock.withLock { if (leases.isEmpty() && connection.onWifi) back("no lease needs Wi-Fi") } + } + } + + private suspend fun moveToWifi() { + val serial = connection.state.value.target?.serial ?: return + val joined = sameNetwork() ?: phoneGroup() ?: return say("staying on Bluetooth") + say("moving the session to ${joined.address}:${joined.port}") + connection.moveToWifi(GlassesTarget(GlassesLink.Wifi(checkNotNull(joined.address), joined.port), serial)) + scope.launch { watch() } + } + + /** Back on Bluetooth if the session does not come up on Wi-Fi, or when it drops there. */ + private suspend fun watch() { + val up = withTimeoutOrNull(UP_MS) { connection.state.first { it.target?.link is GlassesLink.Wifi && it.link == LinkState.READY } } + if (up == null) return lock.withLock { if (connection.onWifi) back("the session did not come up over Wi-Fi") } + say("the session runs over Wi-Fi") + connection.state.first { it.link != LinkState.READY || !connection.onWifi } + lock.withLock { + when { + connection.onWifi -> back("the Wi-Fi session dropped") + // The Live or a media import took the glasses for a Wi-Fi session of their own, and the group with it. + connection.paused -> forget() + // The session was opened again on Bluetooth some other way. + else -> { + group?.remove() + forget() + } + } + } + } + + private fun forget() { + group = null + leases.clear() + } + + private suspend fun back(why: String) { + say("back to Bluetooth: $why") + leases.clear() + connection.backToBluetooth() + group?.remove() + group = null + } + + /** The glasses joined to the network the phone is on, when they keep its passphrase. */ + private suspend fun sameNetwork(): JoinResult? { + val ssid = phoneSsid() ?: return null + val key = networks.keyOf(ssid) ?: return null + return join(ssid, key, BW_UNSPECIFIED)?.also { say("the glasses are on $ssid with the phone") } + } + + /** The glasses joined to a Wi-Fi Direct group of the phone's. */ + private suspend fun phoneGroup(): JoinResult? { + if (!wifiDirectAllowed(context)) return null.also { say("Wi-Fi Direct is not allowed") } + val g = PhoneGroup(context) { say(it) } + try { + g.create(BAND_2GHZ) + } catch (unavailable: WifiUnavailable) { + return null.also { say("Wi-Fi Direct did not come up: ${unavailable.message}") } + } + group = g + return join(g.ssid, g.passphrase, BW_20MHZ) ?: null.also { + g.remove() + group = null + } + } + + // Fresh out of a group, or of a Wi-Fi restart, the glasses answer INTERNAL_ERROR for a while. + private suspend fun join(ssid: String, passphrase: String, bandwidth: Int): JoinResult? { + for (attempt in 1..JOIN_TRIES) { + if (attempt > 1) delay(RETRY_MS) + val result = withTimeoutOrNull(JOIN_MS) { + connection.serviceData + .onSubscription { connection.submit { joinAccessPoint(it, ssid, passphrase, bandwidth) } } + .first { it.serviceId == SVC_ATC && it.messageType == MT_JOIN_ACCESS_POINT_RESULT } + .let { parseJoinResult(it.payload) } + } + if (result?.ok == true) return result + say(if (result == null) "the glasses did not answer the join to $ssid" else "the glasses could not join $ssid (${result.statusName})") + } + return null + } + + @Suppress("DEPRECATION") + private fun phoneSsid(): String? = + context.getSystemService(WifiManager::class.java)?.connectionInfo?.ssid?.removeSurrounding("\"")?.takeIf { it.isNotEmpty() && it != UNKNOWN_SSID } + + private fun say(text: String) = log.get().add("Wi-Fi link: $text") + + private companion object { + const val IDLE_MS = 30_000L + const val UP_MS = 20_000L + const val JOIN_MS = 30_000L + const val RETRY_MS = 4_000L + const val JOIN_TRIES = 3 + const val UNKNOWN_SSID = "" + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt index 8926a5c0..f6445525 100644 --- a/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt +++ b/app/src/main/kotlin/com/macedonga/visor/di/ConnectionModule.kt @@ -23,6 +23,7 @@ import com.macedonga.visor.core.assistant.Voice import com.macedonga.visor.core.assistant.mcp.McpHub import com.macedonga.visor.core.assistant.mcp.McpStore import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.connection.GlassesWifi import com.macedonga.visor.core.connection.RealSessionHostFactory import com.macedonga.visor.core.connection.SessionHostFactory import com.macedonga.visor.core.connection.Shutter @@ -32,6 +33,7 @@ import com.macedonga.visor.core.identity.KeystoreSealer import com.macedonga.visor.core.media.AirshipReceiver import com.macedonga.visor.core.media.GlassesConfig import com.macedonga.visor.core.media.MediaImport +import com.macedonga.visor.core.media.Screencast import com.macedonga.visor.core.media.LiveStream import com.macedonga.visor.core.media.LiveOutputs import com.macedonga.visor.core.media.LiveSettingsStore @@ -45,11 +47,14 @@ import com.macedonga.visor.core.settings.RlDriveResponder import com.macedonga.visor.core.settings.SVC_RLDRIVE import com.macedonga.visor.core.transport.FileWireLog import com.macedonga.visor.core.transport.LinkMeter +import com.macedonga.visor.notify.GlassesNotification +import com.macedonga.visor.notify.NotifyIcons import com.macedonga.visor.phone.PhoneActions import dagger.Module import dagger.Provides import dagger.hilt.InstallIn import dagger.hilt.android.qualifiers.ApplicationContext +import com.macedonga.visor.core.protocol.ServiceId import dagger.hilt.components.SingletonComponent import java.io.File import javax.inject.Singleton @@ -135,6 +140,25 @@ object ConnectionModule { // A Provider, because these need the connection this factory feeds. hearing: javax.inject.Provider, assistant: javax.inject.Provider, + lease: javax.inject.Provider, + icons: NotifyIcons, + internet: com.macedonga.visor.connection.InternetSwitch, + fetcher: com.macedonga.visor.connection.DataFetcher, + tunnel: com.macedonga.visor.connection.PhoneInternet, + panels: com.macedonga.visor.connection.Panels, + bloks: com.macedonga.visor.connection.BloksRelay, + location: com.macedonga.visor.connection.LocationShare, + messages: com.macedonga.visor.connection.PhoneMessages, + sharing: com.macedonga.visor.connection.PhoneSharing, + calls: com.macedonga.visor.connection.PhoneCalls, + media: com.macedonga.visor.connection.PhoneMedia, + captureState: com.macedonga.visor.connection.CaptureState, + leases: com.macedonga.visor.connection.LeaseService, + navigation: com.macedonga.visor.connection.Navigation, + placeSearch: com.macedonga.visor.connection.PlaceSearch, + mapsSupport: com.macedonga.visor.connection.MapsSupport, + mapsQuiet: com.macedonga.visor.connection.MapsQuiet, + webApps: com.macedonga.visor.connection.WebApps, ): SessionHostFactory = RealSessionHostFactory( transports, responders = mapOf( @@ -144,7 +168,29 @@ object ConnectionModule { SVC_ASSISTANT to Responder { message, mux -> assistant.get().respond(message, mux) }, SVC_GLASSES_PHOTO to photos, SVC_TAR to timers, + GlassesNotification.SVC to icons, + com.macedonga.visor.connection.InternetSwitch.SVC to internet, + com.macedonga.visor.connection.InternetSwitch.PING_SVC to internet.ping, + com.macedonga.visor.connection.MotionActivity.SVC to com.macedonga.visor.connection.MotionActivity, + com.macedonga.visor.connection.DataFetcher.SVC to fetcher, + com.macedonga.visor.connection.PhoneInternet.SVC to tunnel, + com.macedonga.visor.connection.Panels.SVC to panels, + com.macedonga.visor.connection.BloksRelay.SVC to bloks, + com.macedonga.visor.connection.LocationShare.SVC to location, + com.macedonga.visor.connection.PhoneMessages.SVC to messages, + com.macedonga.visor.connection.PhoneSharing.SVC to sharing, + com.macedonga.visor.connection.PhoneCalls.SVC to calls, + com.macedonga.visor.connection.PhoneMedia.SVC to media, + com.macedonga.visor.connection.CaptureState.SVC to captureState, + com.macedonga.visor.connection.LeaseService.SVC to leases, + com.macedonga.visor.connection.Navigation.SVC to navigation, + com.macedonga.visor.connection.PlaceSearch.SVC to placeSearch, + com.macedonga.visor.connection.MapsSupport.SVC to mapsSupport, + com.macedonga.visor.connection.MapsQuiet.SVC_PERSISTENT_STORE to mapsQuiet.responder(com.macedonga.visor.connection.MapsQuiet.SVC_PERSISTENT_STORE), + com.macedonga.visor.connection.MapsQuiet.SVC_CONSTELLATION to mapsQuiet.responder(com.macedonga.visor.connection.MapsQuiet.SVC_CONSTELLATION), + com.macedonga.visor.connection.WebApps.SVC to webApps, SVC_HEARING to Responder { message, mux -> hearing.get().respond(message, mux) }, + ServiceId.CONN_DEVICE_LEASE.value to Responder { message, mux -> lease.get().respond(message, mux) }, ), ) @@ -156,6 +202,16 @@ object ConnectionModule { @Singleton fun hearing(connection: GlassesConnection): Hearing = Hearing(connection) + @Provides + @Singleton + fun teleprompter(connection: GlassesConnection): com.macedonga.visor.core.connection.Teleprompter = + com.macedonga.visor.core.connection.Teleprompter(connection) + + @Provides + @Singleton + fun captions(connection: GlassesConnection): com.macedonga.visor.core.connection.Captions = + com.macedonga.visor.core.connection.Captions(connection) + @Provides @Singleton fun shutter(connection: GlassesConnection): Shutter = Shutter(connection) @@ -209,6 +265,7 @@ object ConnectionModule { hearing, { (glassesConfig.state.value.values[com.macedonga.visor.language.PROCESSING_TIMEOUT] as? Number)?.toLong() }, phone, + { com.macedonga.visor.settings.hasDisplay(connection.state.value.info?.build) }, ) @Provides @@ -259,8 +316,17 @@ object ConnectionModule { media: MediaImport, ): GlassesConfig = GlassesConfig(context, connection, airship, media) + @Provides + @Singleton + fun screencast(connection: GlassesConnection, config: GlassesConfig, live: com.macedonga.visor.core.media.LiveStream): Screencast = + Screencast(connection, config, live) + @Provides @Singleton fun glassesSettings(connection: GlassesConnection, drive: RlDriveResponder): GlassesSettings = GlassesSettings(connection, drive) + + @Provides + @Singleton + fun glassesWifi(connection: GlassesConnection): GlassesWifi = GlassesWifi(connection) } diff --git a/app/src/main/kotlin/com/macedonga/visor/display/CaptionPacks.kt b/app/src/main/kotlin/com/macedonga/visor/display/CaptionPacks.kt new file mode 100644 index 00000000..bf907daf --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/display/CaptionPacks.kt @@ -0,0 +1,224 @@ +package com.macedonga.visor.display + +import android.content.Context +import com.macedonga.visor.core.connection.CaptionsMode +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.language.PackOffer +import com.macedonga.visor.core.media.AirshipReceiver +import com.macedonga.visor.core.media.LiveStream +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.ServiceData +import com.macedonga.visor.language.LanguageInstaller +import com.macedonga.visor.language.downloadPack +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.pairs.pairPrefs +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.File +import java.nio.ByteBuffer +import java.util.UUID +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull + +enum class PackStep { DOWNLOADING, SENDING, INSTALLING } + +sealed interface PackWork { + data object Idle : PackWork + data class Working(val feature: String, val step: PackStep, val done: Long = 0, val total: Long = 0) : PackWork + data class Failed(val feature: String, val reason: String) : PackWork +} + +/** + * The Display's captions and translation run only with Meta's pack for the language, a constellation feature + * (`air.live_captions_en`, `air.translation_en_it`) the glasses' feature manager must have on. Installed as the Meta + * app installs it, through the glasses' OTA service (svc29): StartApp for the OTA app, the install request naming the + * pack, the zip on airship `ota` under the id the request gave it, then a feature change that turns it on. The + * glasses do not say which they have, so this phone remembers what it installed. + */ +@Singleton +class CaptionPacks @Inject constructor( + @ApplicationContext private val context: Context, + private val languages: LanguageInstaller, + private val connection: GlassesConnection, + private val live: LiveStream, + private val airship: AirshipReceiver, + private val log: EventLog, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + @Volatile + private var prefs = pairPrefs(context, "caption_packs", connection.pair.value) + private val dir = File(context.filesDir, "packs") + + private val _installed = MutableStateFlow(prefs.getStringSet(INSTALLED, null).orEmpty()) + val installed: StateFlow> = _installed.asStateFlow() + private val _offers = MutableStateFlow>(emptyMap()) + val offers: StateFlow> = _offers.asStateFlow() + private val _work = MutableStateFlow(PackWork.Idle) + val work: StateFlow = _work.asStateFlow() + private var job: Job? = null + + init { + scope.launch { + connection.pair.filterNotNull().distinctUntilChanged().drop(1).collect { serial -> + prefs = pairPrefs(context, "caption_packs", serial) + _installed.value = prefs.getStringSet(INSTALLED, null).orEmpty() + _offers.value = emptyMap() + } + } + } + + /** Ask Meta about [feature]'s pack, for its size. */ + fun look(feature: String) { + if (feature in _offers.value) return + scope.launch { + runCatching { languages.offersFor(listOf(FEATURE + feature)) }.getOrNull()?.firstOrNull() + ?.let { offer -> _offers.update { it + (feature to offer) } } + } + } + + fun install(feature: String) { + if (job?.isActive == true) return + job = scope.launch { + try { + if (connection.state.value.link != LinkState.READY) error("the glasses are not connected") + _work.value = PackWork.Working(feature, PackStep.DOWNLOADING) + val offer = _offers.value[feature] ?: languages.offersFor(listOf(FEATURE + feature)).firstOrNull() + ?: error("Meta has no pack for these glasses") + dir.mkdirs() + val zip = File(dir, offer.resource + ".zip") + if (!zip.exists()) { + if (dir.usableSpace < offer.size + SPARE_BYTES) error("the phone needs ${(offer.size + SPARE_BYTES) / 1_000_000} MB free") + log.add("Captions: downloading ${offer.resource} ${offer.version}") + downloadPack(offer, zip) { _work.value = PackWork.Working(feature, PackStep.DOWNLOADING, it, offer.size) } + } + if (connection.state.value.link != LinkState.READY) error("the glasses are not connected") + log.add("Captions: installing ${zip.name} on the glasses over Wi-Fi") + val why = live.overWifi { mux -> runCatching { installOn(mux, feature, offer, zip) }.exceptionOrNull()?.let { it.message ?: it::class.java.simpleName } } + if (why != null) error(why) + zip.delete() + _installed.update { it + feature } + prefs.edit().putStringSet(INSTALLED, _installed.value).apply() + log.add("Captions: ${offer.resource} is on the glasses") + _work.value = PackWork.Idle + } catch (error: kotlinx.coroutines.CancellationException) { + _work.value = PackWork.Idle + throw error + } catch (error: Throwable) { + log.add("Captions: $feature failed: ${error.message}") + _work.value = PackWork.Failed(feature, error.message ?: error::class.java.simpleName) + } + } + } + + fun cancel() { + job?.cancel() + } + + private suspend fun installOn(mux: DataXConnection, feature: String, offer: PackOffer, zip: File) = coroutineScope { + val fromGlasses = Channel(Channel.UNLIMITED) + val listening = launch { live.wifiData.collect { if (it.serviceId == SVC_OTA) fromGlasses.send(it) } } + suspend fun next(type: Int, ms: Long): ServiceData? = withTimeoutOrNull(ms) { + var m: ServiceData + do { + m = fromGlasses.receive() + if (m.messageType == MT_ERROR) error("the glasses stopped the install: ${errorWords(m.payload)}") + } while (m.messageType != type) + m + } + try { + val artifact = UUID.randomUUID() + val raw = ByteBuffer.allocate(16).putLong(artifact.mostSignificantBits).putLong(artifact.leastSignificantBits).array() + val session = UUID.randomUUID().toString() + val key = Wire.fieldBytes(1, str(UUID.randomUUID().toString())) + Wire.fieldBytes(2, str(session)) + Wire.fieldBytes(3, str(artifact.toString())) + + // The OTA app first: its workflow waits for the install request once started. + val megabytes = (offer.size / 1_000_000 + 1) * 2 + val start = Wire.fieldVarint(1, APP_OTA) + Wire.fieldBytes(2, Wire.fieldBytes(1, Wire.fieldVarint(1, megabytes))) + + intent("ota_type", Wire.fieldBytes(5, str("resource"))) + intent("attempt_id", Wire.fieldBytes(5, str(artifact.toString()))) + + intent("app_transfer_auto", Wire.fieldVarint(2, 1)) + mux.sendOnService(SVC_OTA, MT_START_APP, start) + val started = next(MT_START_APP_RESPONSE, ANSWER_MS) ?: error("the glasses' OTA service did not answer") + val code = runCatching { Wire.parse(started.payload).varint(1) }.getOrDefault(0) + // Already started: its workflow is up, and still takes the request. + if (code != SNAPP_SUCCESS && code != SNAPP_ALREADY_STARTED) error("the glasses would not start an install (code $code)") + + val pack = Wire.fieldVarint(1, APP_RESOURCE) + Wire.fieldBytes(2, raw) + Wire.fieldBytes(3, str(offer.version)) + + Wire.fieldBytes(4, str(offer.appId)) + Wire.fieldBytes(6, str(offer.feature)) + Wire.fieldBytes(7, str("$artifact.zip")) + val request = Wire.fieldBytes(1, key) + Wire.fieldBytes(2, raw) + Wire.fieldVarint(5, zip.length()) + + Wire.fieldBytes(8, pack) + Wire.fieldVarint(9, 1) + mux.sendOnService(SVC_OTA, MT_INSTALL, request) + // They ask for the file once their receiver is ready for it; earlier it is dropped. + next(MT_FILE_STATUS, ANSWER_MS) ?: error("the glasses did not ask for the file") + + _work.value = PackWork.Working(feature, PackStep.SENDING, 0, zip.length()) + airship.send(mux, NAMESPACE, "$artifact.zip", zip, uuid = raw) { + _work.value = PackWork.Working(feature, PackStep.SENDING, it, zip.length()) + }?.let { error(it) } + + _work.value = PackWork.Working(feature, PackStep.INSTALLING) + next(MT_INSTALL_COMPLETE, INSTALL_MS) ?: error("the glasses did not finish installing it") + + val change = Wire.fieldBytes(1, Wire.fieldBytes(2, str(session))) + + Wire.fieldBytes(2, Wire.fieldVarint(1, APP_RESOURCE) + Wire.fieldBytes(2, str(offer.feature)) + Wire.fieldVarint(3, ENABLE)) + mux.sendOnService(SVC_OTA, MT_FEATURE_CHANGE, change) + val changed = next(MT_FEATURE_CHANGE_RESPONSE, ANSWER_MS) + log.add("Captions: feature change answered " + (changed?.payload?.joinToString("") { "%02x".format(it) } ?: "nothing")) + } finally { + listening.cancel() + } + } + + private fun errorWords(payload: ByteArray): String = runCatching { + val f = Wire.parse(payload) + val text = f.bytes(5).decodeToString() + "code ${f.varint(3)}" + if (text.isNotBlank()) ", $text" else "" + }.getOrDefault(payload.joinToString("") { "%02x".format(it) }) + + private fun str(s: String) = s.encodeToByteArray() + + private fun intent(key: String, value: ByteArray) = Wire.fieldBytes(3, Wire.fieldBytes(1, str(key)) + value) + + companion object { + /** The feature a choice needs: one language for captions; for translation, theirs into the wearer's. */ + fun featureFor(mode: CaptionsMode, self: String, other: String): String = + if (mode == CaptionsMode.CAPTIONS) "live_captions_$self" else "translation_${other}_$self" + + private const val FEATURE = "com.meta.constellation.feature.air." + private const val NAMESPACE = "ota" + private const val INSTALLED = "installed" + private const val SPARE_BYTES = 50_000_000L + private const val SVC_OTA = 29 + private const val MT_INSTALL = 1 + private const val MT_ERROR = 3 + private const val MT_FILE_STATUS = 4 + private const val MT_INSTALL_COMPLETE = 7 + private const val MT_FEATURE_CHANGE = 8 + private const val MT_FEATURE_CHANGE_RESPONSE = 9 + private const val MT_START_APP = 2000 + private const val MT_START_APP_RESPONSE = 2001 + private const val APP_OTA = 1L + // RESOURCE_PACK, as the Meta app numbers the package it installs. + private const val APP_RESOURCE = 1L + private const val ENABLE = 1L + private const val SNAPP_SUCCESS = 1L + private const val SNAPP_ALREADY_STARTED = 12L + private const val ANSWER_MS = 15_000L + private const val INSTALL_MS = 240_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/display/CaptionsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/display/CaptionsScreen.kt new file mode 100644 index 00000000..1dbabaa2 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/display/CaptionsScreen.kt @@ -0,0 +1,309 @@ +package com.macedonga.visor.display + +import android.content.Context +import android.content.Intent +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.WindowInsets +import androidx.compose.foundation.layout.asPaddingValues +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.navigationBars +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.lazy.rememberLazyListState +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.macedonga.visor.R +import com.macedonga.visor.core.connection.Captions +import com.macedonga.visor.core.connection.CaptionLine +import com.macedonga.visor.core.connection.CaptionsMode +import com.macedonga.visor.core.connection.CaptionsPhase +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.gallery.RoundIcon +import com.macedonga.visor.settings.ChoiceSheet +import com.macedonga.visor.settings.PillsRow +import com.macedonga.visor.settings.Rows +import com.macedonga.visor.settings.SectionLabel +import com.macedonga.visor.settings.SettingsPage +import com.macedonga.visor.settings.Spinner +import com.macedonga.visor.settings.ValueRow +import com.macedonga.visor.settings.row +import com.macedonga.visor.ui.VisorIcons +import com.macedonga.visor.ui.rememberHaptics +import dagger.hilt.android.lifecycle.HiltViewModel +import dagger.hilt.android.qualifiers.ApplicationContext +import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter +import java.util.Locale +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch + +/** Captions or translation, and the languages, as last chosen. */ +data class CaptionsChoice(val mode: CaptionsMode = CaptionsMode.CAPTIONS, val self: String = "en", val other: String = "en") { + /** The other side's language: for captions there is one. */ + val heard: String get() = if (mode == CaptionsMode.CAPTIONS) self else other +} + +@Singleton +class CaptionsChoices @Inject constructor(@ApplicationContext context: Context) { + private val prefs = context.getSharedPreferences("captions", Context.MODE_PRIVATE) + private val _choice = MutableStateFlow( + CaptionsChoice( + runCatching { CaptionsMode.valueOf(prefs.getString(MODE, null).orEmpty()) }.getOrDefault(CaptionsMode.CAPTIONS), + prefs.getString(SELF, null) ?: "en", + prefs.getString(OTHER, null) ?: "en", + ), + ) + val choice: StateFlow = _choice.asStateFlow() + + fun choose(choice: CaptionsChoice) { + _choice.value = choice + prefs.edit().putString(MODE, choice.mode.name).putString(SELF, choice.self).putString(OTHER, choice.other).apply() + } + + private companion object { + const val MODE = "mode" + const val SELF = "self" + const val OTHER = "other" + } +} + +@HiltViewModel +class CaptionsModel @Inject constructor( + val choices: CaptionsChoices, + val captions: Captions, + val connection: GlassesConnection, + val packs: CaptionPacks, +) : ViewModel() { + /** Keep [choice]; while the glasses caption, they stop and start again with it. */ + fun choose(choice: CaptionsChoice) { + choices.choose(choice) + if (captions.state.value.phase != CaptionsPhase.LIVE) return + viewModelScope.launch { + captions.stop() + captions.state.first { it.phase == CaptionsPhase.IDLE } + captions.start(choice.mode, choice.self, choice.heard) + } + } +} + +/** Captions and translation on the Display: what and in which languages, then what the glasses hear, turn by turn. */ +@Composable +fun CaptionsScreen(model: CaptionsModel = hiltViewModel()) { + val choice by model.choices.choice.collectAsStateWithLifecycle() + val state by model.captions.state.collectAsStateWithLifecycle() + val glasses by model.connection.state.collectAsStateWithLifecycle() + val ready = glasses.link == LinkState.READY + val feature = CaptionPacks.featureFor(choice.mode, choice.self, choice.heard) + val installed by model.packs.installed.collectAsStateWithLifecycle() + val offers by model.packs.offers.collectAsStateWithLifecycle() + val work by model.packs.work.collectAsStateWithLifecycle() + LaunchedEffect(feature, ready) { if (ready) model.packs.look(feature) } + var picking by remember { mutableStateOf(null) } + val haptics = rememberHaptics() + val context = LocalContext.current + val list = rememberLazyListState() + val transcript = state.earlier + state.lines + val last = state.lines.lastOrNull() + // Follow the newest turn, unless the transcript was scrolled back to read. + LaunchedEffect(state.lines.size, last?.text) { + val info = list.layoutInfo + val end = info.visibleItemsInfo.lastOrNull()?.index ?: return@LaunchedEffect + if (end >= info.totalItemsCount - 3) list.animateScrollToItem(info.totalItemsCount - 1) + } + SettingsPage( + stringResource(R.string.captions_title), + list = list, + actions = { + if (transcript.isNotEmpty()) RoundIcon(VisorIcons.Share, stringResource(R.string.transcript_share), { share(context, transcript) }) + }, + sheet = { + CaptionsBar( + ready = ready, + phase = state.phase, + refusal = state.refusal, + pack = PackView(feature in installed, offers[feature]?.size, work.takeIf { it !is PackWork.Idle }), + onGet = { haptics.done(); model.packs.install(feature) }, + onCancel = { model.packs.cancel() }, + onStart = { haptics.done(); model.captions.start(choice.mode, choice.self, choice.heard) }, + onStop = { haptics.toggle(); model.captions.stop() }, + ) + picking?.let { side -> + ChoiceSheet( + stringResource(if (side == Side.SELF) if (choice.mode == CaptionsMode.CAPTIONS) R.string.captions_language else R.string.captions_self else R.string.captions_other), + languageOptions(), + if (side == Side.SELF) choice.self else choice.other, + { model.choose(if (side == Side.SELF) choice.copy(self = it) else choice.copy(other = it)) }, + { picking = null }, + ) + } + }, + ) { + item(key = "choice") { + Rows( + row { + PillsRow( + stringResource(R.string.captions_mode), + listOf(CaptionsMode.CAPTIONS to stringResource(R.string.captions_mode_captions), CaptionsMode.TRANSLATION to stringResource(R.string.captions_mode_translation)), + choice.mode, { model.choose(choice.copy(mode = it)) }, + ) + }, + row { + ValueRow( + stringResource(if (choice.mode == CaptionsMode.CAPTIONS) R.string.captions_language else R.string.captions_self), + languageName(choice.self), { picking = Side.SELF }, + ) + }, + if (choice.mode == CaptionsMode.TRANSLATION) row { + ValueRow(stringResource(R.string.captions_other), languageName(choice.other), { picking = Side.OTHER }) + } else null, + ) + } + if (state.lines.isNotEmpty() || state.phase == CaptionsPhase.LIVE) { + item(key = "heard") { SectionLabel(stringResource(R.string.captions_heard)) } + } + if (state.lines.isEmpty() && state.phase == CaptionsPhase.LIVE) { + item(key = "listening") { Turn(stringResource(R.string.captions_listening), null, current = false) } + } + items(state.lines, key = { "line-" + it.id }) { line -> Turn(line.text, line.source, current = line === last && state.phase == CaptionsPhase.LIVE) } + item(key = "foot") { Spacer(Modifier.height(BAR_ROOM)) } + } +} + +private enum class Side { SELF, OTHER } + +private val CLOCK = DateTimeFormatter.ofPattern("HH:mm") + +/** Plain text, one turn per line as `HH:mm said`; a translation goes on the line under it. */ +private fun share(context: Context, lines: List) { + val text = lines.joinToString("\n") { line -> + val time = CLOCK.format(Instant.ofEpochMilli(line.time).atZone(ZoneId.systemDefault())) + if (line.source == null) "$time ${line.text}" else "$time ${line.source}\n${line.text}" + } + val send = Intent(Intent.ACTION_SEND).setType("text/plain").putExtra(Intent.EXTRA_TEXT, text) + context.startActivity(Intent.createChooser(send, null)) +} + +/** One turn: the newest one bright while the glasses listen, what was said under a translation. */ +@Composable +private fun Turn(text: String, source: String?, current: Boolean) { + val colors = Visor.colors + Column(Modifier.fillMaxWidth().padding(horizontal = 32.dp, vertical = 8.dp)) { + Text(text, style = Visor.type.body, color = if (current) colors.textPrimary else colors.textSecondary) + if (source != null) Text(source, style = Visor.type.label, color = colors.textTertiary, modifier = Modifier.padding(top = 2.dp)) + } +} + +/** At the foot of the page, over a fade: whether the glasses are captioning, and Start or Stop. */ +@Composable +private fun CaptionsBar( + ready: Boolean, + phase: CaptionsPhase, + refusal: Long?, + pack: PackView, + onGet: () -> Unit, + onCancel: () -> Unit, + onStart: () -> Unit, + onStop: () -> Unit, +) { + val colors = Visor.colors + val bottom = WindowInsets.navigationBars.asPaddingValues().calculateBottomPadding() + Box(Modifier.fillMaxSize(), contentAlignment = Alignment.BottomCenter) { + Column( + Modifier + .fillMaxWidth() + .background(Brush.verticalGradient(0f to Color.Transparent, 0.3f to colors.canvas)) + .padding(start = 16.dp, end = 16.dp, top = 28.dp, bottom = bottom + 16.dp), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.Center, + ) { + val work = pack.work + val line = when { + work is PackWork.Working -> stringResource( + when (work.step) { + PackStep.DOWNLOADING -> R.string.language_downloading + PackStep.SENDING -> R.string.language_sending + PackStep.INSTALLING -> R.string.captions_installing + }, + ) + if (work.total > 0) " · ${megabytes(work.done)} / ${megabytes(work.total)}" else "" + work is PackWork.Failed -> stringResource(R.string.captions_pack_failed, work.reason) + !ready -> stringResource(R.string.captions_not_connected) + refusal != null && phase == CaptionsPhase.IDLE -> refusalWords(refusal) + phase == CaptionsPhase.LIVE -> stringResource(R.string.captions_live) + else -> null + } + if (line != null) { + val bad = work is PackWork.Failed || refusal != null && ready && phase == CaptionsPhase.IDLE + Text(line, style = Visor.type.label, color = if (bad) colors.signal else colors.textSecondary, modifier = Modifier.padding(bottom = 12.dp)) + } + when { + work is PackWork.Working -> WidePill(stringResource(R.string.language_cancel), onCancel, Modifier.fillMaxWidth(), quiet = true) + !ready -> Unit + phase == CaptionsPhase.IDLE && !pack.installed -> Row(horizontalArrangement = Arrangement.spacedBy(10.dp)) { + WidePill(stringResource(R.string.captions_start_anyway), onStart, Modifier.weight(1f), quiet = true) + WidePill( + pack.size?.let { stringResource(R.string.captions_get_size, megabytes(it)) } ?: stringResource(R.string.language_download), + onGet, Modifier.weight(1f), + ) + } + phase == CaptionsPhase.STARTING || phase == CaptionsPhase.STOPPING -> Box(Modifier.height(52.dp), contentAlignment = Alignment.Center) { Spinner() } + phase == CaptionsPhase.LIVE -> WidePill(stringResource(R.string.captions_stop), onStop, Modifier.fillMaxWidth(), quiet = true) + else -> WidePill(stringResource(R.string.captions_start), onStart, Modifier.fillMaxWidth()) + } + } + } +} + +/** Whether this phone sent the choice's pack, how big Meta's is, and a download or send under way. */ +private data class PackView(val installed: Boolean, val size: Long?, val work: PackWork?) + +private fun megabytes(bytes: Long) = "${(bytes + 500_000) / 1_000_000} MB" + +/** The card's line: the language, or the two of a translation. */ +@Composable +fun captionsSummary(choice: CaptionsChoice): String = + if (choice.mode == CaptionsMode.CAPTIONS) languageName(choice.self) + else stringResource(R.string.captions_pair, languageName(choice.other), languageName(choice.self)) + +private fun languageName(code: String): String = + Locale.forLanguageTag(code).let { it.getDisplayLanguage(Locale.getDefault()).replaceFirstChar { c -> c.titlecase(Locale.getDefault()) } } + +private fun languageOptions(): List> = + LANGUAGES.map { Triple(it, languageName(it), null) }.sortedBy { it.second } + +// The Display's own list, from its Captions app; packs beyond English come from Meta. +private val LANGUAGES = listOf( + "en", "es", "it", "fr", "de", "pt", "zh", "ja", "ko", "hi", "te", "bn", "nl", "tl", "gu", "pa", "kn", "ml", "da", + "fi", "sv", "ru", "vi", "tr", "he", "pl", "hu", "cs", "hr", "el", "ms", "ta", "mr", "ar", "id", "th", "ur", +) +private val BAR_ROOM = 120.dp diff --git a/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt new file mode 100644 index 00000000..5f685bc1 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/display/DisplayShelf.kt @@ -0,0 +1,121 @@ +package com.macedonga.visor.display + +import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.lazy.LazyRow +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Icon +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.vector.ImageVector +import androidx.compose.ui.res.pluralStringResource +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.macedonga.visor.R +import com.macedonga.visor.core.connection.PrompterPhase +import com.macedonga.visor.core.connection.Teleprompter +import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.ui.VisorIcons +import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject + +/** What the Display's cards on the home open. */ +enum class DisplayTool { TELEPROMPTER, CAPTIONS, GESTURES, MIRROR } + +@HiltViewModel +class ShelfModel @Inject constructor( + val script: PrompterScript, + val prompter: Teleprompter, + val band: com.macedonga.visor.band.BandLink, + val connection: com.macedonga.visor.core.connection.GlassesConnection, + val captions: com.macedonga.visor.core.connection.Captions, + val captionsChoices: CaptionsChoices, + val screencast: com.macedonga.visor.core.media.Screencast, +) : ViewModel() + +/** The Display's tools on the home: a card each, with what it holds now. */ +@Composable +fun DisplayShelf(onOpen: (DisplayTool) -> Unit, modifier: Modifier = Modifier, model: ShelfModel = hiltViewModel()) { + val text by model.script.text.collectAsStateWithLifecycle() + val prompter by model.prompter.state.collectAsStateWithLifecycle() + val band by model.band.state.collectAsStateWithLifecycle() + val glasses by model.connection.state.collectAsStateWithLifecycle() + val bandBattery = glasses.device?.bandBattery ?: band.battery + LazyRow(modifier, contentPadding = PaddingValues(horizontal = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp)) { + item(key = "prompter") { + val words = text.split(Regex("\\s+")).count { it.isNotEmpty() } + ToolCard( + VisorIcons.Lines, + stringResource(R.string.prompter_title), + when { + prompter.phase == PrompterPhase.SHOWING -> stringResource(R.string.prompter_on_glasses) + words == 0 -> stringResource(R.string.prompter_card_empty) + else -> pluralStringResource(R.plurals.prompter_words, words, words) + }, + onClick = { onOpen(DisplayTool.TELEPROMPTER) }, + ) + } + item(key = "captions") { + val captions by model.captions.state.collectAsStateWithLifecycle() + val choice by model.captionsChoices.choice.collectAsStateWithLifecycle() + ToolCard( + VisorIcons.Captions, + stringResource(R.string.captions_title), + if (captions.phase == com.macedonga.visor.core.connection.CaptionsPhase.LIVE) stringResource(R.string.captions_live) else captionsSummary(choice), + onClick = { onOpen(DisplayTool.CAPTIONS) }, + ) + } + item(key = "gestures") { + ToolCard( + VisorIcons.Band, + stringResource(R.string.gestures_title), + bandBattery?.let { stringResource(R.string.gestures_card_battery, it) } ?: stringResource(R.string.gestures_card), + onClick = { onOpen(DisplayTool.GESTURES) }, + ) + } + item(key = "mirror") { + val mirror by model.screencast.state.collectAsStateWithLifecycle() + ToolCard( + VisorIcons.Mirror, + stringResource(R.string.mirror_title), + if (mirror.phase == com.macedonga.visor.core.media.ScreencastPhase.LIVE) stringResource(R.string.mirror_live) else stringResource(R.string.mirror_idle), + onClick = { onOpen(DisplayTool.MIRROR) }, + ) + } + } +} + +@Composable +private fun ToolCard(icon: ImageVector, title: String, status: String, onClick: () -> Unit) { + Column( + Modifier + .width(CARD_WIDTH) + .clip(RoundedCornerShape(18.dp)) + .background(Visor.colors.raised) + .clickable(onClick = onClick) + .padding(18.dp), + ) { + Icon(icon, null, Modifier.size(22.dp), tint = Visor.colors.textPrimary) + Spacer(Modifier.height(28.dp)) + Text(title, style = Visor.type.heading, color = Visor.colors.textPrimary, maxLines = 1, overflow = TextOverflow.Ellipsis) + Spacer(Modifier.height(2.dp)) + Text(status, style = Visor.type.label, color = Visor.colors.textSecondary, maxLines = 1, overflow = TextOverflow.Ellipsis) + } +} + +private val CARD_WIDTH = 160.dp diff --git a/app/src/main/kotlin/com/macedonga/visor/display/DisplaySquare.kt b/app/src/main/kotlin/com/macedonga/visor/display/DisplaySquare.kt new file mode 100644 index 00000000..bf131001 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/display/DisplaySquare.kt @@ -0,0 +1,41 @@ +package com.macedonga.visor.display + +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.BoxScope +import androidx.compose.foundation.layout.BoxWithConstraints +import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalDensity +import androidx.compose.ui.unit.TextUnit +import androidx.compose.ui.unit.dp +import com.macedonga.visor.core.design.Visor + +/** + * The Display's screen as the phone shows it: a black square at its true proportions, the 600 px an app draws on + * across the square. [content] gets how many phone pixels one of theirs is. + */ +@Composable +fun DisplaySquare(modifier: Modifier = Modifier, radius: androidx.compose.ui.unit.Dp = 14.dp, content: @Composable BoxScope.(px: Float) -> Unit) { + val shape = RoundedCornerShape(radius) + BoxWithConstraints(modifier.aspectRatio(1f).clip(shape).background(Color.Black).border(1.dp, Visor.colors.hairline, shape)) { + val px = with(LocalDensity.current) { maxWidth.toPx() } / DISPLAY_PX + Box(Modifier.fillMaxSize()) { content(px) } + } +} + +/** A size in the glasses' sp (one of their pixels: they draw at density 160), at the square's scale. */ +@Composable +fun glassesSp(size: Double, px: Float): TextUnit = with(LocalDensity.current) { (size.toFloat() * px).toSp() } + +const val DISPLAY_PX = 600f + +/** [size] of the glasses' pixels at the square's scale, as a phone length. */ +@Composable +fun glassesDp(size: Float, px: Float): androidx.compose.ui.unit.Dp = with(LocalDensity.current) { (size * px).toDp() } diff --git a/app/src/main/kotlin/com/macedonga/visor/display/GesturesScreen.kt b/app/src/main/kotlin/com/macedonga/visor/display/GesturesScreen.kt new file mode 100644 index 00000000..1b792af8 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/display/GesturesScreen.kt @@ -0,0 +1,180 @@ +package com.macedonga.visor.display + +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.material3.Icon +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateListOf +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.macedonga.visor.R +import com.macedonga.visor.band.BandLink +import com.macedonga.visor.band.Bands +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.core.protocol.Band +import com.macedonga.visor.settings.RowFrame +import com.macedonga.visor.settings.Rows +import com.macedonga.visor.settings.SettingsPage +import com.macedonga.visor.ui.VisorIcons +import com.macedonga.visor.ui.rememberHaptics +import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject +import kotlinx.coroutines.Job +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.launch + +/** One gesture to learn: what it does on the Display and how it is made. */ +enum class Lesson(val title: Int, val how: Int) { + SELECT(R.string.lesson_select, R.string.lesson_select_how), + BACK(R.string.lesson_back, R.string.lesson_back_how), + SCROLL(R.string.lesson_scroll, R.string.lesson_scroll_how), + ASSISTANT(R.string.lesson_assistant, R.string.lesson_assistant_how), + WAKE(R.string.lesson_wake, R.string.lesson_wake_how), + APPS(R.string.lesson_apps, R.string.lesson_apps_how); + + companion object { + /** + * The band's gestures as lessons, as the Display's input filter takes them: an index tap selects, a middle tap goes + * back, a thumb swipe scrolls, the thumb's double tap is Meta AI, the middle's wakes, holding the middle opens the + * apps. Each tap comes as press, release, button release and single tap; the second tap of a double adds a double + * tap right after its single tap. So a single tap waits a moment for a double, and only the final events count. + */ + suspend fun judge(gestures: Flow, onLesson: suspend (Lesson) -> Unit) = coroutineScope { + var pending: Job? = null + var quietUntil = 0L + gestures.collect { g -> + val now = System.currentTimeMillis() + when { + g.derived == DERIVED_SINGLE_TAP -> { + pending?.cancel() + if (now < quietUntil) return@collect + val lesson = when (g.finger) { + FINGER_INDEX -> SELECT + FINGER_MIDDLE -> BACK + else -> null + } ?: return@collect + pending = launch { + delay(DOUBLE_WINDOW_MS) + onLesson(lesson) + } + } + g.derived == DERIVED_DOUBLE_TAP -> { + pending?.cancel() + when (g.finger) { + FINGER_MIDDLE -> onLesson(WAKE) + FINGER_THUMB, FINGER_INDEX -> onLesson(ASSISTANT) + } + } + // A hold, and a swipe, can come as more than one event: the first counts. + now < quietUntil -> Unit + g.derived == DERIVED_HOLD && g.finger == FINGER_MIDDLE -> { + pending?.cancel() + quietUntil = now + QUIET_MS + onLesson(APPS) + } + g.action in ACTION_UP..ACTION_RIGHT || g.derived in DERIVED_UP..DERIVED_RIGHT -> { + quietUntil = now + QUIET_MS + onLesson(SCROLL) + } + } + } + } + + private const val FINGER_THUMB = 1 + private const val FINGER_INDEX = 2 + private const val FINGER_MIDDLE = 3 + private const val ACTION_UP = 6 + private const val ACTION_RIGHT = 9 + private const val DERIVED_SINGLE_TAP = 1 + private const val DERIVED_DOUBLE_TAP = 2 + private const val DERIVED_HOLD = 3 + private const val DERIVED_UP = 5 + private const val DERIVED_RIGHT = 8 + // The second tap of a double came about 200 ms after the first on the user's band. + private const val DOUBLE_WINDOW_MS = 420L + private const val QUIET_MS = 600L + } +} + +@HiltViewModel +class GesturesModel @Inject constructor(val band: BandLink, val bands: Bands, val connection: GlassesConnection) : ViewModel() + +/** + * The band's gestures in a list: what each does and how it is made, three dots that fill as the band hears it and a + * check once it has three times. The band is the phone's while the page is open. + */ +@Composable +fun GesturesScreen(model: GesturesModel = hiltViewModel()) { + val band by model.band.state.collectAsStateWithLifecycle() + val all by model.bands.all.collectAsStateWithLifecycle() + val pair by model.connection.pair.collectAsStateWithLifecycle() + val setUp = all.any { it.glasses == pair } + DisposableEffect(Unit) { + model.band.startLesson() + onDispose { model.band.endLesson() } + } + val haptics = rememberHaptics() + val counts = remember { mutableStateListOf(*Array(Lesson.entries.size) { 0 }) } + LaunchedEffect(Unit) { + Lesson.judge(model.band.gestures) { lesson -> + val i = lesson.ordinal + if (counts[i] >= GOAL) return@judge + counts[i]++ + if (counts[i] < GOAL) haptics.tick(0.7f) else haptics.done() + } + } + SettingsPage( + stringResource(R.string.gestures_title), + // A line only while practice cannot happen. + lede = when { + !setUp -> stringResource(R.string.gestures_no_band) + !band.gestures -> stringResource(R.string.gestures_waiting) + else -> null + }, + ) { + item(key = "lessons") { + Rows( + *Lesson.entries.map { lesson -> + @Composable { + RowFrame(stringResource(lesson.title), onClick = null, sub = stringResource(lesson.how)) { Progress(counts[lesson.ordinal]) } + } + }.toTypedArray(), + ) + } + } +} + +/** Three dots, one lit for each time the band heard it; a check once all are. */ +@Composable +private fun Progress(count: Int) { + if (count >= GOAL) { + Icon(VisorIcons.Check, null, Modifier.padding(start = 12.dp).size(20.dp), tint = Color.White) + return + } + Row(Modifier.padding(start = 12.dp), horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) { + repeat(GOAL) { i -> + Box(Modifier.size(7.dp).clip(CircleShape).background(if (i < count) Color.White else Visor.colors.control)) + } + } +} + +private const val GOAL = 3 diff --git a/app/src/main/kotlin/com/macedonga/visor/display/PrompterScreen.kt b/app/src/main/kotlin/com/macedonga/visor/display/PrompterScreen.kt new file mode 100644 index 00000000..358aa7b4 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/display/PrompterScreen.kt @@ -0,0 +1,334 @@ +package com.macedonga.visor.display + +import android.content.ClipboardManager +import android.content.Context +import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.WindowInsets +import androidx.compose.foundation.layout.asPaddingValues +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.heightIn +import androidx.compose.foundation.layout.navigationBars +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.wrapContentHeight +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.text.BasicTextField +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.SolidColor +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.pluralStringResource +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.em +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.macedonga.visor.R +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.connection.PrompterLook +import com.macedonga.visor.core.connection.PrompterPhase +import com.macedonga.visor.core.connection.Teleprompter +import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.settings.ActionPill +import com.macedonga.visor.settings.PillsRow +import com.macedonga.visor.settings.Rows +import com.macedonga.visor.settings.SettingsPage +import com.macedonga.visor.settings.Spinner +import com.macedonga.visor.settings.VisorSheet +import com.macedonga.visor.settings.SheetTitle +import com.macedonga.visor.settings.row +import com.macedonga.visor.ui.rememberHaptics +import dagger.hilt.android.lifecycle.HiltViewModel +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow + +/** The one script and how it shows, kept on the phone. */ +@Singleton +class PrompterScript @Inject constructor(@ApplicationContext context: Context) { + private val prefs = context.getSharedPreferences("teleprompter", Context.MODE_PRIVATE) + private val _text = MutableStateFlow(prefs.getString(TEXT, "").orEmpty()) + val text: StateFlow = _text.asStateFlow() + private val _look = MutableStateFlow( + PrompterLook( + sizeSp = prefs.getFloat(SIZE, 35f).toDouble(), + speed = prefs.getFloat(SPEED, 32f).toDouble(), + paged = prefs.getBoolean(PAGED, false), + ), + ) + val look: StateFlow = _look.asStateFlow() + + fun write(text: String) { + _text.value = text + prefs.edit().putString(TEXT, text).apply() + } + + fun look(look: PrompterLook) { + _look.value = look + prefs.edit().putFloat(SIZE, look.sizeSp.toFloat()).putFloat(SPEED, look.speed.toFloat()) + .putBoolean(PAGED, look.paged).apply() + } + + private companion object { + const val TEXT = "text" + const val SIZE = "size" + const val SPEED = "speed" + const val PAGED = "paged" + } +} + +@HiltViewModel +class PrompterModel @Inject constructor( + val script: PrompterScript, + val prompter: Teleprompter, + val connection: GlassesConnection, +) : ViewModel() + +/** + * The teleprompter: the script as the glasses will draw it, in their square; the script itself, how big and how fast, + * and the one button that puts it on the glasses, or updates or closes it there. + */ +@Composable +fun PrompterScreen(model: PrompterModel = hiltViewModel()) { + val text by model.script.text.collectAsStateWithLifecycle() + val look by model.script.look.collectAsStateWithLifecycle() + val state by model.prompter.state.collectAsStateWithLifecycle() + val glasses by model.connection.state.collectAsStateWithLifecycle() + val ready = glasses.link == LinkState.READY + var editing by remember { mutableStateOf(false) } + val haptics = rememberHaptics() + SettingsPage( + stringResource(R.string.prompter_title), + sheet = { + PrompterBar( + ready = ready, + empty = text.isBlank(), + phase = state.phase, + changed = state.shown != text || state.look != look, + refusal = state.refusal, + onShow = { haptics.done(); model.prompter.show(text, look) }, + onClose = { haptics.toggle(); model.prompter.close() }, + ) + if (editing) ScriptSheet(text, model.script::write) { editing = false } + }, + ) { + item(key = "square") { + ScriptPreview(text, look, Modifier.padding(horizontal = 16.dp).padding(bottom = 16.dp).clickable { editing = true }) + } + item(key = "script") { Rows(row { ScriptRow(text) { editing = true } }) } + item(key = "look") { + Spacer(Modifier.height(16.dp)) + Rows( + row { + PillsRow( + stringResource(R.string.prompter_size), SIZES.map { it to sizeName(it) }, SIZES.minByOrNull { kotlin.math.abs(it - look.sizeSp) }, + { model.script.look(look.copy(sizeSp = it)) }, + ) + }, + row { + PillsRow( + stringResource(R.string.prompter_speed), SPEEDS.map { it to "%s×".format(speedName(it)) }, + SPEEDS.minByOrNull { kotlin.math.abs(it - look.speed) }, { model.script.look(look.copy(speed = it)) }, + ) + }, + row { + PillsRow( + stringResource(R.string.prompter_layout), + listOf(false to stringResource(R.string.prompter_scroll), true to stringResource(R.string.prompter_pages)), + look.paged, { model.script.look(look.copy(paged = it)) }, + ) + }, + ) + } + item(key = "foot") { Spacer(Modifier.height(BAR_ROOM)) } + } +} + +/** + * The script in the glasses' square, at the size they will draw it: scrolled, the first line sits in the middle, + * where they centre the line being read; paged, it fills from the top as their first card does. + */ +@Composable +fun ScriptPreview(text: String, look: PrompterLook, modifier: Modifier = Modifier, radius: androidx.compose.ui.unit.Dp = 14.dp) { + DisplaySquare(modifier, radius) { px -> + val inset = Modifier.padding(glassesDp(PROMPTER_INSET, px)) + val style = Visor.type.body.copy( + fontSize = glassesSp(look.sizeSp, px), + lineHeight = 1.3.em, + fontWeight = FontWeight.Medium, + color = Color.White, + ) + if (text.isBlank()) { + Text(stringResource(R.string.prompter_empty), style = style.copy(color = Visor.colors.textTertiary), modifier = Modifier.align(Alignment.Center).then(inset)) + } else if (look.paged) { + Text(text, style = style, overflow = TextOverflow.Clip, modifier = Modifier.fillMaxSize().then(inset)) + } else { + Column(Modifier.fillMaxSize().then(inset)) { + Spacer(Modifier.weight(1f)) + Text(text, style = style, overflow = TextOverflow.Clip, modifier = Modifier.weight(1f).wrapContentHeight(Alignment.Top, unbounded = true)) + } + } + } +} + +@Composable +private fun ScriptRow(text: String, onClick: () -> Unit) { + Column(Modifier.fillMaxWidth().clickable(onClick = onClick).padding(horizontal = 16.dp, vertical = 14.dp)) { + Text(stringResource(R.string.prompter_script), style = Visor.type.body, color = Visor.colors.textPrimary) + Spacer(Modifier.height(4.dp)) + if (text.isBlank()) { + Text(stringResource(R.string.prompter_empty), style = Visor.type.label, color = Visor.colors.textSecondary) + } else { + val words = remember(text) { text.split(Regex("\\s+")).count { it.isNotEmpty() } } + Text(pluralStringResource(R.plurals.prompter_words, words, words), style = Visor.type.label, color = Visor.colors.textSecondary) + } + } +} + +/** At the foot of the page, over a fade: what the glasses are doing with the script, and the button for it. */ +@Composable +private fun PrompterBar( + ready: Boolean, + empty: Boolean, + phase: PrompterPhase, + changed: Boolean, + refusal: Long?, + onShow: () -> Unit, + onClose: () -> Unit, +) { + val colors = Visor.colors + val bottom = WindowInsets.navigationBars.asPaddingValues().calculateBottomPadding() + Box(Modifier.fillMaxSize(), contentAlignment = Alignment.BottomCenter) { + Column( + Modifier + .fillMaxWidth() + .background(Brush.verticalGradient(0f to Color.Transparent, 0.3f to colors.canvas)) + .padding(start = 16.dp, end = 16.dp, top = 28.dp, bottom = bottom + 16.dp), + horizontalAlignment = Alignment.CenterHorizontally, + ) { + val line = when { + !ready -> stringResource(R.string.prompter_not_connected) + refusal != null -> refusalWords(refusal) + phase == PrompterPhase.SHOWING -> stringResource(R.string.prompter_on_glasses) + else -> null + } + if (line != null) { + Text(line, style = Visor.type.label, color = if (refusal != null && ready) colors.signal else colors.textSecondary, modifier = Modifier.padding(bottom = 12.dp)) + } + when { + !ready -> Unit + phase == PrompterPhase.STARTING || phase == PrompterPhase.CLOSING -> Box(Modifier.height(52.dp), contentAlignment = Alignment.Center) { Spinner() } + phase == PrompterPhase.SHOWING -> Row(horizontalArrangement = Arrangement.spacedBy(10.dp)) { + WidePill(stringResource(R.string.prompter_close), onClose, Modifier.weight(1f), quiet = true) + WidePill(stringResource(R.string.prompter_update), onShow, Modifier.weight(1f), enabled = changed && !empty) + } + else -> WidePill(stringResource(R.string.prompter_show), onShow, Modifier.fillMaxWidth(), enabled = !empty) + } + } + } +} + +@Composable +internal fun WidePill(text: String, onClick: () -> Unit, modifier: Modifier, quiet: Boolean = false, enabled: Boolean = true) { + Box( + modifier + .height(52.dp) + .clip(RoundedCornerShape(50)) + .background(if (quiet) Visor.colors.control else Color.White.copy(alpha = if (enabled) 1f else 0.25f)) + .clickable(enabled = enabled, onClick = onClick), + contentAlignment = Alignment.Center, + ) { + Text(text, style = Visor.type.body, color = if (quiet) Visor.colors.textPrimary else Color.Black) + } +} + +/** The script, full height, with Paste and Clear; every keystroke is kept. */ +@Composable +private fun ScriptSheet(text: String, onText: (String) -> Unit, onDismiss: () -> Unit) { + val context = LocalContext.current + VisorSheet(onDismiss, full = true) { + SheetTitle(stringResource(R.string.prompter_script)) + Box( + Modifier + .padding(horizontal = 16.dp) + .fillMaxWidth() + .heightIn(min = 320.dp, max = 520.dp) + .clip(RoundedCornerShape(16.dp)) + .background(Visor.colors.raised) + .padding(16.dp), + ) { + if (text.isEmpty()) Text(stringResource(R.string.prompter_empty), style = Visor.type.body, color = Visor.colors.textTertiary) + BasicTextField( + value = text, + onValueChange = onText, + textStyle = Visor.type.body.copy(color = Visor.colors.textPrimary), + cursorBrush = SolidColor(Color.White), + modifier = Modifier.fillMaxSize(), + ) + } + Row(Modifier.fillMaxWidth().padding(horizontal = 24.dp).padding(top = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp), verticalAlignment = Alignment.CenterVertically) { + ActionPill(stringResource(R.string.ai_paste), { pasted(context)?.let { onText(if (text.isBlank()) it else text + "\n" + it) } }, quiet = true) + if (text.isNotEmpty()) ActionPill(stringResource(R.string.settings_clear), { onText("") }, quiet = true) + Spacer(Modifier.weight(1f)) + ActionPill(stringResource(R.string.editor_done), onDismiss) + } + } +} + +@Composable +internal fun refusalWords(code: Long): String = stringResource( + when (code) { + Teleprompter.SNAPP_NOT_ENOUGH_BATTERY -> R.string.prompter_refused_battery + Teleprompter.SNAPP_THERMAL_THROTTLING -> R.string.prompter_refused_heat + Teleprompter.SNAPP_DENIED_BY_PEAK_POWER -> R.string.prompter_refused_power + else -> R.string.prompter_refused + }, +) + +@Composable +private fun sizeName(sp: Double): String = stringResource( + when (sp) { + SIZES[0] -> R.string.prompter_size_small + SIZES[1] -> R.string.prompter_size_medium + SIZES[2] -> R.string.prompter_size_large + else -> R.string.prompter_size_huge + }, +) + +/** As a multiple of the glasses' own default, the one thing the number means for sure. */ +private fun speedName(speed: Double): String = + java.text.NumberFormat.getNumberInstance().apply { maximumFractionDigits = 2 }.format(speed / DEFAULT_SPEED) + +private fun pasted(context: Context): String? = + context.getSystemService(ClipboardManager::class.java)?.primaryClip?.takeIf { it.itemCount > 0 }?.getItemAt(0)?.coerceToText(context)?.toString() + +// The glasses' default size is 35 sp; the Display's own body text runs 24 to 32. +private val SIZES = listOf(28.0, 35.0, 44.0, 56.0) +private const val DEFAULT_SPEED = 32.0 +private val SPEEDS = listOf(16.0, 24.0, 32.0, 48.0, 64.0) +private val BAR_ROOM = 120.dp +// The glasses' own teleprompter layout is not known; this keeps the preview's text off the square's edge. +private const val PROMPTER_INSET = 28f diff --git a/app/src/main/kotlin/com/macedonga/visor/features/Catalog.kt b/app/src/main/kotlin/com/macedonga/visor/features/Catalog.kt index 0e6e6dc0..555b97c1 100644 --- a/app/src/main/kotlin/com/macedonga/visor/features/Catalog.kt +++ b/app/src/main/kotlin/com/macedonga/visor/features/Catalog.kt @@ -37,6 +37,10 @@ sealed interface Gate { data class Setting(val key: String, val on: String, val off: String? = null) : Gate } +// Meta Ray-Ban Display, whose image has no hearingservice. +private val DISPLAY = setOf(259, 270, 271) +private val NO_HEARING_SERVICE = DISPLAY + // Ray-Ban Meta (2023), Oakley Meta HSTN and Ray-Ban Meta Gen 2 share one DSP (hammerhead-rt600). Its hearing pipeline // has one output channel, while the hearing service always sends a left and a right gain map: the right one is // refused and hearing correction never starts (#42). Meta's app offers it on none of them. @@ -79,6 +83,7 @@ enum class Feature( listOf(Gate.Setting("hearing_boost_nux_completed", "true"), Gate.Config("hearing_on_sn.cf_enabled", true, live = true)) + FOCUS_FLOORS, Topic.HEARING, page = true, + absentOn = NO_HEARING_SERVICE, ), // setupForHCModes refuses "HC is not supported on this device" (INVALID_DEVICE) while the first is off. The second // is the version pass Meta's server sets beside it ("update your device"). Both apply at once on the glasses. @@ -90,7 +95,7 @@ enum class Feature( ), Topic.HEARING, page = true, - absentOn = ONE_CHANNEL_HEARING, + absentOn = NO_HEARING_SERVICE + ONE_CHANNEL_HEARING, ), // The group app's MainActivity finishes unless the first two are on; enable_aurachat starts aurachat through // sysprop_map. Without Meta's signed key material the group key is AurachatKeyManager's all-zero DEBUG_KEY. @@ -102,6 +107,8 @@ enum class Feature( Gate.Config("wearable_connectivity_device_features.enable_aurachat", true), ), Topic.HEARING, + // The Display ships neither aurachat nor the group app. + absentOn = DISPLAY, ), // The glasses' TAR gate (isSgAlarmMCEnabled); Visor's timer service keeps the alarms Meta's app refuses. ALARMS( diff --git a/app/src/main/kotlin/com/macedonga/visor/features/Features.kt b/app/src/main/kotlin/com/macedonga/visor/features/Features.kt index 920d810c..5de730bc 100644 --- a/app/src/main/kotlin/com/macedonga/visor/features/Features.kt +++ b/app/src/main/kotlin/com/macedonga/visor/features/Features.kt @@ -10,6 +10,7 @@ import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.core.settings.GlassesSettings import com.macedonga.visor.core.settings.SettingsState import com.macedonga.visor.core.settings.same +import com.macedonga.visor.pairs.pairPrefs import dagger.hilt.android.qualifiers.ApplicationContext import javax.inject.Inject import javax.inject.Singleton @@ -23,6 +24,7 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch @@ -78,10 +80,12 @@ class Features @Inject constructor( private val settings: GlassesSettings, private val connection: GlassesConnection, private val hearing: Hearing, - @ApplicationContext context: Context, + @ApplicationContext private val context: Context, ) { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - private val prefs = context.getSharedPreferences("features", Context.MODE_PRIVATE) + private var prefsFor = connection.pair.value + @Volatile + private var prefs = pairPrefs(context, "features", prefsFor) // MobileConfig values waiting for the next table, and the features they are for. private val queued = MutableStateFlow>(emptyMap()) @@ -309,6 +313,25 @@ class Features @Inject constructor( .apply() } + // Last, so everything it resets is there: another pair has its own marks, sound and test. + init { + scope.launch { + connection.pair.filterNotNull().distinctUntilChanged().collect { serial -> + if (serial == prefsFor) return@collect + prefsFor = serial + prefs = pairPrefs(context, "features", serial) + flush?.cancel() + synchronized(turning) { turning.clear() } + queued.value = emptyMap() + problem.value = null + refused.value = load(REFUSED) + restart.value = load(RESTART).filter { f -> f.gates.any { it is Gate.Config && !it.live } }.toSet() + _focus.value = FocusStyle.entries.getOrElse(prefs.getInt(FOCUS_STYLE, 0)) { FocusStyle.STANDARD } to prefs.getInt(FOCUS_VOLUME, Hearing.DEFAULT_VOLUME) + _tested.value = prefs.getBoolean(HEARING_TESTED, false) + } + } + } + private companion object { const val GATHER_MS = 1_500L const val RESTART = "restart" diff --git a/app/src/main/kotlin/com/macedonga/visor/firmware/FirmwareUpdate.kt b/app/src/main/kotlin/com/macedonga/visor/firmware/FirmwareUpdate.kt new file mode 100644 index 00000000..d3496509 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/firmware/FirmwareUpdate.kt @@ -0,0 +1,224 @@ +package com.macedonga.visor.firmware + +import com.macedonga.visor.core.identity.token.withToken +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.firmware.FirmwareOffer +import com.macedonga.visor.core.identity.firmware.fetchFirmwareOffer +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.ServiceData +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.login.MetaLogin +import java.util.UUID +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull + +sealed interface FirmwareWork { + data object Idle : FirmwareWork + /** [percent] is null until the glasses report some progress. */ + data class Installing(val percent: Int? = null) : FirmwareWork + data object Installed : FirmwareWork + data class Failed(val reason: String) : FirmwareWork +} + +/** + * The glasses' firmware from Meta. [check] asks Meta for the newest full image; [install] sends its address to the + * glasses over svc29 (the OTA service), and the glasses fetch the image over their own Wi-Fi. If they ask for the + * image over Bluetooth instead, the install stops: that fallback is not built. + */ +@Singleton +class FirmwareUpdate @Inject constructor( + private val login: MetaLogin, + private val connection: GlassesConnection, + private val log: EventLog, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + + private val _checking = MutableStateFlow(false) + val checking: StateFlow = _checking.asStateFlow() + /** Meta answered for the glasses' current build, offer or none. */ + private val _checked = MutableStateFlow(false) + val checked: StateFlow = _checked.asStateFlow() + private val _offer = MutableStateFlow(null) + val offer: StateFlow = _offer.asStateFlow() + private val _problem = MutableStateFlow("") + val problem: StateFlow = _problem.asStateFlow() + private val _work = MutableStateFlow(FirmwareWork.Idle) + val work: StateFlow = _work.asStateFlow() + private var job: Job? = null + + init { + // An offer is for one pair's build: a different pair starts over. + scope.launch { + connection.pair.filterNotNull().distinctUntilChanged().drop(1).collect { + _offer.value = null + _checked.value = false + _problem.value = "" + } + } + } + + fun check() { + if (_checking.value || job?.isActive == true) return + scope.launch { + _checking.value = true + try { + val info = connection.state.value.info ?: error("connect the glasses once, so they say which they are") + // Meta/greatwhite/greatwhite:14/UKQ1.250303.001/69951180085500080:user/release-keys + val parts = info.build.split('/') + val firmware = parts.getOrNull(4)?.substringBefore(':').orEmpty() + val type = parts.getOrNull(4)?.substringAfter(':', "user").orEmpty() + val product = parts.getOrNull(1).orEmpty() + if (firmware.isEmpty() || product.isEmpty()) error("the glasses' build is unknown") + val found = login.source.withToken { fetchFirmwareOffer(it, info.serial, "ota.$product.$type", firmware) } + _offer.value = found + _problem.value = "" + _checked.value = true + log.add("Firmware: " + (found?.let { "${it.targetVersion} is offered" } ?: "no newer build is offered")) + } catch (error: Throwable) { + if (error is kotlinx.coroutines.CancellationException) throw error + _problem.value = error.message ?: error::class.java.simpleName + log.add("Firmware: check failed: ${error.message}") + } finally { + _checking.value = false + } + } + } + + fun install() { + val offer = _offer.value ?: return + if (job?.isActive == true) return + job = scope.launch { + try { + if (connection.state.value.link != LinkState.READY) error("the glasses are not connected") + _work.value = FirmwareWork.Installing() + log.add("Firmware: installing ${offer.targetVersion} on the glasses") + val why = runCatching { installOn(offer) }.exceptionOrNull()?.let { it.message ?: it::class.java.simpleName } + if (why != null) error(why) + _offer.value = null + _work.value = FirmwareWork.Installed + log.add("Firmware: ${offer.targetVersion} is installed, the glasses restart") + } catch (error: kotlinx.coroutines.CancellationException) { + _work.value = FirmwareWork.Idle + throw error + } catch (error: Throwable) { + log.add("Firmware: install failed: ${error.message}") + _work.value = FirmwareWork.Failed(error.message ?: error::class.java.simpleName) + } + } + } + + fun cancel() { + job?.cancel() + } + + /** Back to the start after an install ended. */ + fun dismiss() { + if (job?.isActive != true) _work.value = FirmwareWork.Idle + } + + private suspend fun installOn(offer: FirmwareOffer) = coroutineScope { + val fromGlasses = Channel(Channel.UNLIMITED) + // Subscribed before the first message goes out, so no answer is missed. + val listening = launch(start = CoroutineStart.UNDISPATCHED) { + connection.serviceData.collect { if (it.serviceId == SVC_OTA) fromGlasses.send(it) } + } + suspend fun next(type: Int, ms: Long): ServiceData? = withTimeoutOrNull(ms) { + var m: ServiceData + do { + m = fromGlasses.receive() + when (m.messageType) { + MT_ERROR -> error(errorWords(m.payload)) + MT_FILE_STATUS -> error("the glasses asked for the image over Bluetooth, not over their Wi-Fi") + MT_PROGRESS -> progress(m.payload) + } + } while (m.messageType != type) + m + } + try { + val install = UUID.randomUUID().toString() + val megabytes = (offer.size / 1_000_000 + 1) * 2 + val start = Wire.fieldVarint(1, APP_OTA) + Wire.fieldBytes(2, Wire.fieldBytes(1, Wire.fieldVarint(1, megabytes))) + + intent("ota_type", Wire.fieldBytes(5, str("firmware"))) + intent("attempt_id", Wire.fieldBytes(5, str(install))) + connection.submit { it.sendOnService(SVC_OTA, MT_START_APP, start) } + val started = next(MT_START_APP_RESPONSE, ANSWER_MS) ?: error("the glasses' OTA service did not answer") + val code = runCatching { Wire.parse(started.payload).varint(1) }.getOrDefault(0) + // Already started: its workflow is up, and still takes the request. + if (code != SNAPP_SUCCESS && code != SNAPP_ALREADY_STARTED) error("the glasses would not start an install (code $code)") + + val key = Wire.fieldBytes(1, str(offer.sessionId)) + Wire.fieldBytes(2, str(install)) + val request = Wire.fieldBytes(1, key) + + Wire.fieldBytes(3, str(offer.baseVersion)) + + Wire.fieldBytes(4, str(offer.targetVersion)) + + Wire.fieldVarint(5, payloadSize(offer)) + + Wire.fieldVarint(6, offer.payloadOffset) + + (if (offer.metaHeaders.isNotEmpty()) Wire.fieldBytes(7, str(offer.metaHeaders)) else ByteArray(0)) + + Wire.fieldBytes(10, str(offer.url)) + + (offer.kekId?.let { Wire.fieldVarint(11, it) } ?: ByteArray(0)) + + (if (offer.kekHash.isNotEmpty()) Wire.fieldBytes(12, str(offer.kekHash)) else ByteArray(0)) + connection.submit { it.sendOnService(SVC_OTA, MT_INSTALL, request) } + _work.value = FirmwareWork.Installing() + next(MT_INSTALL_COMPLETE, INSTALL_MS) ?: error("the glasses did not finish installing it") + } finally { + listening.cancel() + } + } + + // The payload's size is the FILE_SIZE line of the install headers, as the Meta app reads it; else the image's size. + private fun payloadSize(offer: FirmwareOffer): Long = + offer.metaHeaders.lineSequence().firstOrNull { it.startsWith("FILE_SIZE") }?.substringAfter('=')?.trim()?.toLongOrNull() ?: offer.size + + /** Progress for the sheet: the percentage the glasses give, when it is in range. */ + private fun progress(payload: ByteArray) { + val f = runCatching { Wire.parse(payload) }.getOrNull() ?: return + if (!f.has(3)) return + val percent = f.float(3).toInt() + if (percent in 0..100) _work.value = FirmwareWork.Installing(percent) + } + + private fun errorWords(payload: ByteArray): String = runCatching { + val f = Wire.parse(payload) + val stage = STAGES.getOrElse(f.varint(2).toInt() - 1) { "stage ${f.varint(2)}" } + "the glasses stopped the install at $stage, code ${f.varint(3)}, error ${f.varint(5)}" + + if (f.varint(4) == 1L) " (retry)" else "" + }.getOrDefault("the glasses stopped the install") + + private fun str(s: String) = s.encodeToByteArray() + + private fun intent(key: String, value: ByteArray) = Wire.fieldBytes(3, Wire.fieldBytes(1, str(key)) + value) + + private companion object { + const val SVC_OTA = 29 + const val MT_INSTALL = 1 + const val MT_PROGRESS = 2 + const val MT_ERROR = 3 + const val MT_FILE_STATUS = 4 + const val MT_INSTALL_COMPLETE = 7 + const val MT_START_APP = 2000 + const val MT_START_APP_RESPONSE = 2001 + // The OTA app, as the Meta app numbers it for svc29 starts. + const val APP_OTA = 1L + const val SNAPP_SUCCESS = 1L + const val SNAPP_ALREADY_STARTED = 12L + const val ANSWER_MS = 15_000L + // A download over the glasses' Wi-Fi and the A/B apply: a guess, no timing was read. + const val INSTALL_MS = 60 * 60_000L + // OtaUpdateInstallStage, in the order the Meta app numbers them from 1. + val STAGES = listOf("transfer", "apply", "reboot", "stopped", "startup", "complete", "download") + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt index 04c62a70..661772ef 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeFeed.kt @@ -58,6 +58,7 @@ import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.ImageBitmap import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.graphics.graphicsLayer +import androidx.compose.ui.graphics.lerp import androidx.compose.ui.graphics.vector.ImageVector import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.platform.LocalContext @@ -68,6 +69,7 @@ import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import androidx.lifecycle.compose.LifecycleResumeEffect import com.macedonga.visor.R +import com.macedonga.visor.core.identity.language.GlassesLook import com.macedonga.visor.core.connection.CaseBattery import com.macedonga.visor.core.connection.GlassesState import com.macedonga.visor.core.design.Visor @@ -150,6 +152,9 @@ fun StatusPill( onSetUp: () -> Unit, onChangeOwner: (address: String?) -> Unit, modifier: Modifier, + // The Display's Neural Band, when Visor has read it. + band: Int? = null, + bandCharging: Boolean = false, ) { when { !allowed -> return Pill(stringResource(R.string.home_allow_bluetooth), onAllow, modifier) @@ -205,6 +210,11 @@ fun StatusPill( Level(VisorIcons.Case, stringResource(R.string.glasses_case_label), case, critical = false) if (caseCharging) Icon(VisorIcons.Bolt, stringResource(R.string.mood_charging), Modifier.padding(start = 2.dp).size(13.dp), tint = colors.textPrimary) } + if (band != null) { + Spacer(Modifier.size(14.dp)) + Level(VisorIcons.Band, stringResource(R.string.topic_band), band, critical = band <= 10 && !bandCharging) + if (bandCharging) Icon(VisorIcons.Bolt, stringResource(R.string.mood_charging), Modifier.padding(start = 2.dp).size(13.dp), tint = colors.textPrimary) + } } } // Over Wi-Fi Direct the battery is not reported: the last reading stays, and the import takes the place of @@ -233,6 +243,79 @@ fun StatusPill( } } +/** + * A pair that is not connected, in one capsule: how it is, or how long ago it said so, its batteries as last read, + * dimmed once old, and the button that connects to it set into the capsule's end. No button while the Live or an + * import has the pair that is connected. + */ +@Composable +fun OtherPairPill(mood: Mood, seen: PairSeen?, peeking: Boolean, onConnect: (() -> Unit)?, modifier: Modifier) { + val colors = Visor.colors + val said = seen?.at?.takeIf { it > 0 } + val fresh = said != null && rememberNow() - said < FRESH_MS + Row( + modifier + .clip(RoundedCornerShape(50)) + .background(colors.surface) + .animateContentSize(spring(dampingRatio = 0.8f, stiffness = 300f)) + .padding(start = 16.dp, end = if (onConnect != null) 4.dp else 16.dp, top = 4.dp, bottom = 4.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + val words = when { + peeking || fresh -> stringResource(moodWords(mood)) + said != null -> stringResource(R.string.pill_seen, shortAgo(said)) + else -> stringResource(R.string.pill_not_connected) + } + Text(words, style = Visor.type.label, color = colors.textSecondary, modifier = Modifier.padding(vertical = 5.dp)) + val glasses = seen?.glasses + if (glasses != null && !peeking) { + Spacer(Modifier.size(12.dp)) + Row(Modifier.graphicsLayer { alpha = if (fresh) 1f else 0.6f }, verticalAlignment = Alignment.CenterVertically) { + Level(VisorIcons.Glasses, stringResource(R.string.glasses_battery), glasses, critical = false) + seen.case?.let { + Spacer(Modifier.size(14.dp)) + Level(VisorIcons.Case, stringResource(R.string.glasses_case_label), it, critical = false) + } + } + } + if (onConnect != null) { + Spacer(Modifier.size(12.dp)) + Box( + Modifier + .clip(RoundedCornerShape(50)) + .background(colors.textPrimary) + .clickable(onClick = onConnect) + .padding(horizontal = 14.dp, vertical = 5.dp), + ) { + Text(stringResource(R.string.glasses_connect), style = Visor.type.label, color = colors.canvas) + } + } + } +} + +/** Which of the pairs the stage shows: one dot each, the shown one drawn out. */ +@Composable +fun PairDots(count: Int, index: Int, modifier: Modifier = Modifier) { + val colors = Visor.colors + Row(modifier, horizontalArrangement = Arrangement.spacedBy(6.dp)) { + repeat(count) { i -> + val on by animateFloatAsState(if (i == index) 1f else 0f, spring(dampingRatio = 0.8f, stiffness = 400f), label = "dot") + Box( + Modifier + .size(width = (5 + 9 * on).dp, height = 5.dp) + .clip(RoundedCornerShape(50)) + .background(lerp(colors.control, colors.textPrimary, on)), + ) + } + } +} + +/** What to call a pair: the name it gave itself, else its frame and colour as Meta has them. */ +@Composable +fun pairName(seen: PairSeen?, look: GlassesLook?): String = + seen?.name ?: look?.styleName?.let { style -> look.colorName?.let { stringResource(R.string.pair_look_name, style, it) } ?: style } + ?: stringResource(R.string.glasses_default_name) + /** * The case's charge as a percentage. The glasses report a level on a scale that is not always 100, and they * know it while the case is away too, so neither a missing percent nor an absent case hides it. @@ -529,20 +612,22 @@ private fun Fact(label: String, value: String, dim: Boolean, modifier: Modifier) } @Composable -fun Places(onMedia: () -> Unit, onLive: () -> Unit, onMetaAi: () -> Unit, onSettings: () -> Unit, modifier: Modifier) { +fun Places(onMedia: () -> Unit, onLive: () -> Unit, onMetaAi: () -> Unit, onSettings: () -> Unit, modifier: Modifier, forThisPair: Boolean = true) { Column(modifier) { Place(VisorIcons.Gallery, stringResource(R.string.home_place_media), stringResource(R.string.home_place_media_sub), onMedia) - Place(VisorIcons.Live, stringResource(R.string.home_place_live), stringResource(R.string.home_place_live_sub), onLive) - Place(VisorIcons.Voice, stringResource(R.string.home_place_meta_ai), stringResource(R.string.home_place_meta_ai_sub), onMetaAi) - Place(VisorIcons.Settings, stringResource(R.string.home_place_settings), stringResource(R.string.home_place_settings_sub), onSettings) + // The Live, Meta AI and settings are the connected pair's: not while the home shows another. + Place(VisorIcons.Live, stringResource(R.string.home_place_live), stringResource(R.string.home_place_live_sub), onLive, forThisPair) + Place(VisorIcons.Voice, stringResource(R.string.home_place_meta_ai), stringResource(R.string.home_place_meta_ai_sub), onMetaAi, forThisPair) + Place(VisorIcons.Settings, stringResource(R.string.home_place_settings), stringResource(R.string.home_place_settings_sub), onSettings, forThisPair) } } @Composable -private fun Place(icon: ImageVector, title: String, detail: String, onClick: () -> Unit) { +private fun Place(icon: ImageVector, title: String, detail: String, onClick: () -> Unit, enabled: Boolean = true) { val colors = Visor.colors + val shown by animateFloatAsState(if (enabled) 1f else 0.35f, tween(220), label = "place") Row( - Modifier.fillMaxWidth().clickable(onClick = onClick).padding(horizontal = 24.dp, vertical = 14.dp), + Modifier.fillMaxWidth().graphicsLayer { alpha = shown }.clickable(enabled = enabled, onClick = onClick).padding(horizontal = 24.dp, vertical = 14.dp), verticalAlignment = Alignment.CenterVertically, ) { Box(Modifier.size(44.dp).clip(RoundedCornerShape(13.dp)).background(colors.raised), contentAlignment = Alignment.Center) { @@ -645,6 +730,8 @@ fun moodWords(mood: Mood): Int = when (mood) { } private val RISE = CubicBezierEasing(0.2f, 0f, 0f, 1f) +// A reading newer than this, from a quick look, counts as now. +private const val FRESH_MS = 5 * 60_000L private const val FIRST_MS = 350f private const val STEP_MS = 80f private const val RISE_MS = 620f diff --git a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt index 7a79adc5..879d1fa9 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/HomeScreen.kt @@ -28,6 +28,8 @@ import androidx.compose.foundation.gestures.scrollable import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.BoxWithConstraints import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.clickable import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.WindowInsets @@ -43,8 +45,10 @@ import androidx.compose.foundation.layout.wrapContentHeight import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.rememberLazyListState import androidx.compose.foundation.shape.GenericShape +import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.SideEffect import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue @@ -53,11 +57,14 @@ import androidx.compose.runtime.mutableIntStateOf import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.rememberUpdatedState import androidx.compose.runtime.setValue import androidx.compose.runtime.snapshotFlow import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clipToBounds +import androidx.compose.ui.draw.clip +import androidx.compose.ui.res.stringResource import androidx.compose.ui.draw.drawBehind import androidx.compose.ui.geometry.CornerRadius import androidx.compose.ui.geometry.Offset @@ -84,6 +91,9 @@ import com.macedonga.visor.connection.GlassesService import com.macedonga.visor.core.connection.GlassesLink import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.connection.GlassesState +import com.macedonga.visor.pairs.Pairs +import com.macedonga.visor.R +import com.macedonga.visor.settings.TextSheet import com.macedonga.visor.core.design.Visor import com.macedonga.visor.core.identity.IdentityStore import com.macedonga.visor.core.media.LiveStream @@ -101,11 +111,13 @@ import com.macedonga.visor.stage.LightPool import com.macedonga.visor.stage.LocalStage import com.macedonga.visor.stage.Looks import com.macedonga.visor.stage.Mood +import com.macedonga.visor.stage.modelled import com.macedonga.visor.stage.POOL_Y import com.macedonga.visor.stage.SHEET_RADIUS import com.macedonga.visor.stage.STAGE_ASPECT import com.macedonga.visor.stage.StageDebug import com.macedonga.visor.stage.StageLight +import com.macedonga.visor.stage.StagePages import com.macedonga.visor.stage.StageSpot import com.macedonga.visor.stage.closedOf import com.macedonga.visor.stage.liveliness @@ -142,7 +154,10 @@ class HomeViewModel @Inject constructor( private val store: IdentityStore, keeper: LeKeeper, val looks: Looks, - lastSeen: LastSeen, + val lastSeen: LastSeen, + val pairs: Pairs, + val band: com.macedonga.visor.band.BandLink, + val bands: com.macedonga.visor.band.Bands, val star: StarAsk, ) : ViewModel() { /** The LE link is up: the glasses are right here, whatever RFCOMM says. */ @@ -151,29 +166,10 @@ class HomeViewModel @Inject constructor( val paired = MutableStateFlow(null) val shots = MutableStateFlow>(emptyList()) val firstName = MutableStateFlow(null) - private val prefs = context.getSharedPreferences("home", Context.MODE_PRIVATE) val lastBattery = lastSeen.battery - val lastFacts = MutableStateFlow(readFacts()) val lastPosture = lastSeen.posture init { - viewModelScope.launch { - connection.state.collect { state -> - val was = lastFacts.value - // A volume the glasses have not reported comes as -1: until they do, the one Android holds for them. - val now = GlassesFacts( - freeMb = state.device?.storage?.freeMb?.takeIf { it >= 0 } ?: was.freeMb, - volume = state.device?.volume?.percent?.toInt()?.takeIf { it in 0..100 } - ?: heldVolume(context, (state.target?.link as? GlassesLink.Bluetooth)?.address) ?: was.volume, - temperatureC = state.device?.battery?.temperatureC?.takeIf { it > -40 && it < 100 } ?: was.temperatureC, - ) - if (now != was) { - lastFacts.value = now - prefs.edit().putLong("free_mb", now.freeMb ?: -1).putInt("volume", now.volume ?: -1) - .putFloat("temperature", now.temperatureC?.toFloat() ?: Float.NaN).apply() - } - } - } // Whatever is saved shows up here as soon as it lands. viewModelScope.launch { media.saved.collect { shots.value = recentShots(context) } } } @@ -182,13 +178,8 @@ class HomeViewModel @Inject constructor( viewModelScope.launch(Dispatchers.IO) { paired.value = runCatching { store.all().isNotEmpty() }.getOrDefault(false) } viewModelScope.launch { shots.value = recentShots(context) } viewModelScope.launch { firstName.value = ownerFirstName(context) } + pairs.refresh() } - - private fun readFacts() = GlassesFacts( - freeMb = prefs.getLong("free_mb", -1).takeIf { it >= 0 }, - volume = prefs.getInt("volume", -1).takeIf { it in 0..100 }, - temperatureC = prefs.getFloat("temperature", Float.NaN).takeIf { !it.isNaN() }?.toDouble(), - ) } /** @@ -232,6 +223,7 @@ fun HomeScreen( onLive: () -> Unit, onMetaAi: () -> Unit, onSettings: () -> Unit, + onDisplayTool: (com.macedonga.visor.display.DisplayTool) -> Unit = {}, model: HomeViewModel = hiltViewModel(), ) { val context = LocalContext.current @@ -247,10 +239,28 @@ fun HomeScreen( val shots by model.shots.collectAsStateWithLifecycle() val lastBattery by model.lastBattery.collectAsStateWithLifecycle() val firstName by model.firstName.collectAsStateWithLifecycle() - val lastFacts by model.lastFacts.collectAsStateWithLifecycle() val lastPosture by model.lastPosture.collectAsStateWithLifecycle() val nearby by model.nearby.collectAsStateWithLifecycle() val paired by model.paired.collectAsStateWithLifecycle() + // Every pair set up here; the stage shows one at a time and a drag goes to the next. + val pairs by model.pairs.all.collectAsStateWithLifecycle() + val seen by model.lastSeen.seen.collectAsStateWithLifecycle() + val looks by model.looks.all.collectAsStateWithLifecycle() + val pictures by model.looks.pictures.collectAsStateWithLifecycle() + val peeking by model.pairs.peeking.collectAsStateWithLifecycle() + val current by model.connection.pair.collectAsStateWithLifecycle() + // The page shown: a pair's serial, null for the pair Visor is for, or ADD_PAGE after the last pair. + var page by remember { mutableStateOf(null) } + val adding = page == ADD_PAGE && pairs.isNotEmpty() + val shownIndex = if (adding) pairs.size else pairs.indexOfFirst { it.serial == (page ?: current) } + val shown = pairs.getOrNull(shownIndex) + // A pair that is not the one Visor is for: it shows as last heard from, and can be connected. + val other = shown != null && current != null && shown.serial != current + // Away from the pair Visor is for, its Live, Meta AI and settings are not this page's. + val elsewhere = other || adding + val shownSeen = shown?.let { seen[it.serial] } + val lastFacts = (if (elsewhere) shownSeen else current?.let(seen::get))?.facts ?: GlassesFacts(null, null, null) + var renaming by remember { mutableStateOf(null) } val forced by StageDebug.mood.collectAsStateWithLifecycle() val variant by StageDebug.variant.collectAsStateWithLifecycle() val haptics = rememberHaptics() @@ -366,7 +376,13 @@ fun HomeScreen( wasLive = live } val mood = forced ?: when { - paired == false -> Mood.ABSENT + paired == false || adding -> Mood.ABSENT + other -> when { + peeking == shown?.serial -> Mood.SEARCHING + shownSeen?.inCase == true -> Mood.IN_CASE + shownSeen?.folded == true -> Mood.ASLEEP + else -> Mood.FAR + } transferring -> Mood.IMPORTING live && state.link != LinkState.READY -> Mood.RECORDING (importing || settling || liveSettling) && state.link != LinkState.READY -> Mood.CONNECTED @@ -384,13 +400,38 @@ fun HomeScreen( LaunchedEffect(stage) { if (firstVisit) stage.light.dark() } StageLight(lookOf(mood, colors)) val look by model.looks.look.collectAsStateWithLifecycle() - stage.pose.variants = variant?.let(::listOf) ?: variantsOf(look) + val shownLook = if (other) shown?.let { looks[it.serial] } else look + stage.pose.variants = variant?.let(::listOf) ?: variantsOf(shownLook) + // The model is a Wayfarer: another frame, or a pair not connected, shows as Meta pictures it, once Meta has. + val currentPicture = current?.takeIf { !modelled(looks[it]) }?.let(pictures::get) + val picture = when { + other -> shown?.let { pictures[it.serial] } + !modelled(shownLook) -> currentPicture + else -> null + } + SideEffect { + stage.picture = picture + stage.vacant = adding + } + // Other screens show the pair Visor is for. + val leavingPicture = rememberUpdatedState(currentPicture) + DisposableEffect(Unit) { + onDispose { + stage.picture = leavingPicture.value + stage.vacant = false + } + } + // A pair shown that is not connected gets a quick look at its batteries. + LaunchedEffect(shown?.serial, other) { + if (other && shown != null) model.pairs.peek(shown.serial) + } + DisposableEffect(Unit) { onDispose { model.pairs.stopPeek() } } // Other screens that show the pair take its mood and hinge from here. SideEffect { stage.mood = mood stage.lastFolded = lastPosture.folded } - Choreography(state, mood, stage.light, stage.pose, haptics::connected, haptics::shutter) + Choreography(if (elsewhere) GlassesState() else state, mood, stage.light, stage.pose, haptics::connected, haptics::shutter) val list = rememberLazyListState() remember { @@ -414,7 +455,7 @@ fun HomeScreen( if (list.firstVisibleItemIndex == 0) list.firstVisibleItemScrollOffset.toFloat() else fullPx } // Folded, the pair sits small in the middle of the strip, clear of the camera cut-out; at launch it fades in. - val spot = StageSpot(closedOf(state, mood, lastPosture.folded), liveliness(mood)) { + val spot = StageSpot(if (elsewhere) closedOf(GlassesState(), mood, shownSeen?.folded ?: true) else closedOf(state, mood, lastPosture.folded), liveliness(mood)) { folding(fullPx, stripPx, fold, HOME_STRIP_SCALE, topPx + (stripPx - topPx) / 2) alpha = entranceAlpha(entrance, STAGE_FADE_FROM_MS, STAGE_FADE_MS) } @@ -475,11 +516,16 @@ fun HomeScreen( waitingCount = fresh.size, leaving = leaving, onLeft = { leaving = false }, ) } - Facts(lastFacts, live = here, lastCapture = lastCapture, modifier = Modifier.padding(top = GAP).appear(entrance, 2)) + Facts(lastFacts, live = here && !elsewhere, lastCapture = lastCapture, modifier = Modifier.padding(top = GAP).appear(entrance, 2)) Spacer(Modifier.height(FOOT)) } } - item(key = "places") { Places(onMedia, onLive, onMetaAi, onSettings, Modifier.appear(entrance, 5)) } + // The Display's tools, for a Display: known by its build while connected, by its band while away. + val bandsHere = model.bands.all.value + if (!elsewhere && (com.macedonga.visor.settings.hasDisplay(state.info?.build) || bandsHere.any { it.glasses == current })) { + item(key = "display") { com.macedonga.visor.display.DisplayShelf(onDisplayTool, Modifier.padding(top = 4.dp, bottom = 8.dp).appear(entrance, 4)) } + } + item(key = "places") { Places(onMedia, onLive, onMetaAi, onSettings, Modifier.appear(entrance, 5), forThisPair = !elsewhere) } } @@ -493,18 +539,68 @@ fun HomeScreen( layout(placeable.width, h) { placeable.place(0, 0) } } .scrollable(list, Orientation.Vertical, reverseDirection = true) - .turnsThePair { haptics.tick(0.35f) }, + .turnsThePair( + // The pairs, then the page for adding one. + pages = if (pairs.isNotEmpty() && shownIndex >= 0) StagePages( + can = { d -> shownIndex + d in 0..pairs.size }, + go = { d -> page = pairs.getOrNull(shownIndex + d)?.serial ?: ADD_PAGE }, + ) else null, + ) { haptics.tick(0.35f) }, ) { val fade = Modifier.graphicsLayer { alpha = (1f - (fold * 2.5f).coerceIn(0f, 1f)) * stage.shown(spot) } - Greeting( - firstName, - Modifier.align(Alignment.TopStart).statusBarsPadding().padding(horizontal = 24.dp, vertical = 12.dp).then(fade).appear(entrance, 0), - ) - StatusPill( - state, mood, lastBattery, paired, allowed, transferring, importProgress, waiting = if (importing) 0 else fresh.size, onImport = onImport, - onAllow = { ask.launch(wanted()) }, onSetUp = onSetUp, onChangeOwner = onChangeOwner, - modifier = Modifier.align(Alignment.BottomCenter).padding(bottom = 26.dp).then(fade).appear(entrance, 0), - ) + Column(Modifier.align(Alignment.TopStart).statusBarsPadding().padding(horizontal = 24.dp, vertical = 12.dp).then(fade).appear(entrance, 0)) { + Greeting(firstName, Modifier) + // Which pair this is; a tap names it. + if (shown != null) { + Text( + pairName(shownSeen, looks[shown.serial]), + style = Visor.type.label, + color = Visor.colors.textSecondary, + modifier = Modifier + .padding(top = 2.dp) + .clip(RoundedCornerShape(6.dp)) + .clickable { renaming = shown.serial } + .padding(vertical = 4.dp), + ) + } + } + Column( + Modifier.align(Alignment.BottomCenter).padding(bottom = if (pairs.size > 1) 12.dp else 26.dp).then(fade).appear(entrance, 0), + horizontalAlignment = Alignment.CenterHorizontally, + ) { + if (adding) { + Pill(stringResource(R.string.glasses_set_up_new), onSetUp) + } else if (other && shown != null) { + OtherPairPill( + mood, shownSeen, peeking = peeking == shown.serial, + onConnect = if (live || importing) null else ({ + haptics.toggle() + model.pairs.connect(shown.serial) + }), + modifier = Modifier, + ) + } else { + val band by model.band.state.collectAsStateWithLifecycle() + val bands by model.bands.all.collectAsStateWithLifecycle() + val ours = band.takeIf { b -> b.battery != null && bands.any { it.serial == b.serial && it.glasses == current } } + StatusPill( + state, mood, lastBattery, paired, allowed, transferring, importProgress, waiting = if (importing) 0 else fresh.size, onImport = onImport, + onAllow = { ask.launch(wanted()) }, onSetUp = onSetUp, onChangeOwner = onChangeOwner, modifier = Modifier, + band = state.device?.bandBattery ?: ours?.battery, + bandCharging = state.device?.bandCharging ?: (ours?.charging == true), + ) + } + if (pairs.isNotEmpty()) PairDots(pairs.size + 1, shownIndex, Modifier.padding(top = 12.dp)) + } + renaming?.let { serial -> + TextSheet( + stringResource(R.string.glasses_name), seen[serial]?.name.orEmpty(), + onSave = { model.lastSeen.rename(serial, it.trim()) }, + onDismiss = { renaming = null }, + hint = pairName(null, looks[serial]), + valid = { it.isNotBlank() && it.trim().length <= 40 }, + ) + } } } @@ -566,4 +662,7 @@ private val FOOT = 20.dp private const val SETTLE_MS = 25_000L private const val DONE_HOLD_MS = 2_000L private const val REFRESH_MS = 60_000L + +/** The home's page after the last pair, for setting up another. */ +private const val ADD_PAGE = "+" private const val STAR_DELAY_MS = 4_000L diff --git a/app/src/main/kotlin/com/macedonga/visor/home/LastSeen.kt b/app/src/main/kotlin/com/macedonga/visor/home/LastSeen.kt index 58d310db..2573902b 100644 --- a/app/src/main/kotlin/com/macedonga/visor/home/LastSeen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/home/LastSeen.kt @@ -2,9 +2,12 @@ package com.macedonga.visor.home import android.content.Context import com.macedonga.visor.core.connection.GlassesConnection -import com.macedonga.visor.core.media.LiveStream +import com.macedonga.visor.core.connection.GlassesLink +import com.macedonga.visor.core.connection.GlassesState import com.macedonga.visor.core.media.LivePhase +import com.macedonga.visor.core.media.LiveStream import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.settings.GlassesSettings import dagger.hilt.android.qualifiers.ApplicationContext import javax.inject.Inject import javax.inject.Singleton @@ -12,9 +15,16 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch +import org.json.JSONObject /** The glasses' batteries as last read, for when they are out of reach. */ data class LastBattery(val glasses: Int, val case: Int?, val at: Long) @@ -22,61 +32,198 @@ data class LastBattery(val glasses: Int, val case: Int?, val at: Long) /** How the glasses were the last time they said: folded, docked in the case. Their sleep looks like silence. */ data class LastPosture(val folded: Boolean, val inCase: Boolean) +/** One pair as last heard from: batteries, how it sat, the home's numbers, and the name it gave itself. */ +data class PairSeen( + val glasses: Int? = null, + val case: Int? = null, + // The case was on its charger at that reading. + val caseCharging: Boolean = false, + val at: Long = 0, + val folded: Boolean = false, + val inCase: Boolean = false, + val freeMb: Long? = null, + val volume: Int? = null, + val temperatureC: Double? = null, + val name: String? = null, + // Named here while away: the glasses take the name when they are next connected. + val nameToSend: Boolean = false, +) { + val battery: LastBattery? get() = glasses?.let { LastBattery(it, case, at) } + val posture: LastPosture get() = LastPosture(folded, inCase) + val facts: GlassesFacts get() = GlassesFacts(freeMb, volume, temperatureC) +} + /** - * How the glasses were when last heard from, kept across launches. One for the process, so the service writes it - * while no screen is open and the home, the widget and the tile all read the same thing. + * How each pair was when last heard from, kept across launches. One for the process, so the service writes it while + * no screen is open and the home, the widget and the tile all read the same thing. [battery] and [posture] are the + * current pair's; the home's other pairs read [seen]. */ @Singleton class LastSeen @Inject constructor( - @ApplicationContext context: Context, - connection: GlassesConnection, + @ApplicationContext private val context: Context, + private val connection: GlassesConnection, + private val settings: GlassesSettings, live: LiveStream, ) { - private val prefs = context.getSharedPreferences("home", Context.MODE_PRIVATE) + private val prefs = context.getSharedPreferences("pairs_seen", Context.MODE_PRIVATE) private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - private val _battery = MutableStateFlow(readBattery()) - private val _posture = MutableStateFlow(LastPosture(prefs.getBoolean("folded", false), prefs.getBoolean("in_case", false))) - val battery: StateFlow = _battery.asStateFlow() - val posture: StateFlow = _posture.asStateFlow() + private val savedAt = java.util.concurrent.ConcurrentHashMap() + private val _seen = MutableStateFlow(prefs.all.mapNotNull { (serial, text) -> (text as? String)?.let(::decode)?.let { serial to it } }.toMap()) + + /** Every pair heard from, by serial. */ + val seen: StateFlow> = _seen.asStateFlow() + + private val current = combine(connection.pair, _seen) { serial, all -> serial?.let(all::get) } + val battery: StateFlow = current.map { it?.battery }.distinctUntilChanged().stateIn(scope, SharingStarted.Eagerly, null) + val posture: StateFlow = + current.map { it?.posture ?: LastPosture(false, false) }.distinctUntilChanged().stateIn(scope, SharingStarted.Eagerly, LastPosture(false, false)) init { + // The pair Visor is for has its readings from the start, the old shared ones included. + scope.launch { connection.pair.filterNotNull().collect { serial -> if (serial !in _seen.value) update(serial) { it } } } + scope.launch { connection.state.collect(::record) } + // The glasses' own name, unless one given here is still on its way to them. scope.launch { - connection.state.collect { state -> - val battery = state.device?.battery ?: return@collect - val case = state.device?.case?.levelPercent() - val was = _battery.value - val now = LastBattery(battery.percent.toInt(), case ?: was?.case, System.currentTimeMillis()) - if (now.glasses != was?.glasses || now.case != was?.case) writeBattery(now) - _battery.value = now - } - } - scope.launch { - connection.state.collect { state -> - val device = state.device ?: return@collect - if (state.link != LinkState.READY) return@collect - // The case's "present" only says the case is known; charging folded is what says they sit in it. - val was = _posture.value - val inCase = device.battery?.charging?.let { it && device.hinge != "OPEN" } - writePosture(LastPosture(folded = device.hinge?.let { it == "CLOSED" } ?: was.folded, inCase = inCase ?: was.inCase)) + settings.state.map { it.pair to (it.readAt != null) to it.stored[NAME_KEY] }.distinctUntilChanged().collect { (read, stored) -> + val (serial, wasRead) = read + if (serial == null || !wasRead) return@collect + val mine = _seen.value[serial]?.takeIf { it.nameToSend }?.name + when { + mine == null -> update(serial) { it.copy(name = stored?.takeIf { n -> n.isNotBlank() && n != LEFTOVER_NAME }) } + stored == mine -> update(serial) { it.copy(nameToSend = false) } + else -> settings.write(NAME_KEY, mine) + } } } // The glasses stream only open and out of the case: a Live that ran says so while Bluetooth is away for it, and // the home it goes back to does not unfold the pair once the link is back. scope.launch { - live.status.collect { status -> if (status.phase == LivePhase.RUNNING) writePosture(LastPosture(folded = false, inCase = false)) } + live.status.collect { status -> + val serial = connection.pair.value + if (status.phase == LivePhase.RUNNING && serial != null) update(serial) { it.copy(folded = false, inCase = false) } + } + } + } + + /** Keep what [state] says about its pair: the connection's own, or a quick look at another pair. */ + fun record(state: GlassesState) { + val target = state.target ?: return + val device = state.device ?: return + update(target.serial) { was -> + val battery = device.battery + var now = was + if (battery != null) { + // The case reads out only through the glasses in it. One that was charging has gone on charging since: + // its last reading is out of date. + val case = device.case?.levelPercent() + now = now.copy( + glasses = battery.percent.toInt(), at = System.currentTimeMillis(), + case = case ?: was.case.takeUnless { was.caseCharging }, + caseCharging = case != null && device.case?.charging == true, + ) + } + // The case's "present" only says the case is known; charging folded is what says they sit in it. + if (state.link == LinkState.READY) { + val inCase = battery?.charging?.let { it && device.hinge != "OPEN" } + now = now.copy(folded = device.hinge?.let { it == "CLOSED" } ?: was.folded, inCase = inCase ?: was.inCase) + } + // A volume the glasses have not reported comes as -1: until they do, the one Android holds for them. + now.copy( + freeMb = device.storage?.freeMb?.takeIf { it >= 0 } ?: was.freeMb, + volume = device.volume?.percent?.toInt()?.takeIf { it in 0..100 } + ?: heldVolume(context, (target.link as? GlassesLink.Bluetooth)?.address) ?: was.volume, + temperatureC = battery?.temperatureC?.takeIf { it > -40 && it < 100 } ?: was.temperatureC, + ) } } - private fun writePosture(now: LastPosture) { - if (now == _posture.value) return - _posture.value = now - prefs.edit().putBoolean("folded", now.folded).putBoolean("in_case", now.inCase).apply() + /** Call [serial] [name]: the glasses take it at once when they are here, else when they are next connected. */ + fun rename(serial: String, name: String) { + val here = connection.state.value.let { it.target?.serial == serial && it.link == LinkState.READY } && settings.state.value.pair == serial + val sent = here && settings.write(NAME_KEY, name) + update(serial) { it.copy(name = name, nameToSend = !sent) } } - private fun readBattery(): LastBattery? = prefs.getInt("glasses", -1).takeIf { it >= 0 }?.let { - LastBattery(it, prefs.getInt("case", -1).takeIf { c -> c >= 0 }, prefs.getLong("at", 0)) + /** Hold [seen] for [serial] in memory only: a debug build's demo pair. */ + @Synchronized + fun keep(serial: String, seen: PairSeen) { + _seen.value = _seen.value + (serial to seen) } - private fun writeBattery(b: LastBattery) = - prefs.edit().putInt("glasses", b.glasses).putInt("case", b.case ?: -1).putLong("at", b.at).apply() + /** A pair forgotten: what it last said goes too. */ + @Synchronized + fun forget(serial: String) { + _seen.value = _seen.value - serial + savedAt.remove(serial) + prefs.edit().remove(serial).apply() + } + + // The connection, a quick look and the name all write here, from their own threads. + @Synchronized + private fun update(serial: String, change: (PairSeen) -> PairSeen) { + val was = _seen.value[serial] ?: migrated(serial) + val now = change(was) + if (now == _seen.value[serial]) return + _seen.value = _seen.value + (serial to now) + // Every reading moves the time; the file follows only a change, or a minute. + if (now.copy(at = 0) != was.copy(at = 0) || now.at - (savedAt[serial] ?: 0) > 60_000) { + savedAt[serial] = now.at + prefs.edit().putString(serial, encode(now)).apply() + } + } + + /** Before Visor kept pairs apart, the one pair's readings were the home's: the first pair heard from takes them. */ + private fun migrated(serial: String): PairSeen { + val home = context.getSharedPreferences("home", Context.MODE_PRIVATE) + if (!home.contains("glasses") && !home.contains("folded") && !home.contains("free_mb")) return PairSeen() + val old = PairSeen( + glasses = home.getInt("glasses", -1).takeIf { it >= 0 }, + case = home.getInt("case", -1).takeIf { it >= 0 }, + at = home.getLong("at", 0), + folded = home.getBoolean("folded", false), + inCase = home.getBoolean("in_case", false), + freeMb = home.getLong("free_mb", -1).takeIf { it >= 0 }, + volume = home.getInt("volume", -1).takeIf { it in 0..100 }, + temperatureC = home.getFloat("temperature", Float.NaN).takeIf { !it.isNaN() }?.toDouble(), + ) + home.edit().remove("glasses").remove("case").remove("at").remove("folded").remove("in_case") + .remove("free_mb").remove("volume").remove("temperature").apply() + return old + } + + private fun encode(s: PairSeen): String = JSONObject().apply { + s.glasses?.let { put("glasses", it) } + s.case?.let { put("case", it) } + if (s.caseCharging) put("case_charging", true) + put("at", s.at) + put("folded", s.folded) + put("in_case", s.inCase) + s.freeMb?.let { put("free_mb", it) } + s.volume?.let { put("volume", it) } + s.temperatureC?.let { put("temperature", it) } + s.name?.let { put("name", it) } + if (s.nameToSend) put("name_to_send", true) + }.toString() + + private fun decode(text: String): PairSeen? = runCatching { + val o = JSONObject(text) + PairSeen( + glasses = o.optInt("glasses", -1).takeIf { it >= 0 }, + case = o.optInt("case", -1).takeIf { it >= 0 }, + caseCharging = o.optBoolean("case_charging"), + at = o.optLong("at", 0), + folded = o.optBoolean("folded"), + inCase = o.optBoolean("in_case"), + freeMb = o.optLong("free_mb", -1).takeIf { it >= 0 }, + volume = o.optInt("volume", -1).takeIf { it in 0..100 }, + temperatureC = o.optDouble("temperature").takeIf { !it.isNaN() }, + name = o.optString("name").takeIf { it.isNotEmpty() }, + nameToSend = o.optBoolean("name_to_send"), + ) + }.getOrNull() + + companion object { + const val NAME_KEY = "device_custom_name" + const val LEFTOVER_NAME = "VisorTest" + } } diff --git a/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt b/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt index 51eb5f3a..b0e89360 100644 --- a/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt +++ b/app/src/main/kotlin/com/macedonga/visor/language/LanguageInstaller.kt @@ -1,5 +1,6 @@ package com.macedonga.visor.language +import com.macedonga.visor.core.identity.token.withToken import android.content.Context import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.identity.language.LANGUAGE_FEATURE @@ -11,26 +12,23 @@ import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.core.settings.GlassesSettings import com.macedonga.visor.log.EventLog import com.macedonga.visor.login.MetaLogin +import com.macedonga.visor.pairs.pairPrefs import dagger.hilt.android.qualifiers.ApplicationContext import java.io.File -import java.io.FileOutputStream -import java.net.HttpURLConnection -import java.net.URL -import java.security.MessageDigest import javax.inject.Inject import javax.inject.Singleton -import kotlin.coroutines.coroutineContext import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.ensureActive import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext /** Where an install is. [done] and [total] are bytes for the download and the transfer. */ sealed interface LanguageWork { @@ -58,7 +56,9 @@ class LanguageInstaller @Inject constructor( private val log: EventLog, ) { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) - private val prefs = context.getSharedPreferences("languages", Context.MODE_PRIVATE) + // Installed on the glasses: each pair has its own. + @Volatile + private var prefs = pairPrefs(context, "languages", connection.pair.value) private val dir = File(context.filesDir, "packs") private val _work = MutableStateFlow(LanguageWork.Idle) @@ -68,6 +68,15 @@ class LanguageInstaller @Inject constructor( private val _installed = MutableStateFlow(prefs.getStringSet(KEY_INSTALLED, null).orEmpty() + "en_US") val installed: StateFlow> = _installed.asStateFlow() + init { + scope.launch { + connection.pair.filterNotNull().distinctUntilChanged().drop(1).collect { serial -> + prefs = pairPrefs(context, "languages", serial) + _installed.value = prefs.getStringSet(KEY_INSTALLED, null).orEmpty() + "en_US" + } + } + } + /** What Meta has, by pack suffix, and why it could not be asked. */ private val _offers = MutableStateFlow>(emptyMap()) val offers: StateFlow> = _offers.asStateFlow() @@ -105,7 +114,7 @@ class LanguageInstaller @Inject constructor( if (!zip.exists()) { if (dir.usableSpace < offer.size + SPARE_BYTES) error("the phone needs ${mb(offer.size + SPARE_BYTES)} free") log.add("Language: downloading ${offer.resource} ${offer.version}, ${mb(offer.size)}") - download(offer, zip) { done -> _work.value = LanguageWork.Working(language, LanguageWork.Step.DOWNLOADING, done, offer.size) } + downloadPack(offer, zip) { done -> _work.value = LanguageWork.Working(language, LanguageWork.Step.DOWNLOADING, done, offer.size) } } if (connection.state.value.link != LinkState.READY) error("the glasses are not connected") step(language, LanguageWork.Step.SENDING, 0, zip.length()) @@ -154,70 +163,17 @@ class LanguageInstaller @Inject constructor( prefs.edit().putStringSet(KEY_INSTALLED, _installed.value - "en_US").apply() } - private fun ask(packs: List): List { + private fun ask(packs: List): List = offersFor(packs.map { LANGUAGE_FEATURE + it }) + + /** What Meta has for these glasses under the constellation [features], named in full. */ + fun offersFor(features: List): List { val info = connection.state.value.info ?: error("connect the glasses once, so they say which they are") // Meta/hammerhead/greathammerhead:14/UKQ1.250303.001/72387580077200100:user/release-keys val parts = info.build.split('/') val firmware = parts.getOrNull(4)?.substringBefore(':').orEmpty() val type = parts.getOrNull(4)?.substringAfter(':', "user").orEmpty() if (firmware.isEmpty()) error("the glasses' build is unknown") - val token = login.source.get() - return fetchPackOffers(token, info.serial, firmware, packs.map { LANGUAGE_FEATURE + it }, "ota.${parts.getOrElse(1) { "hammerhead" }}.$type") - } - - /** The zip, resumed where an earlier try stopped, checked against Meta's SHA-256. */ - private suspend fun download(offer: PackOffer, file: File, onProgress: (Long) -> Unit) = withContext(Dispatchers.IO) { - val part = File(file.path + ".part") - var have = if (part.exists()) part.length() else 0L - val c = URL(offer.url).openConnection() as HttpURLConnection - try { - c.connectTimeout = 30_000 - c.readTimeout = 60_000 - if (have > 0) c.setRequestProperty("Range", "bytes=$have-") - when (c.responseCode) { - 206 -> Unit - 200 -> have = 0 - else -> error("the download answered ${c.responseCode}") - } - FileOutputStream(part, have > 0).use { out -> - c.inputStream.use { input -> - val buffer = ByteArray(256 * 1024) - var lastReport = 0L - while (true) { - coroutineContext.ensureActive() - val n = input.read(buffer) - if (n < 0) break - out.write(buffer, 0, n) - have += n - if (have - lastReport > 1_000_000) { - lastReport = have - onProgress(have) - } - } - } - } - } finally { - c.disconnect() - } - onProgress(have) - if (offer.sha256.isNotEmpty() && sha256(part) != offer.sha256.lowercase()) { - part.delete() - error("the download did not match Meta's checksum; try again") - } - check(part.renameTo(file)) { "could not keep the download" } - } - - private fun sha256(file: File): String { - val digest = MessageDigest.getInstance("SHA-256") - file.inputStream().use { input -> - val buffer = ByteArray(1024 * 1024) - while (true) { - val n = input.read(buffer) - if (n < 0) break - digest.update(buffer, 0, n) - } - } - return digest.digest().joinToString("") { "%02x".format(it) } + return login.source.withToken { token -> fetchPackOffers(token, info.serial, firmware, features, "ota.${parts.getOrElse(1) { "hammerhead" }}.$type") } } private companion object { diff --git a/app/src/main/kotlin/com/macedonga/visor/language/PackDownload.kt b/app/src/main/kotlin/com/macedonga/visor/language/PackDownload.kt new file mode 100644 index 00000000..5172b198 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/language/PackDownload.kt @@ -0,0 +1,67 @@ +package com.macedonga.visor.language + +import com.macedonga.visor.core.identity.language.PackOffer +import java.io.File +import java.io.FileOutputStream +import java.net.HttpURLConnection +import java.net.URL +import java.security.MessageDigest +import kotlin.coroutines.coroutineContext +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.withContext + +/** The zip, resumed where an earlier try stopped, checked against Meta's SHA-256. */ +internal suspend fun downloadPack(offer: PackOffer, file: File, onProgress: (Long) -> Unit) = withContext(Dispatchers.IO) { + val part = File(file.path + ".part") + var have = if (part.exists()) part.length() else 0L + val c = URL(offer.url).openConnection() as HttpURLConnection + try { + c.connectTimeout = 30_000 + c.readTimeout = 60_000 + if (have > 0) c.setRequestProperty("Range", "bytes=$have-") + when (c.responseCode) { + 206 -> Unit + 200 -> have = 0 + else -> error("the download answered ${c.responseCode}") + } + FileOutputStream(part, have > 0).use { out -> + c.inputStream.use { input -> + val buffer = ByteArray(256 * 1024) + var lastReport = 0L + while (true) { + coroutineContext.ensureActive() + val n = input.read(buffer) + if (n < 0) break + out.write(buffer, 0, n) + have += n + if (have - lastReport > 1_000_000) { + lastReport = have + onProgress(have) + } + } + } + } + } finally { + c.disconnect() + } + onProgress(have) + if (offer.sha256.isNotEmpty() && sha256(part) != offer.sha256.lowercase()) { + part.delete() + error("the download did not match Meta's checksum; try again") + } + check(part.renameTo(file)) { "could not keep the download" } +} + +private fun sha256(file: File): String { + val digest = MessageDigest.getInstance("SHA-256") + file.inputStream().use { input -> + val buffer = ByteArray(1024 * 1024) + while (true) { + val n = input.read(buffer) + if (n < 0) break + digest.update(buffer, 0, n) + } + } + return digest.digest().joinToString("") { "%02x".format(it) } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/log/EventLog.kt b/app/src/main/kotlin/com/macedonga/visor/log/EventLog.kt index 65d6dc3f..f1b20ffd 100644 --- a/app/src/main/kotlin/com/macedonga/visor/log/EventLog.kt +++ b/app/src/main/kotlin/com/macedonga/visor/log/EventLog.kt @@ -8,6 +8,7 @@ import com.macedonga.visor.core.assistant.Assistant import com.macedonga.visor.core.assistant.Hearing import com.macedonga.visor.core.assistant.Timers import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.connection.GlassesWifi import com.macedonga.visor.core.connection.GlassesState import com.macedonga.visor.core.media.LiveStream import com.macedonga.visor.core.media.MediaImport @@ -37,6 +38,7 @@ class EventLog @Inject constructor( keeper: LeKeeper, media: MediaImport, settings: GlassesSettings, + wifi: GlassesWifi, live: LiveStream, assistant: Assistant, tools: com.macedonga.visor.core.assistant.mcp.McpHub, @@ -64,7 +66,9 @@ class EventLog @Inject constructor( } scope.launch { connection.serviceData.collect { data -> - // The assistant says what its svc15 traffic means itself. + // The assistant says what its svc15 traffic means itself; the internet tunnel and the data fetcher go to + // the Requests page, a packet or a request at a time being far too many for this log. + if (data.serviceId in QUIET) return@collect if (data.serviceId != 15) add("Message from service ${data.serviceId}, type ${data.messageType}, ${data.payload.size} bytes") // Debug builds also print the bytes, so a service Visor does not answer can be read over adb. if (BuildConfig.DEBUG && data.serviceId !in setOf(15, 20)) { @@ -81,6 +85,9 @@ class EventLog @Inject constructor( scope.launch { settings.notes.collect { add("Settings: $it") } } + scope.launch { + wifi.notes.collect { add("Wi-Fi networks: $it") } + } scope.launch { live.notes.collect { add("Live: $it") } } @@ -147,6 +154,7 @@ class EventLog @Inject constructor( private companion object { const val LIMIT = 1000 + val QUIET = setOf(com.macedonga.visor.connection.DataFetcher.SVC, com.macedonga.visor.connection.PhoneInternet.SVC) val TIME: DateTimeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS") } } diff --git a/app/src/main/kotlin/com/macedonga/visor/log/EventLogScreen.kt b/app/src/main/kotlin/com/macedonga/visor/log/EventLogScreen.kt index 84983823..196ac763 100644 --- a/app/src/main/kotlin/com/macedonga/visor/log/EventLogScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/log/EventLogScreen.kt @@ -61,7 +61,7 @@ class EventLogViewModel @Inject constructor(val log: EventLog, val media: MediaI /** The log as one card of lines, opened at the end: what matters is what just happened. */ @Composable -fun EventLogScreen(model: EventLogViewModel = hiltViewModel()) { +fun EventLogScreen(onRequests: () -> Unit = {}, model: EventLogViewModel = hiltViewModel()) { val context = LocalContext.current val lines by model.log.lines.collectAsStateWithLifecycle() val list = rememberLazyListState() @@ -79,6 +79,7 @@ fun EventLogScreen(model: EventLogViewModel = hiltViewModel()) { light = 0.3f, list = list, actions = { + RoundIcon(VisorIcons.Globe, stringResource(R.string.requests_title), onRequests) RoundIcon(VisorIcons.Download, stringResource(R.string.event_log_save), { create.launch("visor-log-${LocalDateTime.now().format(FILE_TIME)}.txt") }) RoundIcon(VisorIcons.Share, stringResource(R.string.event_log_share), { share(context, model.log.text()) }) RoundIcon(VisorIcons.Trash, stringResource(R.string.event_log_clear), model.log::clear) diff --git a/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt new file mode 100644 index 00000000..a44579b4 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/log/RequestLogScreen.kt @@ -0,0 +1,119 @@ +package com.macedonga.visor.log + +import android.content.ClipData +import android.content.ClipboardManager +import android.content.Context +import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.itemsIndexed +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.RectangleShape +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.macedonga.visor.R +import com.macedonga.visor.connection.GlassesRequest +import com.macedonga.visor.connection.RequestLog +import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.gallery.RoundIcon +import com.macedonga.visor.settings.Footnote +import com.macedonga.visor.settings.Hairline +import com.macedonga.visor.settings.LocalHost +import com.macedonga.visor.settings.MONO +import com.macedonga.visor.settings.SettingsPage +import com.macedonga.visor.ui.VisorIcons +import dagger.hilt.android.lifecycle.HiltViewModel +import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter +import javax.inject.Inject + +@HiltViewModel +class RequestLogViewModel @Inject constructor(val log: RequestLog) : ViewModel() + +/** What the glasses' apps asked the phone to fetch, newest first; a tap copies the whole link. */ +@Composable +fun RequestLogScreen(model: RequestLogViewModel = hiltViewModel()) { + val items by model.log.items.collectAsStateWithLifecycle() + val shown = items.asReversed() + SettingsPage( + stringResource(R.string.requests_title), + light = 0.3f, + actions = { RoundIcon(VisorIcons.Trash, stringResource(R.string.event_log_clear), model.log::clear) }, + ) { + if (shown.isEmpty()) item(key = "empty") { Footnote(stringResource(R.string.requests_empty)) } + itemsIndexed(shown, key = { _, r -> r.id }) { i, r -> + val shape = when { + shown.size == 1 -> RoundedCornerShape(18.dp) + i == 0 -> RoundedCornerShape(topStart = 18.dp, topEnd = 18.dp) + i == shown.lastIndex -> RoundedCornerShape(bottomStart = 18.dp, bottomEnd = 18.dp) + else -> RectangleShape + } + Column(Modifier.padding(horizontal = 16.dp).fillMaxWidth().clip(shape).background(Visor.colors.raised)) { + if (i > 0) Hairline() + RequestRow(r) + } + } + } +} + +@Composable +private fun RequestRow(r: GlassesRequest) { + val colors = Visor.colors + val context = LocalContext.current + val host = LocalHost.current + val copied = stringResource(R.string.requests_copied) + Column( + Modifier.fillMaxWidth().clickable { + context.getSystemService(ClipboardManager::class.java)?.setPrimaryClip(ClipData.newPlainText("url", r.url)) + host.notice.show(copied) + }.padding(horizontal = 16.dp, vertical = 10.dp), + ) { + Text(place(r.url), style = Visor.type.label, color = colors.textPrimary, maxLines = 2, overflow = TextOverflow.Ellipsis) + val outcome = r.outcome ?: stringResource(R.string.requests_running) + val parts = listOfNotNull( + TIME.format(Instant.ofEpochMilli(r.at).atZone(ZoneId.systemDefault())), + r.times.takeIf { it > 1 }?.let { "×$it" }, + outcome, + r.bytes.takeIf { it > 0 }?.let(::size), + r.millis.takeIf { r.outcome != null }?.let { "$it ms" }, + when (r.source) { + -1 -> stringResource(R.string.requests_tunnel) + -2 -> stringResource(R.string.requests_meta) + -3 -> stringResource(R.string.requests_location) + -4 -> stringResource(R.string.requests_sms) + -5 -> stringResource(R.string.requests_calls) + -6 -> stringResource(R.string.navigation_requests_maps) + else -> stringResource(R.string.requests_source, r.source) + }, + r.options.ifEmpty { null }, + ) + Text(parts.joinToString(" · "), style = MONO, color = if (r.failed) colors.signal else colors.textTertiary, modifier = Modifier.padding(top = 2.dp)) + } +} + +/** Host and path; the query, often a long signature, is left to the copied link. */ +private fun place(url: String): String { + val noQuery = url.substringBefore('?') + return noQuery.substringAfter("://").ifEmpty { url }.let { if (url.startsWith("http")) it else noQuery } +} + +private fun size(bytes: Long): String = when { + bytes >= 1_000_000 -> "%.1f MB".format(bytes / 1_000_000.0) + bytes >= 1_000 -> "${bytes / 1_000} KB" + else -> "$bytes B" +} + +private val TIME = DateTimeFormatter.ofPattern("HH:mm:ss") diff --git a/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt b/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt index c14319b2..492d22ea 100644 --- a/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt +++ b/app/src/main/kotlin/com/macedonga/visor/login/MetaLogin.kt @@ -11,7 +11,11 @@ import com.macedonga.visor.core.identity.UrlConnectionPost import com.macedonga.visor.core.identity.bloks.BloksClient import com.macedonga.visor.core.identity.bloks.BloksHost import com.macedonga.visor.core.identity.bloks.BloksSession +import com.macedonga.visor.core.identity.pyUrlencode import com.macedonga.visor.core.identity.token.AuthorizeRequest +import com.macedonga.visor.core.identity.token.SessionEnded +import com.macedonga.visor.core.identity.token.Token +import com.macedonga.visor.core.identity.token.withToken import com.macedonga.visor.core.identity.token.WebLoginSource import com.macedonga.visor.log.EventLog import dagger.hilt.android.qualifiers.ApplicationContext @@ -27,6 +31,7 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch +import org.json.JSONObject /** The Meta login, for as long as the process lives. */ @Singleton @@ -69,6 +74,25 @@ class MetaLogin @Inject constructor( log.add("Login: opening Meta's login page") } + /** One query on Meta's ar GraphQL as the user, its answer as text. A token Meta has dropped is replaced, once. */ + fun graphql(name: String, doc: String, variables: String): String { + val body = pyUrlencode( + linkedMapOf( + "fb_api_req_friendly_name" to name, "client_doc_id" to doc, "variables" to variables, + "method" to "post", "locale" to "en_US", "pretty" to "false", "format" to "json", + ), + ).toByteArray() + return source.withToken { token -> + post(token, body).also { answer -> + // OAuthException 190: the session behind the token has ended. + if (runCatching { JSONObject(answer).optJSONObject("error")?.optInt("code") }.getOrNull() == 190) throw SessionEnded() + } + } + } + + private fun post(token: Token, body: ByteArray): String = + String(UrlConnectionPost.post(GRAPH_URL, mapOf("Authorization" to "OAuth ${token.value}", "Content-Type" to "application/x-www-form-urlencoded"), body).body) + /** What the login page saw: hosts, paths and statuses, never a query string. */ fun note(text: String) = log.add("Login page: $text") @@ -172,13 +196,14 @@ class MetaLogin @Inject constructor( companion object { private const val TRACE_LIMIT = 4L * 1024 * 1024 + private const val GRAPH_URL = "https://ar-genai.graph.meta.com/graphql" /** Every Bloks call of the last native login, shared with the event log. */ fun traceFile(context: Context): File = File(context.filesDir, "bloks-trace.log") } /** One id per install, as the Meta app keeps its `device_id`: the login's server state is tied to it. */ - private val deviceId: String by lazy { + val deviceId: String by lazy { val prefs = context.getSharedPreferences("meta_login", Context.MODE_PRIVATE) prefs.getString("device_id", null) ?: UUID.randomUUID().toString().also { prefs.edit().putString("device_id", it).apply() } } diff --git a/app/src/main/kotlin/com/macedonga/visor/media/MirrorDecoder.kt b/app/src/main/kotlin/com/macedonga/visor/media/MirrorDecoder.kt new file mode 100644 index 00000000..abad6971 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/media/MirrorDecoder.kt @@ -0,0 +1,61 @@ +package com.macedonga.visor.media + +import android.media.MediaCodec +import android.media.MediaFormat +import android.view.Surface +import com.macedonga.visor.core.media.ScreencastFrame + +/** H.264 decoding of the Display's screen onto a Surface. One thread only; starts on a parameter set or key frame. */ +class MirrorDecoder(private val surface: Surface) { + private var codec: MediaCodec? = null + private var lastPtsUs = -1L + private val info = MediaCodec.BufferInfo() + + fun feed(frame: ScreencastFrame) { + val c = codec ?: (if (frame.key || frame.config) start() else null) ?: return + try { + val i = c.dequeueInputBuffer(INPUT_WAIT_US) + if (i >= 0) { + c.getInputBuffer(i)!!.apply { clear(); put(frame.data) } + lastPtsUs = maxOf(frame.timestampMs * 1000, lastPtsUs + 1) + val flags = if (frame.config) MediaCodec.BUFFER_FLAG_CODEC_CONFIG else 0 + c.queueInputBuffer(i, 0, frame.data.size, lastPtsUs, flags) + } + drain(c) + } catch (e: RuntimeException) { + // A codec error: the next key frame starts a new decoder. + release() + } + } + + fun release() { + codec?.let { runCatching { it.stop(); it.release() } } + codec = null + } + + private fun start(): MediaCodec? = try { + MediaCodec.createDecoderByType(MIME).apply { + configure(MediaFormat.createVideoFormat(MIME, MAX_W, MAX_H), surface, null, 0) + setVideoScalingMode(MediaCodec.VIDEO_SCALING_MODE_SCALE_TO_FIT) + start() + }.also { codec = it } + } catch (e: Exception) { + null + } + + private fun drain(c: MediaCodec) { + while (true) { + val o = c.dequeueOutputBuffer(info, 0) + if (o == MediaCodec.INFO_TRY_AGAIN_LATER) return + if (o >= 0) c.releaseOutputBuffer(o, true) + } + } + + private companion object { + const val MIME = "video/avc" + // The size the Display picks is read from the stream's parameter sets; this is only the most it may pick. + const val MAX_W = 1920 + const val MAX_H = 1080 + const val INPUT_WAIT_US = 10_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/media/MirrorScreen.kt b/app/src/main/kotlin/com/macedonga/visor/media/MirrorScreen.kt new file mode 100644 index 00000000..b72f8e38 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/media/MirrorScreen.kt @@ -0,0 +1,134 @@ +package com.macedonga.visor.media + +import android.view.Surface +import android.view.SurfaceHolder +import android.view.SurfaceView +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.compose.ui.viewinterop.AndroidView +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.lifecycle.viewModelScope +import com.macedonga.visor.R +import com.macedonga.visor.core.design.Visor +import com.macedonga.visor.core.media.Screencast +import com.macedonga.visor.core.media.ScreencastFault +import com.macedonga.visor.core.media.ScreencastPhase +import com.macedonga.visor.core.media.ScreencastState +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.settings.ActionPill +import com.macedonga.visor.settings.SettingsPage +import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.launch + +@HiltViewModel +class MirrorModel @Inject constructor(val screencast: Screencast, private val log: EventLog) : ViewModel() { + // The decoder is touched on this one thread only. + private val decoding = CoroutineScope(SupervisorJob() + Dispatchers.Default.limitedParallelism(1)) + private var decoder: MirrorDecoder? = null + + private val feed = decoding.launch { screencast.frames.collect { decoder?.feed(it) } } + + init { + viewModelScope.launch { + screencast.state.collect { s -> + when (s.phase) { + ScreencastPhase.FAILED -> log.add("Mirror failed: ${s.fault}") + ScreencastPhase.LIVE -> log.add("Mirror live") + else -> Unit + } + } + } + } + + /** The surface the picture goes to, or null when it is gone. */ + fun attach(surface: Surface?) { + decoding.launch { + decoder?.release() + decoder = surface?.let { MirrorDecoder(it) } + } + } + + override fun onCleared() { + decoding.launch { decoder?.release(); decoder = null } + feed.cancel() + super.onCleared() + } +} + +/** The Display's screen, on the phone. Closing the page stops the stream. */ +@Composable +fun MirrorScreen(model: MirrorModel = hiltViewModel()) { + val state by model.screencast.state.collectAsStateWithLifecycle() + val active = state.phase == ScreencastPhase.STARTING || state.phase == ScreencastPhase.LIVE + DisposableEffect(model) { onDispose { model.screencast.stop() } } + + SettingsPage(title = stringResource(R.string.mirror_title)) { + item(key = "picture") { + AndroidView( + modifier = Modifier.fillMaxWidth().aspectRatio(1f).background(Color.Black), + factory = { context -> + SurfaceView(context).apply { + holder.addCallback(object : SurfaceHolder.Callback { + override fun surfaceCreated(holder: SurfaceHolder) = model.attach(holder.surface) + override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) = Unit + override fun surfaceDestroyed(holder: SurfaceHolder) = model.attach(null) + }) + } + }, + ) + } + item(key = "controls") { + Row(Modifier.padding(horizontal = 16.dp, vertical = 12.dp), horizontalArrangement = Arrangement.spacedBy(8.dp)) { + ActionPill(stringResource(R.string.mirror_start), onClick = { model.screencast.start() }, enabled = !active) + ActionPill(stringResource(R.string.mirror_stop), onClick = { model.screencast.stop() }, quiet = true, enabled = active) + } + } + item(key = "state") { + Text( + phraseOf(state), + style = Visor.type.label, + color = Visor.colors.textSecondary, + modifier = Modifier.padding(horizontal = 16.dp), + ) + } + } +} + +@Composable +private fun phraseOf(s: ScreencastState): String = when (s.phase) { + ScreencastPhase.IDLE -> stringResource(R.string.mirror_idle) + ScreencastPhase.STARTING -> stringResource(R.string.mirror_starting) + ScreencastPhase.LIVE -> stringResource(R.string.mirror_live) + ScreencastPhase.FAILED -> when (s.fault) { + ScreencastFault.NOT_CONNECTED -> stringResource(R.string.mirror_not_connected) + ScreencastFault.NOT_READ -> stringResource(R.string.mirror_not_read) + ScreencastFault.NO_FLAG -> stringResource(R.string.mirror_no_flag) + ScreencastFault.FLAG_NOT_SENT -> stringResource(R.string.mirror_flag_not_sent, s.detail) + ScreencastFault.NEEDS_RESTART -> stringResource(R.string.mirror_needs_restart) + ScreencastFault.NO_WIFI -> stringResource(R.string.mirror_no_wifi, s.detail) + ScreencastFault.NO_ANSWER -> stringResource(R.string.mirror_no_answer) + ScreencastFault.ASLEEP -> stringResource(R.string.mirror_asleep) + ScreencastFault.GLASSES_TIMEOUT -> stringResource(R.string.mirror_timeout) + ScreencastFault.UNKNOWN, null -> stringResource(R.string.mirror_unknown) + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/GlassesNotification.kt b/app/src/main/kotlin/com/macedonga/visor/notify/GlassesNotification.kt index 28d24770..c94340a3 100644 --- a/app/src/main/kotlin/com/macedonga/visor/notify/GlassesNotification.kt +++ b/app/src/main/kotlin/com/macedonga/visor/notify/GlassesNotification.kt @@ -28,13 +28,14 @@ data class GlassesNotification( val flags: Int = 0, val category: String? = null, val channel: String? = null, + // The uuid this notification carries, which a removal names. + val uuid: String = UUID.randomUUID().toString(), ) { /** svc46 type 1: a single ChunkEnvelope holding the NotificationData. */ fun envelope(): ByteArray = Wire.fieldBytes(1, UUID.randomUUID().toString().toByteArray()) + Wire.fieldVarint(2, 0) + Wire.fieldVarint(3, 1) + Wire.fieldBytes(4, data()) fun data(): ByteArray { - val uuid = UUID.randomUUID().toString() val out = ByteArrayOutputStream() fun str(field: Int, s: String) { if (s.isNotEmpty()) out.write(Wire.fieldBytes(field, s.toByteArray())) @@ -57,6 +58,8 @@ data class GlassesNotification( val foa = ByteArrayOutputStream().apply { if (isGroup && groupName.isNotEmpty()) write(Wire.fieldBytes(1, groupName.toByteArray())) write(Wire.fieldBytes(2, threadId.ifEmpty { key }.toByteArray())) + // Explicit TEXT, so the Display offers smart replies: an absent type reads as none. + write(Wire.fieldVarint(9, 0)) if (isGroup) write(Wire.fieldVarint(10, 1)) if (provider == 2) write(Wire.fieldVarint(11, 1)) if (senderId.isNotEmpty()) write(Wire.fieldBytes(12, senderId.toByteArray())) @@ -67,14 +70,16 @@ data class GlassesNotification( }.toByteArray() out.write(Wire.fieldBytes(12, foa)) } - // Native and SMS: NativeMessageNotificationExtras {1 phone number, 7 threadId, 8 threadName, 12 senderName}. + // Native and SMS: NativeMessageNotificationExtras {1 phone number, 7 threadId, 8 threadName, 11 senderID, + // 12 senderName, 16 shared contact}. The Display's Messages app files it by threadId and senderID. 3, 7 -> out.write( Wire.fieldBytes( 13, (if (senderId.isNotEmpty()) Wire.fieldBytes(1, senderId.toByteArray()) else ByteArray(0)) + Wire.fieldBytes(7, threadId.ifEmpty { key }.toByteArray()) + Wire.fieldBytes(8, (if (isGroup) groupName.ifEmpty { title } else sender.ifEmpty { title }).toByteArray()) + - Wire.fieldBytes(12, sender.ifEmpty { title }.toByteArray()), + (if (senderId.isNotEmpty()) Wire.fieldBytes(11, senderId.toByteArray()) else ByteArray(0)) + + Wire.fieldBytes(12, sender.ifEmpty { title }.toByteArray()) + Wire.fieldVarint(16, 1), ), ) } @@ -97,6 +102,7 @@ data class GlassesNotification( companion object { const val SVC = 46 const val MT_UPSERT_CHUNK = 1 + const val MT_REMOVE = 1022 const val TYPE_MIRRORED = 14 } } diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt b/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt index e5af177f..73567a4c 100644 --- a/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt +++ b/app/src/main/kotlin/com/macedonga/visor/notify/MirrorListener.kt @@ -1,14 +1,18 @@ package com.macedonga.visor.notify +import android.Manifest import android.app.Notification import android.app.NotificationManager import android.app.Person import android.content.ComponentName +import android.content.pm.PackageManager import android.graphics.Bitmap import android.graphics.Canvas import android.media.session.MediaController import android.media.session.MediaSessionManager import android.media.session.PlaybackState +import android.net.Uri +import android.provider.ContactsContract.CommonDataKinds.Phone import android.provider.Telephony import android.service.notification.NotificationListenerService import android.service.notification.StatusBarNotification @@ -23,6 +27,7 @@ import kotlinx.coroutines.delay import com.macedonga.visor.core.assistant.Conversation import com.macedonga.visor.core.assistant.Messaging import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.protocol.Wire import com.macedonga.visor.core.session.LinkState import com.macedonga.visor.core.settings.GlassesSettings import com.macedonga.visor.settings.hasDisplay @@ -60,6 +65,12 @@ class MirrorListener : NotificationListenerService(), Messaging { @Inject lateinit var apps: NotifyApps + @Inject + lateinit var textReplies: TextReplies + + @Inject + lateinit var recent: RecentChats + // Each conversation sent to the glasses, with the app's reply action, newest last. private val replyable = LinkedHashMap>() @@ -106,6 +117,9 @@ class MirrorListener : NotificationListenerService(), Messaging { val chatProvider = PROVIDERS[sbn.packageName] ?: PROVIDER_WHATSAPP.takeIf { !display && chat(sbn.notification) } if (chatProvider == null && !display) return apps.saw(sbn.packageName) + // The Messages app's conversations are kept whether or not the glasses listen: the Display's list reads them. + val sms = sbn.packageName == Telephony.Sms.getDefaultSmsPackage(this) + if (sms) remember(sbn) if (!choice.enabled.value || !ready() || sbn.packageName in apps.muted.value) return val n = sbn.notification val extras = n.extras @@ -113,9 +127,10 @@ class MirrorListener : NotificationListenerService(), Messaging { val text = (extras.getCharSequence(Notification.EXTRA_BIG_TEXT) ?: extras.getCharSequence(Notification.EXTRA_TEXT))?.toString().orEmpty() val appName = runCatching { packageManager.getApplicationLabel(packageManager.getApplicationInfo(sbn.packageName, 0)).toString() }.getOrDefault(sbn.packageName) val r = Ranking() - chatProvider?.let { provider -> + // The phone's texts, SMS or RCS, go as native messages, which the glasses' Messages app files by thread. + (if (sms) PROVIDER_NATIVE else chatProvider)?.let { provider -> message(sbn, provider, appName)?.let { m -> - deliver(sbn.key, m.envelope()) + deliver(m) replyAction(n)?.let { action -> replyable.remove(sbn.key) replyable[sbn.key] = Conversation(sbn.key, m.sender, appName) to action @@ -140,28 +155,47 @@ class MirrorListener : NotificationListenerService(), Messaging { category = n.category, channel = n.channelId, ) - deliver(sbn.key, note.envelope()) + deliver(note) } // What came while the wearer was talking with the assistant, newest per notification, and since when. - private val held = LinkedHashMap() + private val held = LinkedHashMap() private var heldSince = 0L + // The provider and uuid of the last notification sent for each key, which a removal names. + private val mirrored = LinkedHashMap>() + /** To the glasses now, or once the conversation with the assistant is over. */ - private fun deliver(key: String, envelope: ByteArray) { + private fun deliver(note: GlassesNotification) { if (talking()) { synchronized(held) { if (held.isEmpty()) heldSince = System.currentTimeMillis() - held.remove(key) - held[key] = envelope + held.remove(note.key) + held[note.key] = note } return } - send(envelope) + send(note) } - private fun send(envelope: ByteArray) = + private fun send(note: GlassesNotification) { + mirrored.remove(note.key) + mirrored[note.key] = note.provider to note.uuid + while (mirrored.size > 300) mirrored.remove(mirrored.keys.first()) + val envelope = note.envelope() connection.submit { it.sendOnService(GlassesNotification.SVC, GlassesNotification.MT_UPSERT_CHUNK, envelope) } + } + + /** A notification Visor sent is gone from the phone: the glasses take it away too. */ + override fun onNotificationRemoved(sbn: StatusBarNotification) { + // One held and not yet sent goes no further; one already on the glasses is removed. + synchronized(held) { held.remove(sbn.key) } + textReplies.remove(sbn.key) + val (provider, uuid) = mirrored.remove(sbn.key) ?: return + if (!display()) return + val request = Wire.fieldBytes(1, sbn.key.toByteArray()) + Wire.fieldVarint(2, provider.toLong()) + Wire.fieldBytes(6, uuid.toByteArray()) + connection.submit { it.sendOnService(GlassesNotification.SVC, GlassesNotification.MT_REMOVE, request) } + } /** The glasses listening, thinking or answering, Visor's model at work, or a Live AI session. */ private fun talking(): Boolean = @@ -210,9 +244,15 @@ class MirrorListener : NotificationListenerService(), Messaging { sentAt[sbn.key] = time while (sentAt.size > 300) sentAt.remove(sentAt.keys.first()) val group = extras.getBoolean(Notification.EXTRA_IS_GROUP_CONVERSATION) + val native = provider == PROVIDER_NATIVE + // A text names its sender by number and its thread as the SMS store does, as svc70 lists them. + val number = if (native) numberOf(person, sender) else null + // The thread the glasses answer with (FoA 2) names the notification's reply action, for every provider. + val threadId = threadOf(sbn, number, group) + replyAction(sbn.notification)?.let { textReplies.put(threadId, sbn.key, it) } return GlassesNotification( key = sbn.key, - pkg = if (sbn.packageName in PROVIDERS) sbn.packageName else WHATSAPP, + pkg = APP_IDS[provider] ?: sbn.packageName, appName = appName, title = sender, text = text, @@ -220,20 +260,64 @@ class MirrorListener : NotificationListenerService(), Messaging { type = TYPE_INCOMING_MESSAGE, provider = provider, sender = sender, - senderId = person?.key ?: person?.uri?.substringAfter(':').orEmpty(), - threadId = sbn.notification.shortcutId ?: sbn.key, + senderId = number ?: person?.key ?: person?.uri?.substringAfter(':').orEmpty(), + threadId = threadId, isGroup = group, groupName = extras.getCharSequence(Notification.EXTRA_CONVERSATION_TITLE)?.toString().orEmpty(), ) } + /** The other side's number: the person's tel: uri, else the sender's number or contact. */ + private fun numberOf(person: android.app.Person?, sender: String): String? = + person?.uri?.takeIf { it.startsWith("tel:") }?.let { Uri.decode(it.substringAfter("tel:")) } ?: numberOf(sender) + + /** The SMS store's thread for a one-to-one text, else the glasses' key for the conversation. */ + private fun threadOf(sbn: StatusBarNotification, number: String?, group: Boolean): String = + number?.takeIf { !group }?.let { runCatching { Telephony.Threads.getOrCreateThreadId(this, it).toString() }.getOrNull() } + ?: sbn.notification.shortcutId ?: sbn.key + /** A message the user sent: no sender, and the user's own person when one is given. */ - private fun fromUser(m: Bundle, user: Person?): Boolean { + private fun fromUser(m: android.os.Bundle, user: Person?): Boolean { @Suppress("DEPRECATION") - val person = m.getParcelable("sender_person") as? Person + val person = m.getParcelable("sender_person") as? android.app.Person return m.getCharSequence("sender") == null && (person == null || user != null && person.key == user.key && person.name == user.name) } + /** Keeps a Messages conversation as its notification shows it, for the Display's list (see RecentChats). */ + private fun remember(sbn: StatusBarNotification) { + val extras = sbn.notification.extras + @Suppress("DEPRECATION") + val user = extras.getParcelable(Notification.EXTRA_MESSAGING_PERSON) as? Person + val group = extras.getBoolean(Notification.EXTRA_IS_GROUP_CONVERSATION) + @Suppress("DEPRECATION") + val bundles = extras.getParcelableArray(Notification.EXTRA_MESSAGES).orEmpty().filterIsInstance() + val numbers = LinkedHashSet() + val messages = bundles.mapNotNull { m -> + val text = m.getCharSequence("text")?.toString()?.takeIf { it.isNotBlank() } ?: return@mapNotNull null + @Suppress("DEPRECATION") + val person = m.getParcelable("sender_person") as? android.app.Person + val outgoing = fromUser(m, user) + val sender = m.getCharSequence("sender")?.toString() ?: person?.name?.toString().orEmpty() + val number = if (outgoing) null else numberOf(person, sender) + number?.let { numbers += it } + RecentChats.Message(text, sender, number.orEmpty(), m.getLong("time").takeIf { it > 0 } ?: sbn.postTime, outgoing) + } + if (messages.isEmpty()) return + val title = extras.getCharSequence(Notification.EXTRA_CONVERSATION_TITLE)?.takeIf { group } ?: extras.getCharSequence(Notification.EXTRA_TITLE) + val id = threadOf(sbn, numbers.singleOrNull(), group) + recent.remember(id, title?.toString().orEmpty(), group, numbers.toList(), messages) + } + + /** A contact's number by their name, or the name itself when it is a number. */ + private fun numberOf(name: String): String? { + if (name.isNotEmpty() && name.all { it.isDigit() || it in "+ -()" }) return name.filter { it.isDigit() || it == '+' } + if (checkSelfPermission(Manifest.permission.READ_CONTACTS) != PackageManager.PERMISSION_GRANTED) return null + return runCatching { + contentResolver.query(Phone.CONTENT_URI, arrayOf(Phone.NUMBER), "${Phone.DISPLAY_NAME} = ?", arrayOf(name), null) + ?.use { if (it.moveToFirst()) it.getString(0) else null } + }.getOrNull() + } + /** A conversation in the chat layout (MessagingStyle), which the glasses' announcer reads as a message. */ @Suppress("DEPRECATION") private fun chat(n: Notification): Boolean = n.extras.getParcelableArray(Notification.EXTRA_MESSAGES)?.isNotEmpty() == true @@ -256,11 +340,11 @@ class MirrorListener : NotificationListenerService(), Messaging { private fun ready() = connection.state.value.link == LinkState.READY + // The Display mirrors the phone's texts and removals too; other glasses get them over Bluetooth already. private fun display() = hasDisplay(connection.state.value.info?.build) // The Meta app's filters: nothing ongoing, no group summaries, nothing empty or below default - // importance, and not the SMS app or the dialer, whose messages and calls reach the glasses by - // Bluetooth already. + // importance, and not the dialer, whose calls reach the glasses by Bluetooth already. private fun wanted(sbn: StatusBarNotification, ranking: RankingMap): Boolean { val n = sbn.notification if (sbn.packageName == packageName || sbn.packageName in SYSTEM) return false @@ -269,8 +353,8 @@ class MirrorListener : NotificationListenerService(), Messaging { if (extras.getCharSequence(Notification.EXTRA_TITLE).isNullOrBlank() && extras.getCharSequence(Notification.EXTRA_TEXT).isNullOrBlank()) return false val r = Ranking() if (ranking.getRanking(sbn.key, r) && r.importance < NotificationManager.IMPORTANCE_DEFAULT) return false - if (sbn.packageName == Telephony.Sms.getDefaultSmsPackage(this)) return false if (sbn.packageName == getSystemService(TelecomManager::class.java)?.defaultDialerPackage) return false + if (sbn.packageName == Telephony.Sms.getDefaultSmsPackage(this) && !display()) return false return true } @@ -297,10 +381,13 @@ class MirrorListener : NotificationListenerService(), Messaging { companion object { private const val PROVIDER_MIRRORED = 8 + private const val PROVIDER_NATIVE = 3 private const val PROVIDER_WHATSAPP = 2 - private const val WHATSAPP = "com.whatsapp" private const val TYPE_INCOMING_MESSAGE = 1 + // The app id the glasses gate on, per provider, as the Meta app sends it. Messenger and Instagram are not their package names. + private val APP_IDS = mapOf(1 to "com.facebook.messenger", 2 to "com.whatsapp", 3 to "com.android.sms", 4 to "com.instagram") + // The chat apps the glasses announce themselves, by their NotificationProvider. private val PROVIDERS = mapOf( "com.whatsapp" to 2, "com.whatsapp.w4b" to 2, diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/NotifyIcons.kt b/app/src/main/kotlin/com/macedonga/visor/notify/NotifyIcons.kt new file mode 100644 index 00000000..f9ba9a3e --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/notify/NotifyIcons.kt @@ -0,0 +1,65 @@ +package com.macedonga.visor.notify + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Canvas +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.DataXConnection +import com.macedonga.visor.core.session.DataXMessage +import com.macedonga.visor.core.session.Responder +import com.macedonga.visor.log.EventLog +import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.ByteArrayOutputStream +import javax.inject.Inject +import javax.inject.Provider +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +/** The glasses' svc46 requests: an app's icon for a notification (type 9), and their acks (types 2 and 1027). */ +class NotifyIcons @Inject constructor( + @ApplicationContext private val context: Context, + // A Provider, because EventLog needs the connection this responder is registered on. + private val log: Provider, +) : Responder { + + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { + when (message.info.messageType) { + MT_ICON_REQUEST -> { + val pkg = runCatching { String(Wire.parse(message.payload).bytes(1)) }.getOrDefault("") + val jpeg = withContext(Dispatchers.Default) { render(pkg) } + val answer = if (jpeg != null) { + Wire.fieldVarint(2, 1) + Wire.fieldBytes(3, jpeg) + } else { + Wire.fieldVarint(2, 0) + Wire.fieldBytes(4, "unknown package".toByteArray()) + } + connection.send(DataXConnection.mirror(message.info.channelId), Wire.fieldBytes(1, pkg.toByteArray()) + answer, messageType = MT_ICON_RESPONSE) + } + MT_RESPONSE, MT_RESPONSE_ACK -> log.get().add("Glasses answered a notification (type ${message.info.messageType})") + } + } + + /** The app's icon as a 78 x 78 JPEG, or null when the package has none. */ + private fun render(pkg: String): ByteArray? = runCatching { + val d = runCatching { context.packageManager.getApplicationIcon(pkg) }.getOrNull() + ?: context.packageManager.getApplicationIcon(installed(pkg) ?: return null) + val bmp = Bitmap.createBitmap(ICON, ICON, Bitmap.Config.ARGB_8888) + d.setBounds(0, 0, ICON, ICON) + d.draw(Canvas(bmp)) + ByteArrayOutputStream().also { bmp.compress(Bitmap.CompressFormat.JPEG, 100, it) }.toByteArray() + }.getOrNull() + + // Chat notifications carry the app ids the glasses gate on, not the phone's packages. + private fun installed(appId: String): String? = when (appId) { + "com.android.sms" -> android.provider.Telephony.Sms.getDefaultSmsPackage(context) + else -> APP_PACKAGES[appId] + } + + private companion object { + val APP_PACKAGES = mapOf("com.facebook.messenger" to "com.facebook.orca", "com.instagram" to "com.instagram.android") + const val MT_RESPONSE = 2 + const val MT_ICON_REQUEST = 9 + const val MT_ICON_RESPONSE = 10 + const val MT_RESPONSE_ACK = 1027 + const val ICON = 78 + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/RecentChats.kt b/app/src/main/kotlin/com/macedonga/visor/notify/RecentChats.kt new file mode 100644 index 00000000..e13757ee --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/notify/RecentChats.kt @@ -0,0 +1,53 @@ +package com.macedonga.visor.notify + +import javax.inject.Inject +import javax.inject.Singleton + +/** + * The SMS app's conversations as its notifications show them, for this process only. Google Messages files RCS chats + * as restricted MMS rows that the phone's texts list cannot read, so the Display's list takes these instead. Each chat + * keeps its newest messages, and the least recently updated chat goes first. + */ +@Singleton +class RecentChats @Inject constructor() { + class Message(val text: String, val sender: String, val number: String, val time: Long, val outgoing: Boolean) + + class Chat(val id: String, val title: String, val group: Boolean, val numbers: List, val messages: List) + + private val chats = LinkedHashMap() + + /** Adds [incoming] (any order) to chat [id], newest first, without repeats. */ + @Synchronized + fun remember(id: String, title: String, group: Boolean, numbers: List, incoming: List) { + val old = chats.remove(id) + val messages = ((old?.messages.orEmpty()) + incoming) + .distinctBy { it.time to it.text } + .sortedByDescending { it.time } + .take(MAX_MESSAGES) + chats[id] = Chat(id, title, group, ((old?.numbers.orEmpty()) + numbers).distinct(), messages) + while (chats.size > MAX_CHATS) chats.remove(chats.keys.first()) + } + + /** The chats, newest message first. */ + @Synchronized + fun chats(): List = chats.values.sortedByDescending { it.messages.firstOrNull()?.time ?: 0L } + + @Synchronized + fun find(id: String): Chat? = chats[id] + + /** The chat with a participant in [numbers], if any. */ + @Synchronized + fun withNumber(numbers: List): Chat? = + chats.values.firstOrNull { c -> c.numbers.any { n -> numbers.any { sameNumber(n, it) } } } + + companion object { + private const val MAX_CHATS = 30 + private const val MAX_MESSAGES = 50 + + /** Two numbers are one line when their last ten digits match. */ + fun sameNumber(a: String, b: String): Boolean { + val x = a.filter(Char::isDigit).takeLast(10) + return x.isNotEmpty() && x == b.filter(Char::isDigit).takeLast(10) + } + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/notify/TextReplies.kt b/app/src/main/kotlin/com/macedonga/visor/notify/TextReplies.kt new file mode 100644 index 00000000..59b5e66e --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/notify/TextReplies.kt @@ -0,0 +1,40 @@ +package com.macedonga.visor.notify + +import android.app.Notification +import android.app.RemoteInput +import android.content.Context +import android.content.Intent +import android.os.Bundle +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Singleton + +/** + * The Messages app's reply actions, by the SMS store's thread each conversation is, while its notification is up. A + * text sent through one goes the way the app sends it, RCS included, which no API offers other apps. + */ +@Singleton +class TextReplies @Inject constructor() { + private val byThread = ConcurrentHashMap>() + + fun put(thread: String, key: String, action: Notification.Action) { + byThread[thread] = key to action + } + + /** The notification [key] is gone, and its reply with it. */ + fun remove(key: String) { + byThread.values.removeIf { it.first == key } + } + + fun has(thread: String) = byThread.containsKey(thread) + + /** Sends [text] in [thread] through the app's reply; null when it went, else why not. */ + fun reply(context: Context, thread: String, text: String): String? { + val action = byThread[thread]?.second ?: return "No reply in that conversation" + val inputs = action.remoteInputs ?: return "The app gives no reply field" + val results = Bundle().apply { inputs.forEach { putCharSequence(it.resultKey, text) } } + val fill = Intent() + RemoteInput.addResultsToIntent(inputs, fill, results) + return runCatching { action.actionIntent.send(context, 0, fill) }.exceptionOrNull()?.let { it.message ?: it.toString() } + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/pairs/PairPrefs.kt b/app/src/main/kotlin/com/macedonga/visor/pairs/PairPrefs.kt new file mode 100644 index 00000000..b9a9a336 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/pairs/PairPrefs.kt @@ -0,0 +1,34 @@ +package com.macedonga.visor.pairs + +import android.content.Context +import android.content.SharedPreferences + +/** + * Visor's own notes about one pair, in a preferences file of its own (`_`). The first pair to ask takes + * what the shared file held from before Visor kept pairs apart; with no pair known yet, the shared file it is. + */ +fun pairPrefs(context: Context, name: String, serial: String?): SharedPreferences { + val shared = context.getSharedPreferences(name, Context.MODE_PRIVATE) + serial ?: return shared + val own = context.getSharedPreferences("${name}_$serial", Context.MODE_PRIVATE) + val old = shared.all + if (old.isNotEmpty()) { + if (own.all.isEmpty()) { + val edit = own.edit() + old.forEach { (key, value) -> + @Suppress("UNCHECKED_CAST") + when (value) { + is Boolean -> edit.putBoolean(key, value) + is Int -> edit.putInt(key, value) + is Long -> edit.putLong(key, value) + is Float -> edit.putFloat(key, value) + is String -> edit.putString(key, value) + is Set<*> -> edit.putStringSet(key, value as Set) + } + } + edit.commit() + } + shared.edit().clear().commit() + } + return own +} diff --git a/app/src/main/kotlin/com/macedonga/visor/pairs/Pairs.kt b/app/src/main/kotlin/com/macedonga/visor/pairs/Pairs.kt new file mode 100644 index 00000000..9362e8b9 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/pairs/Pairs.kt @@ -0,0 +1,140 @@ +package com.macedonga.visor.pairs + +import android.content.Context +import com.macedonga.visor.connection.AndroidTransportFactory +import com.macedonga.visor.connection.GlassesService +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.connection.GlassesLink +import com.macedonga.visor.core.connection.GlassesTarget +import com.macedonga.visor.core.connection.RealSessionHostFactory +import com.macedonga.visor.core.identity.IdentityStore +import com.macedonga.visor.core.media.LiveStream +import com.macedonga.visor.core.media.MediaImport +import com.macedonga.visor.core.transport.LeKeeper +import com.macedonga.visor.home.LastSeen +import com.macedonga.visor.home.PairSeen +import com.macedonga.visor.log.EventLog +import com.macedonga.visor.stage.Looks +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull + +/** A pair set up on this phone. [demo] is one a debug build shows to try the home with: never connected or looked at. */ +data class KnownPair(val serial: String, val address: String, val demo: Boolean = false) + +/** + * The pairs set up on this phone, in the order they were set up. One is connected at a time; the home shows the others + * as they were last heard from, and a pair it shows that is not connected gets a quick look: its own LE link and a + * session that only reads the batteries, the case and the hinge, then lets go. + */ +@Singleton +class Pairs @Inject constructor( + @ApplicationContext private val context: Context, + private val store: IdentityStore, + private val connection: GlassesConnection, + private val live: LiveStream, + private val media: MediaImport, + private val lastSeen: LastSeen, + private val looks: Looks, + private val log: EventLog, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val stored = MutableStateFlow>(emptyList()) + private val demoPair = MutableStateFlow(null) + val all: StateFlow> = combine(stored, demoPair) { pairs, extra -> pairs + listOfNotNull(extra) } + .stateIn(scope, SharingStarted.Eagerly, emptyList()) + + private val _peeking = MutableStateFlow(null) + + /** The pair being looked at right now, if any. */ + val peeking: StateFlow = _peeking.asStateFlow() + + private var peek: Job? = null + private val peekedAt = mutableMapOf() + + fun refresh() { + scope.launch { + val bundles = withContext(Dispatchers.IO) { runCatching { store.all() }.getOrDefault(emptyList()) } + stored.value = bundles.sortedBy { it.createdAt }.map { KnownPair(it.serial, it.glassesAddress) } + bundles.forEach { looks.want(it.serial) } + // Kept in memory and small: the stage needs one for every pair but a Wayfarer, and to tell pairs apart. + looks.wantPictures(bundles.map { it.serial }) + } + } + + /** Connect to [serial] instead of the pair connected now. False while the Live or an import has the glasses. */ + fun connect(serial: String): Boolean { + if (live.running.value || media.importing.value) return false + val pair = all.value.firstOrNull { it.serial == serial }?.takeIf { !it.demo } ?: return false + stopPeek() + GlassesService.start(context, GlassesTarget(GlassesLink.Bluetooth(pair.address), pair.serial)) + return true + } + + /** A quick look at [serial], a pair that is not connected: at most once a minute, never during a Live or an import. */ + fun peek(serial: String) { + if (serial == connection.state.value.target?.serial || serial == connection.pair.value && connection.running.value) return + if (live.running.value || media.importing.value) return + val now = System.currentTimeMillis() + if (now - (peekedAt[serial] ?: 0) < PEEK_EVERY_MS) return + val pair = all.value.firstOrNull { it.serial == serial }?.takeIf { !it.demo } ?: return + peekedAt[serial] = now + peek?.cancel() + peek = scope.launch { + val keeper = LeKeeper(context) + val look = GlassesConnection(store, RealSessionHostFactory(AndroidTransportFactory(context, keeper = keeper)), chooses = false) + _peeking.value = serial + log.add("Pairs: looking at ${pair.serial}") + try { + keeper.hold(pair.address) + // Asleep, they answer on LE only: wait for it a moment before the session picks its link. + withTimeoutOrNull(LE_WAIT_MS) { keeper.connected.first { it } } + look.start(this, GlassesTarget(GlassesLink.Bluetooth(pair.address), pair.serial)) + val seen = withTimeoutOrNull(PEEK_MS) { look.state.first { it.device?.battery != null } } + // The case comes in its own update, when there is one. + if (seen != null) withTimeoutOrNull(CASE_MS) { look.state.first { it.device?.case != null } } + look.state.value.let(lastSeen::record) + log.add(if (seen != null) "Pairs: ${pair.serial} at ${seen.device?.battery?.percent?.toInt()}%" else "Pairs: ${pair.serial} did not answer") + } finally { + // Let go of the socket even when the look was called off: the glasses hold one session. + withContext(NonCancellable) { look.stop() } + keeper.release() + _peeking.value = null + } + } + } + + /** Debug builds: show [pair] beside the real ones, as last heard from with [seen]; null takes it away. */ + fun demo(pair: KnownPair?, seen: PairSeen?) { + demoPair.value?.let { lastSeen.forget(it.serial) } + if (pair != null && seen != null) lastSeen.keep(pair.serial, seen) + demoPair.value = pair?.copy(demo = true) + } + + fun stopPeek() { + peek?.cancel() + peek = null + } + + private companion object { + const val PEEK_EVERY_MS = 60_000L + const val LE_WAIT_MS = 6_000L + const val PEEK_MS = 20_000L + const val CASE_MS = 2_000L + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/phone/Companion.kt b/app/src/main/kotlin/com/macedonga/visor/phone/Companion.kt index 4e1b5413..001eef16 100644 --- a/app/src/main/kotlin/com/macedonga/visor/phone/Companion.kt +++ b/app/src/main/kotlin/com/macedonga/visor/phone/Companion.kt @@ -3,10 +3,12 @@ package com.macedonga.visor.phone import android.companion.AssociationRequest import android.companion.BluetoothDeviceFilter import android.companion.CompanionDeviceManager +import android.companion.ObservingDevicePresenceRequest import android.content.Context import android.content.IntentSender import android.content.pm.PackageManager import android.os.Build +import com.macedonga.visor.log.EventLog /** * The glasses as Visor's companion device, as the Meta app holds them: Android lets a companion app open other apps @@ -52,4 +54,21 @@ fun associateCompanion(context: Context, address: String?, ask: (IntentSender) - else cdm.associate(request, callback, null) } +/** + * Asks Android to tell [com.macedonga.visor.connection.GlassesPresence] when each association's glasses appear. + * Called at every start of the app; repeating it is harmless. + */ +fun observeCompanions(context: Context, log: EventLog) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) return + val cdm = context.getSystemService(CompanionDeviceManager::class.java) ?: return + runCatching { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.BAKLAVA) { + cdm.myAssociations.forEach { cdm.startObservingDevicePresence(ObservingDevicePresenceRequest.Builder().setAssociationId(it.id).build()) } + } else { + @Suppress("DEPRECATION") + cdm.associations.forEach { cdm.startObservingDevicePresence(it) } + } + }.onFailure { log.add("Companion presence not observed: ${it.message}") } +} + private const val ASKED = "asked" diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt new file mode 100644 index 00000000..3a69a7f8 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/settings/AccountsScreen.kt @@ -0,0 +1,194 @@ +package com.macedonga.visor.settings + +import androidx.compose.foundation.layout.padding +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.lifecycle.viewModelScope +import com.macedonga.visor.R +import com.macedonga.visor.connection.AccountLinking +import com.macedonga.visor.connection.LinkOutcome +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.token.LinkedAccountRef +import com.macedonga.visor.core.identity.token.LinkedAccountType +import com.macedonga.visor.core.session.LinkState +import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch + +enum class AccountRow { META, WHATSAPP, FACEBOOK, INSTAGRAM } + +@HiltViewModel +class AccountsModel @Inject constructor( + val connection: GlassesConnection, + private val linking: AccountLinking, + private val log: com.macedonga.visor.log.EventLog, +) : ViewModel() { + + // Null until the glasses answer. + private val _meta = MutableStateFlow(null) + val meta: StateFlow = _meta.asStateFlow() + + private val _busy = MutableStateFlow(null) + val busy: StateFlow = _busy.asStateFlow() + + private val _metaResult = MutableStateFlow(null) + val metaResult: StateFlow = _metaResult.asStateFlow() + + private val _whatsappResult = MutableStateFlow(null) + val whatsappResult: StateFlow = _whatsappResult.asStateFlow() + + private val _linkedResults = MutableStateFlow>(emptyMap()) + val linkedResults: StateFlow> = _linkedResults.asStateFlow() + + // The provider accounts Meta lists for the user, null until read. + private val _accounts = MutableStateFlow?>(null) + val accounts: StateFlow?> = _accounts.asStateFlow() + + fun refresh() { + viewModelScope.launch { linking.metaPresent()?.let { _meta.value = it } } + viewModelScope.launch { linking.linkedAccounts()?.let { list -> _accounts.value = list.associateBy { it.type } } } + } + + fun linkMeta() = task(AccountRow.META) { + _metaResult.value = linking.syncMeta() + if (_metaResult.value == LinkOutcome.LINKED) _meta.value = true + } + + fun linkWhatsApp() = task(AccountRow.WHATSAPP) { + _whatsappResult.value = linking.linkWhatsApp() + } + + fun linkProvider(row: AccountRow, type: LinkedAccountType) = task(row) { + _linkedResults.value = _linkedResults.value + (type to linking.syncLinked(type)) + } + + private fun task(row: AccountRow, work: suspend () -> Unit) { + if (_busy.value != null) return + _busy.value = row + viewModelScope.launch { + try { + work() + } catch (error: kotlinx.coroutines.CancellationException) { + throw error + } catch (error: Exception) { + // A failure the linking did not foresee stays on this page; it must not take the app down. + log.add("Accounts: ${error.javaClass.simpleName}: ${error.message}") + } finally { + _busy.value = null + } + } + } +} + +/** The accounts the Display's apps log in with: the Meta account, and WhatsApp's link. */ +@Composable +fun AccountsScreen(model: AccountsModel = hiltViewModel()) { + val glasses by model.connection.state.collectAsStateWithLifecycle() + val ready = glasses.link == LinkState.READY + val meta by model.meta.collectAsStateWithLifecycle() + val busy by model.busy.collectAsStateWithLifecycle() + val metaResult by model.metaResult.collectAsStateWithLifecycle() + val whatsappResult by model.whatsappResult.collectAsStateWithLifecycle() + val linked by model.linkedResults.collectAsStateWithLifecycle() + val accounts by model.accounts.collectAsStateWithLifecycle() + LaunchedEffect(ready) { if (ready) model.refresh() } + + SettingsPage(stringResource(R.string.accounts_title), light = 0f) { + item(key = "accounts") { + Rows( + { + RowFrame( + stringResource(R.string.accounts_meta), + onClick = null, + sub = metaResult?.let { outcomeWords(it) } ?: when (meta) { + true -> stringResource(R.string.accounts_linked) + false -> stringResource(R.string.accounts_not_linked) + null -> stringResource(R.string.settings_unknown) + }, + locked = !ready, + busy = busy == AccountRow.META, + ) { + if (meta == false) { + ActionPill( + stringResource(R.string.accounts_link), { model.linkMeta() }, + quiet = true, modifier = Modifier.padding(start = 12.dp), enabled = ready && busy == null, + ) + } + } + }, + { + RowFrame( + stringResource(R.string.accounts_whatsapp), + onClick = null, + sub = when { + busy == AccountRow.WHATSAPP -> stringResource(R.string.accounts_scan) + else -> whatsappResult?.let { outcomeWords(it) } + }, + locked = !ready, + busy = busy == AccountRow.WHATSAPP, + ) { + ActionPill( + stringResource(R.string.accounts_link), { model.linkWhatsApp() }, + quiet = true, modifier = Modifier.padding(start = 12.dp), enabled = ready && busy == null, + ) + } + }, + { LinkedRow(model, AccountRow.FACEBOOK, LinkedAccountType.FACEBOOK, R.string.accounts_facebook, ready, busy, linked, accounts) }, + { LinkedRow(model, AccountRow.INSTAGRAM, LinkedAccountType.INSTAGRAM, R.string.accounts_instagram, ready, busy, linked, accounts) }, + ) + } + } +} + +@Composable +private fun LinkedRow( + model: AccountsModel, + row: AccountRow, + type: LinkedAccountType, + label: Int, + ready: Boolean, + busy: AccountRow?, + results: Map, + accounts: Map?, +) { + val account = accounts?.get(type) + RowFrame( + stringResource(label), + onClick = null, + sub = results[type]?.let { outcomeWords(it) } ?: when { + accounts == null -> null + account == null -> stringResource(R.string.accounts_not_linked) + account.forGlasses -> stringResource(R.string.accounts_linked_as, account.name) + else -> stringResource(R.string.accounts_off_for_glasses, account.name) + }, + locked = !ready, + busy = busy == row, + ) { + ActionPill( + stringResource(R.string.accounts_link), { model.linkProvider(row, type) }, + quiet = true, modifier = Modifier.padding(start = 12.dp), enabled = ready && busy == null, + ) + } +} + +@Composable +private fun outcomeWords(outcome: LinkOutcome): String = stringResource( + when (outcome) { + LinkOutcome.LINKED -> R.string.accounts_done + LinkOutcome.LOGGED_OUT -> R.string.accounts_logged_out + LinkOutcome.UNREACHABLE -> R.string.accounts_unreachable + LinkOutcome.NO_ANSWER -> R.string.accounts_no_answer + LinkOutcome.REFUSED -> R.string.accounts_refused + LinkOutcome.NOT_LINKED -> R.string.accounts_not_linked + }, +) diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt new file mode 100644 index 00000000..2e35aad7 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/settings/BandPart.kt @@ -0,0 +1,54 @@ +package com.macedonga.visor.settings + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.ui.res.stringResource +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.macedonga.visor.R +import com.macedonga.visor.core.session.LinkState + +/** The Neural Band as the Display holds it: whether it is paired and connected, its battery and its fit. */ +@Composable +internal fun BandPart(model: SettingsViewModel) { + val glasses by model.connection.state.collectAsStateWithLifecycle() + val paired by model.bandPeers.bands.collectAsStateWithLifecycle() + val ready = glasses.link == LinkState.READY + LaunchedEffect(ready) { if (ready) model.bandPeers.refresh() } + val device = glasses.device + val band = paired?.firstOrNull() + Rows( + { + ValueRow( + stringResource(R.string.band_glasses), + when { + !ready || paired == null -> stringResource(R.string.settings_unknown) + band == null -> stringResource(R.string.band_not_paired) + band.connected -> stringResource(R.string.band_paired_connected) + else -> stringResource(R.string.band_paired_away) + }, + onClick = null, + ) + }, + device?.bandBattery?.let { b -> row { ValueRow(stringResource(R.string.band_battery), batteryWords(b, device.bandCharging), onClick = null) } }, + fitWords(device?.bandFit)?.let { fit -> + row { + ValueRow( + stringResource(R.string.band_fit), fit, onClick = null, + problem = if (device?.bandOrientation == "INCORRECT") stringResource(R.string.band_wrong_way) else null, + ) + } + }, + ) +} + +@Composable +internal fun batteryWords(percent: Int, charging: Boolean?): String = + stringResource(if (charging == true) R.string.glasses_battery_charging else R.string.band_percent, percent) + +@Composable +private fun fitWords(fit: String?): String? = when (fit) { + "LOOSE" -> stringResource(R.string.band_fit_loose) + "TIGHT" -> stringResource(R.string.band_fit_tight) + else -> null +} diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt index bc171750..05f2095a 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/GlassesPage.kt @@ -27,10 +27,16 @@ import androidx.lifecycle.compose.LifecycleResumeEffect import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.lifecycle.viewModelScope import androidx.navigation3.runtime.NavKey +import com.macedonga.visor.AccountsKey import com.macedonga.visor.OnboardingKey +import com.macedonga.visor.WebAppsKey +import com.macedonga.visor.WifiNetworksKey import com.macedonga.visor.R import com.macedonga.visor.connection.GlassesService import com.macedonga.visor.connection.LinkKind +import com.macedonga.visor.firmware.FirmwareUpdate +import com.macedonga.visor.firmware.FirmwareWork +import com.macedonga.visor.home.pairName import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.connection.GlassesLink import com.macedonga.visor.core.connection.GlassesTarget @@ -60,9 +66,11 @@ class GlassesPageModel @Inject constructor( val connection: GlassesConnection, val settings: GlassesSettings, val login: MetaLogin, + val firmware: FirmwareUpdate, val kind: LinkKind, private val store: IdentityStore, val looks: Looks, + val lastSeen: com.macedonga.visor.home.LastSeen, ) : ViewModel() { private val _glasses = MutableStateFlow>(emptyList()) @@ -83,6 +91,7 @@ class GlassesPageModel @Inject constructor( fun forget(serial: String) { viewModelScope.launch(Dispatchers.IO) { store.remove(serial) + lastSeen.forget(serial) _glasses.value = runCatching { store.all() }.getOrDefault(emptyList()) } } @@ -95,6 +104,8 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( val glasses by model.connection.state.collectAsStateWithLifecycle() val running by model.connection.running.collectAsStateWithLifecycle() val bundles by model.glasses.collectAsStateWithLifecycle() + val seen by model.lastSeen.seen.collectAsStateWithLifecycle() + val looks by model.looks.all.collectAsStateWithLifecycle() val login by model.login.status.collectAsStateWithLifecycle() val le by model.kind.overLe.collectAsStateWithLifecycle() LifecycleResumeEffect(Unit) { @@ -120,22 +131,29 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( val serial = glasses.target?.serial ?: info?.serial var renaming by remember { mutableStateOf(false) } var forgetting by remember { mutableStateOf(false) } + var firmwareOpen by remember { mutableStateOf(false) } + var confirmingFirmware by remember { mutableStateOf(false) } + val fwChecking by model.firmware.checking.collectAsStateWithLifecycle() + val fwChecked by model.firmware.checked.collectAsStateWithLifecycle() + val fwOffer by model.firmware.offer.collectAsStateWithLifecycle() + val fwProblem by model.firmware.problem.collectAsStateWithLifecycle() + val fwWork by model.firmware.work.collectAsStateWithLifecycle() SettingsPage(name, light = 0.6f) { item(key = "power") { Row(Modifier.fillMaxWidth().padding(horizontal = SIDE).padding(bottom = 8.dp), horizontalArrangement = Arrangement.spacedBy(36.dp)) { // Away, the last reading the home kept, dimmed. - val last = remember { context.getSharedPreferences("home", android.content.Context.MODE_PRIVATE) } + val last by model.lastSeen.battery.collectAsStateWithLifecycle() val glassesNow = device?.battery?.percent?.toInt() val caseNow = device?.case?.levelPercent() Readout( stringResource(R.string.glasses_battery), - (glassesNow ?: last.getInt("glasses", -1).takeIf { it >= 0 })?.let { "$it%" }, + (glassesNow ?: last?.glasses)?.let { "$it%" }, device?.battery?.charging == true, fresh = glassesNow != null, ) Readout( stringResource(R.string.glasses_case_label), - (caseNow ?: last.getInt("case", -1).takeIf { it >= 0 })?.let { "$it%" }, + (caseNow ?: last?.case)?.let { "$it%" }, device?.case?.charging == true, fresh = caseNow != null, ) } @@ -168,7 +186,7 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( }, look?.let { l -> l.colorName ?: l.color }?.let { v -> @Composable { ValueRow(stringResource(R.string.glasses_color), v, onClick = null) } }, look?.let { l -> l.lensName ?: l.lens }?.let { v -> @Composable { ValueRow(stringResource(R.string.glasses_lenses), v, onClick = null) } }, - { ValueRow(stringResource(R.string.glasses_firmware), info?.build?.let(::firmwareWords) ?: stringResource(R.string.settings_unknown), onClick = null) }, + { ValueRow(stringResource(R.string.glasses_firmware), info?.build?.let(::firmwareWords) ?: stringResource(R.string.settings_unknown), onClick = { firmwareOpen = true }) }, { ValueRow(stringResource(R.string.glasses_serial), serial ?: stringResource(R.string.settings_unknown), onClick = null) }, device?.thermal?.let { t -> @Composable { ValueRow(stringResource(R.string.glasses_heat), heatWords(t.health, t.throttling), onClick = null) } }, device?.storage?.let { s -> @Composable { ValueRow(stringResource(R.string.glasses_storage), stringResource(R.string.glasses_storage_free, gb(s.freeMb), gb(s.totalMb)), onClick = null) } }, @@ -183,6 +201,7 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( RowFrame( stringResource( when { + ready && glasses.target?.link is GlassesLink.Wifi -> R.string.glasses_link_wifi ready -> if (le) R.string.glasses_link_le else R.string.glasses_link_rfcomm running && glasses.link != LinkState.DISCONNECTED && glasses.link != LinkState.LOST -> R.string.glasses_searching else -> R.string.glasses_link_off @@ -201,14 +220,19 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( }, *bundles.filter { it.serial != serial }.map { b -> @Composable { - RowFrame(b.serial, onClick = null, sub = b.glassesAddress) { + val other = seen[b.serial] + RowFrame( + pairName(other, looks[b.serial]), onClick = null, + sub = other?.glasses?.let { g -> other.case?.let { c -> stringResource(R.string.glasses_other_batteries, g, c) } ?: "$g%" }, + ) { ActionPill(stringResource(R.string.glasses_connect), { - GlassesService.stop(context) GlassesService.start(context, GlassesTarget(GlassesLink.Bluetooth(b.glassesAddress), b.serial)) }, quiet = true, modifier = Modifier.padding(start = 12.dp)) } } }.toTypedArray(), + if (hasDisplay(info?.build)) row { ValueRow(stringResource(R.string.wifi_networks), null, onClick = { open(WifiNetworksKey) }, locked = !ready) } else null, + if (hasDisplay(info?.build)) row { ValueRow(stringResource(R.string.webapps_row), null, onClick = { open(WebAppsKey) }, locked = !ready) } else null, ) } } @@ -227,6 +251,7 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( } }, { ValueRow(stringResource(R.string.glasses_change_owner), null, onClick = { open(OnboardingKey(login = true, address = (glasses.target?.link as? GlassesLink.Bluetooth)?.address)) }, sub = stringResource(R.string.glasses_change_owner_sub)) }, + if (hasDisplay(info?.build)) row { ValueRow(stringResource(R.string.accounts_title), null, onClick = { open(AccountsKey) }, locked = !ready) } else null, ) } } @@ -264,6 +289,43 @@ fun GlassesPage(open: (NavKey) -> Unit, model: GlassesPageModel = hiltViewModel( } } } + if (firmwareOpen) { + VisorSheet({ firmwareOpen = false; confirmingFirmware = false; model.firmware.dismiss() }) { + SheetTitle(stringResource(R.string.firmware_title)) + Rows( + { ValueRow(stringResource(R.string.firmware_current), info?.build?.let(::firmwareWords) ?: stringResource(R.string.settings_unknown), onClick = null) }, + fwOffer?.let { o -> @Composable { ValueRow(stringResource(R.string.firmware_update), o.displayName.ifEmpty { o.targetVersion }, onClick = null) } }, + ) + val status: String? = when (val w = fwWork) { + is FirmwareWork.Failed -> stringResource(R.string.firmware_failed, w.reason) + FirmwareWork.Installed -> stringResource(R.string.firmware_installed) + is FirmwareWork.Installing -> w.percent?.let { stringResource(R.string.firmware_progress, it) } ?: stringResource(R.string.firmware_installing) + FirmwareWork.Idle -> when { + fwChecking -> stringResource(R.string.firmware_checking) + fwProblem.isNotEmpty() -> fwProblem + fwChecked && fwOffer == null -> stringResource(R.string.firmware_up_to_date) + else -> null + } + } + Column(Modifier.padding(horizontal = SIDE, vertical = 16.dp)) { + if (status != null) Footnote(status) + val offer = fwOffer + val installing = fwWork is FirmwareWork.Installing + if (confirmingFirmware && offer != null) { + Row(Modifier.padding(top = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp)) { + ActionPill(stringResource(R.string.firmware_install), { confirmingFirmware = false; model.firmware.install() }) + ActionPill(stringResource(R.string.language_cancel), { confirmingFirmware = false }, quiet = true) + } + Footnote(stringResource(R.string.firmware_confirm, offer.displayName.ifEmpty { offer.targetVersion })) + } else { + Row(Modifier.padding(top = 16.dp), horizontalArrangement = Arrangement.spacedBy(10.dp)) { + ActionPill(stringResource(R.string.firmware_check), { model.firmware.check() }, quiet = true, enabled = !fwChecking && !installing) + if (offer != null) ActionPill(stringResource(R.string.firmware_install), { confirmingFirmware = true }, enabled = !installing) + } + } + } + } + } } private const val NAME_KEY = "device_custom_name" diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt b/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt index 46970bb3..e8b72966 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/PhoneAccess.kt @@ -46,6 +46,9 @@ enum class Need(val title: Int, val why: Int) { NOTIFY(R.string.access_notify, R.string.access_notify_why), MIC(R.string.access_mic, R.string.access_mic_why), CONTACTS(R.string.access_contacts, R.string.access_contacts_why), + SMS(R.string.access_sms, R.string.access_sms_why), + CALLS(R.string.access_calls, R.string.access_calls_why), + CALL_CONTROL(R.string.access_call_control, R.string.access_call_control_why), LOCATION(R.string.access_location, R.string.access_location_why), /** Location while Visor is in the background: Android asks it on its own, once [LOCATION] is given. */ @@ -62,6 +65,10 @@ enum class Need(val title: Int, val why: Int) { NOTIFY -> context.allows(Manifest.permission.POST_NOTIFICATIONS) MIC -> context.allows(Manifest.permission.RECORD_AUDIO) CONTACTS -> context.allows(Manifest.permission.READ_CONTACTS) + SMS -> context.allows(Manifest.permission.READ_SMS) && context.allows(Manifest.permission.SEND_SMS) + CALLS -> context.allows(Manifest.permission.READ_CALL_LOG) + CALL_CONTROL -> context.allows(Manifest.permission.CALL_PHONE) && context.allows(Manifest.permission.ANSWER_PHONE_CALLS) && + context.allows(Manifest.permission.READ_PHONE_STATE) LOCATION -> context.allows(Manifest.permission.ACCESS_FINE_LOCATION) LOCATION_ALWAYS -> context.allows(Manifest.permission.ACCESS_BACKGROUND_LOCATION) WIFI -> wifiDirectAllowed(context) @@ -75,6 +82,9 @@ enum class Need(val title: Int, val why: Int) { NOTIFY -> arrayOf(Manifest.permission.POST_NOTIFICATIONS) MIC -> arrayOf(Manifest.permission.RECORD_AUDIO) CONTACTS -> arrayOf(Manifest.permission.READ_CONTACTS) + SMS -> arrayOf(Manifest.permission.READ_SMS, Manifest.permission.SEND_SMS) + CALLS -> arrayOf(Manifest.permission.READ_CALL_LOG) + CALL_CONTROL -> arrayOf(Manifest.permission.CALL_PHONE, Manifest.permission.ANSWER_PHONE_CALLS, Manifest.permission.READ_PHONE_STATE) LOCATION -> arrayOf(Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_COARSE_LOCATION) // Android grants it only after the foreground one, so that one comes first when it is missing. LOCATION_ALWAYS -> if (context.allows(Manifest.permission.ACCESS_FINE_LOCATION)) arrayOf(Manifest.permission.ACCESS_BACKGROUND_LOCATION) diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/SettingControls.kt b/app/src/main/kotlin/com/macedonga/visor/settings/SettingControls.kt index cd9d8311..87fe3bb1 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/SettingControls.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/SettingControls.kt @@ -147,7 +147,7 @@ fun GlassesRow(key: String, engineering: Boolean = false, overline: String? = nu is Control.Choice -> { val current = value.trim() // Options the Meta app keeps for other glasses show only in Advanced. - val shown = c.options.filter { engineering || it.offeredOn(state.product) || it.value == current } + val shown = c.options.filter { engineering || it.value == current || (!it.hidden && it.offeredOn(state.product)) } if (shown.size <= 4 && shown.all { it.label.length <= 16 && it.detail == null } && shown.any { it.value == current }) { PillsRow( spec.label, shown.map { it.value to it.label }, current, onPick = write, diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt index c9909f75..2d6498e2 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt @@ -124,7 +124,6 @@ import com.macedonga.visor.gallery.ProgressRing import com.macedonga.visor.gallery.RoundIcon import com.macedonga.visor.gallery.placesAllowed import com.macedonga.visor.gallery.words -import com.macedonga.visor.home.LastBattery import com.macedonga.visor.home.StatusPill import com.macedonga.visor.language.languageOfVoice import com.macedonga.visor.media.spanWords @@ -173,7 +172,11 @@ class SettingsViewModel @Inject constructor( val backups: Backups, val updates: com.macedonga.visor.update.Updates, val features: com.macedonga.visor.features.Features, + val lastSeen: com.macedonga.visor.home.LastSeen, val hearing: com.macedonga.visor.core.assistant.Hearing, + val band: com.macedonga.visor.band.BandLink, + val bands: com.macedonga.visor.band.Bands, + val bandPeers: com.macedonga.visor.band.BandPeers, val developer: com.macedonga.visor.outside.DeveloperMode, ) : ViewModel() @@ -245,7 +248,9 @@ fun SettingsScreen(open: (NavKey) -> Unit, model: SettingsViewModel = hiltViewMo for ((i, card) in Topic.CARDS.withIndex()) { item(key = "card-$i") { Column(Modifier.padding(top = if (i == 0) 18.dp else 14.dp)) { - for (topic in card) NavRow(topic.icon, stringResource(topic.title), summary(topic, model, state), onClick = { open(placeOf(topic)) }) + for (topic in card.filter { it.shownOn(state.product) }) { + NavRow(topic.icon, stringResource(topic.title), summary(topic, model, state), onClick = { open(placeOf(topic)) }) + } } } } @@ -290,11 +295,7 @@ fun SettingsScreen(open: (NavKey) -> Unit, model: SettingsViewModel = hiltViewMo .turnsThePair { host.haptics.tick(0.35f) } .clickable(interactionSource = remember { MutableInteractionSource() }, indication = null) { open(GlassesKey) }, ) { - val remembered = remember { - context.getSharedPreferences("home", android.content.Context.MODE_PRIVATE).let { p -> - p.getInt("glasses", -1).takeIf { it >= 0 }?.let { LastBattery(it, p.getInt("case", -1).takeIf { c -> c >= 0 }, p.getLong("at", 0)) } - } - } + val remembered by model.lastSeen.battery.collectAsStateWithLifecycle() StatusPill( glasses, mood, remembered, paired = glasses.target?.let { true }, allowed = true, importing = false, importProgress = 0f, waiting = 0, onImport = {}, onAllow = {}, onSetUp = { open(OnboardingKey()) }, onChangeOwner = { open(OnboardingKey(login = true, address = it)) }, @@ -564,6 +565,14 @@ private fun summary(topic: Topic, model: SettingsViewModel, state: SettingsState shown("capture_mode")?.let { stringResource(R.string.sum_controls_press, it) }, shown("tap_and_hold_mapping")?.let { stringResource(R.string.sum_controls_hold, it) }, ).joinToString(" · ") + Topic.BAND -> { + // The Display reports the band it holds. + val glasses by model.connection.state.collectAsStateWithLifecycle() + val paired by model.bandPeers.bands.collectAsStateWithLifecycle() + val device = glasses.device + device?.bandBattery?.let { batteryWords(it, device.bandCharging) } + ?: if (paired?.isEmpty() == true) stringResource(R.string.band_not_paired) else "" + } Topic.SOUND -> listOfNotNull( shown("notification_volume")?.let { stringResource(R.string.sum_sound_volume, it) }, stringResource(R.string.sum_headphone).takeIf { isOn("sg_headphone_mode_enabled") == true }, diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/TopicScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/TopicScreen.kt index e8972054..c04bf6bf 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/TopicScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/TopicScreen.kt @@ -116,6 +116,7 @@ fun TopicScreen(name: String, open: (NavKey) -> Unit, model: SettingsViewModel = ) } Topic.PHOTOS -> item(key = "visor") { PhotosPart(model) } + Topic.BAND -> item(key = "visor") { BandPart(model) } else -> Unit } // Hearing is nothing but features, so they need no heading of their own there. diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt b/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt index be3260e6..96564acf 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/Topics.kt @@ -2,6 +2,7 @@ package com.macedonga.visor.settings import androidx.compose.ui.graphics.vector.ImageVector import com.macedonga.visor.R +import com.macedonga.visor.core.settings.productId import com.macedonga.visor.features.Feature import com.macedonga.visor.ui.VisorIcons @@ -34,6 +35,8 @@ enum class Topic(val title: Int, val icon: ImageVector, val keys: List = "spotify_tap_enabled", ), ), + // The Display's: the band itself, then what the glasses do with it. + BAND(R.string.topic_band, VisorIcons.Pulse, listOf("sg_emg_display_off", "sg_emg_instant_action", "sg_invert_wrist_roll_direction")), // Spatial audio is the feature row: the setting does nothing while Meta's switch is off. SOUND( R.string.topic_sound, VisorIcons.Speaker, @@ -72,7 +75,7 @@ enum class Topic(val title: Int, val icon: ImageVector, val keys: List = /** The main page's cards. */ val CARDS = listOf( listOf(META_AI, LANGUAGE), - listOf(NOTIFICATIONS, CONTROLS, SOUND, HEARING, WEAR), + listOf(NOTIFICATIONS, CONTROLS, BAND, SOUND, HEARING, WEAR), listOf(CAMERA, PHOTOS, LIVE), listOf(PRIVACY, PHONE), ) @@ -100,10 +103,15 @@ fun tileIcon(id: String): ImageVector = when (id) { /** Groups the firmware keeps for glasses with a display; they matter only to a model that has one. */ val DISPLAY_GROUPS = setOf("Display", "Neural band gestures") -/** Whether the connected glasses have a display: the build fingerprint names the product, "Meta//...". */ -fun hasDisplay(build: String?): Boolean { - val product = build?.split('/')?.getOrNull(1)?.lowercase() ?: return false - return DISPLAY_PRODUCTS.any { it in product } -} +/** Whether the connected glasses have a display: the build fingerprint names the device, "Meta/greatwhite/greatwhite:14/...". */ +fun hasDisplay(build: String?): Boolean = productId(build) in DISPLAY_PRODUCTS + +// Meta Ray-Ban Display. +private val DISPLAY_PRODUCTS = setOf(259, 270, 271) -private val DISPLAY_PRODUCTS = listOf("hypernova", "display") +/** Whether [product] has anything on this topic: the band is the Display's, and hearing is nothing but features. */ +fun Topic.shownOn(product: Int?): Boolean = when (this) { + Topic.BAND -> product in DISPLAY_PRODUCTS + Topic.HEARING -> Feature.entries.any { it.topic == Topic.HEARING && it.on(product) } + else -> true +} diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/WakeWord.kt b/app/src/main/kotlin/com/macedonga/visor/settings/WakeWord.kt index 39ee0237..40c36c25 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/WakeWord.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/WakeWord.kt @@ -45,11 +45,13 @@ import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.lifecycle.viewModelScope import com.macedonga.visor.R import com.macedonga.visor.core.assistant.Assistant +import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.design.Visor import com.macedonga.visor.core.settings.GlassesSettings import com.macedonga.visor.core.settings.asBool import com.macedonga.visor.features.ChoicePill import com.macedonga.visor.features.StoryWords +import com.macedonga.visor.pairs.pairPrefs import com.macedonga.visor.setup.BigPill import com.macedonga.visor.stage.LightLook import com.macedonga.visor.stage.LocalStage @@ -91,9 +93,11 @@ sealed interface WakeStep { class WakeWordModel @Inject constructor( private val assistant: Assistant, val settings: GlassesSettings, + connection: GlassesConnection, @ApplicationContext context: Context, ) : ViewModel() { - private val prefs = context.getSharedPreferences("wake_word", Context.MODE_PRIVATE) + // Enrolled on the glasses: each pair has its own phrase. + private val prefs = pairPrefs(context, "wake_word", connection.pair.value) private val _enrolled = MutableStateFlow(prefs.getString(ENROLLED, null)) diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt new file mode 100644 index 00000000..307b11b7 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/settings/WebAppsScreen.kt @@ -0,0 +1,197 @@ +package com.macedonga.visor.settings + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.lifecycle.viewModelScope +import com.macedonga.visor.R +import com.macedonga.visor.connection.AddResult +import com.macedonga.visor.connection.WebApp +import com.macedonga.visor.connection.WebApps +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.identity.appstore.CatalogueApp +import com.macedonga.visor.core.identity.appstore.fetchAppCatalogue +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.login.MetaLogin +import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +enum class WebAppNote { NOT_CONNECTED, UNREADABLE, OPENED, NOT_OPENED, NOT_HTTPS, FULL } + +sealed interface CatalogueState { + data object Idle : CatalogueState + data object Loading : CatalogueState + data class Loaded(val apps: List) : CatalogueState + data object Failed : CatalogueState +} + +@HiltViewModel +class WebAppsModel @Inject constructor( + private val webApps: WebApps, + val connection: GlassesConnection, + private val login: MetaLogin, +) : ViewModel() { + val apps: StateFlow> = webApps.apps + + private val _note = MutableStateFlow(null) + val note: StateFlow = _note.asStateFlow() + + private val _catalogue = MutableStateFlow(CatalogueState.Idle) + val catalogue: StateFlow = _catalogue.asStateFlow() + + fun add(name: String, url: String) { + _note.value = noteOf(webApps.add(name, url)) + } + + /** Fetch Meta's list for these glasses, with the user's Meta login. */ + fun browse() { + val info = connection.state.value.info + if (info == null || info.wearablesRelease.isEmpty()) { + _catalogue.value = CatalogueState.Failed + return + } + _catalogue.value = CatalogueState.Loading + viewModelScope.launch { + _catalogue.value = runCatching { + withContext(Dispatchers.IO) { fetchAppCatalogue(login.source.get(), info.serial, info.wearablesRelease) } + }.fold({ CatalogueState.Loaded(it) }, { CatalogueState.Failed }) + } + } + + fun addFromCatalogue(app: CatalogueApp) { + _note.value = noteOf(webApps.add(app.name, app.url, app.manifest)) + } + + fun remove(key: String) { + _note.value = if (webApps.remove(key)) null else WebAppNote.NOT_CONNECTED + } + + private fun noteOf(result: AddResult): WebAppNote? = when (result) { + AddResult.ADDED, AddResult.EXISTS -> null + AddResult.NOT_HTTPS -> WebAppNote.NOT_HTTPS + AddResult.FULL -> WebAppNote.FULL + AddResult.NOT_CONNECTED -> WebAppNote.NOT_CONNECTED + AddResult.UNREADABLE -> WebAppNote.UNREADABLE + } + + fun open(app: WebApp) { + viewModelScope.launch { + _note.value = if (webApps.open(app)) WebAppNote.OPENED else WebAppNote.NOT_OPENED + } + } +} + +/** The Display's web apps, as the glasses list them, and opening one on the glasses. */ +@Composable +fun WebAppsScreen(model: WebAppsModel = hiltViewModel()) { + val glasses by model.connection.state.collectAsStateWithLifecycle() + val ready = glasses.link == LinkState.READY + val apps by model.apps.collectAsStateWithLifecycle() + val note by model.note.collectAsStateWithLifecycle() + val catalogue by model.catalogue.collectAsStateWithLifecycle() + var adding by remember { mutableStateOf(false) } + var browsing by remember { mutableStateOf(false) } + + SettingsPage(stringResource(R.string.webapps_title), light = 0f) { + item(key = "apps") { + Column { + Rows( + *apps.map { app -> + @Composable { + RowFrame(app.name, onClick = null, sub = app.url) { + ActionPill( + stringResource(R.string.webapps_open), { model.open(app) }, quiet = true, + modifier = Modifier.padding(start = 12.dp), enabled = ready, + ) + ActionPill( + stringResource(R.string.webapps_remove), { model.remove(app.key) }, quiet = true, + modifier = Modifier.padding(start = 12.dp), enabled = ready, + ) + } + } + }.toTypedArray(), + { ValueRow(stringResource(R.string.webapps_add), null, onClick = { adding = true }, locked = !ready) }, + { ValueRow(stringResource(R.string.webapps_browse), null, onClick = { browsing = true; model.browse() }, locked = !ready) }, + ) + note?.let { Footnote(stringResource(noteText(it))) } + if (apps.isEmpty()) Footnote(stringResource(R.string.webapps_none)) + } + } + } + + if (browsing) { + VisorSheet({ browsing = false }, full = true) { + SheetTitle(stringResource(R.string.webapps_browse)) + when (val c = catalogue) { + CatalogueState.Idle, CatalogueState.Loading -> Spinner(Modifier.padding(SIDE).padding(top = 16.dp)) + CatalogueState.Failed -> Footnote(stringResource(R.string.webapps_catalogue_failed)) + is CatalogueState.Loaded -> if (c.apps.isEmpty()) { + Footnote(stringResource(R.string.webapps_catalogue_none)) + } else { + Rows( + *c.apps.map { app -> + @Composable { + val on = apps.any { it.url == app.url } + RowFrame(app.name, onClick = null, sub = app.url) { + if (on) { + ActionPill(stringResource(R.string.webapps_on_glasses), {}, quiet = true, modifier = Modifier.padding(start = 12.dp), enabled = false) + } else { + ActionPill(stringResource(R.string.webapps_add), { model.addFromCatalogue(app) }, quiet = true, modifier = Modifier.padding(start = 12.dp)) + } + } + } + }.toTypedArray(), + ) + } + } + note?.let { Footnote(stringResource(noteText(it))) } + } + } + + if (adding) { + var name by remember { mutableStateOf("") } + var url by remember { mutableStateOf("https://") } + VisorSheet({ adding = false }) { + SheetTitle(stringResource(R.string.webapps_add)) + SheetField(name, { name = it }, hint = stringResource(R.string.webapps_name), focus = true) + Spacer(Modifier.height(10.dp)) + SheetField(url, { url = it }, hint = stringResource(R.string.webapps_url)) + Row(Modifier.fillMaxWidth().padding(horizontal = SIDE).padding(top = 16.dp), horizontalArrangement = Arrangement.End) { + ActionPill(stringResource(R.string.settings_save), { + model.add(name.trim(), url.trim()) + adding = false + }, enabled = url.isNotBlank()) + } + } + } +} + +private fun noteText(note: WebAppNote): Int = when (note) { + WebAppNote.NOT_CONNECTED -> R.string.webapps_not_connected + WebAppNote.UNREADABLE -> R.string.webapps_unreadable + WebAppNote.OPENED -> R.string.webapps_opened + WebAppNote.NOT_OPENED -> R.string.webapps_not_opened + WebAppNote.NOT_HTTPS -> R.string.webapps_not_https + WebAppNote.FULL -> R.string.webapps_full +} diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/WifiNetworksScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/WifiNetworksScreen.kt new file mode 100644 index 00000000..7475508e --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/settings/WifiNetworksScreen.kt @@ -0,0 +1,155 @@ +package com.macedonga.visor.settings + +import android.content.Context +import android.net.wifi.WifiManager +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.unit.dp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.ViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.lifecycle.viewModelScope +import com.macedonga.visor.R +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.connection.GlassesWifi +import com.macedonga.visor.core.session.LinkState +import dagger.hilt.android.lifecycle.HiltViewModel +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch + +enum class WifiResult { SAVED, WRONG_PASSWORD, REMOVED, FAILED } + +@HiltViewModel +class WifiNetworksModel @Inject constructor( + @ApplicationContext private val context: Context, + val connection: GlassesConnection, + private val wifi: GlassesWifi, +) : ViewModel() { + + // Null until the glasses answer. + private val _networks = MutableStateFlow?>(null) + val networks: StateFlow?> = _networks.asStateFlow() + + private val _busy = MutableStateFlow(false) + val busy: StateFlow = _busy.asStateFlow() + + private val _result = MutableStateFlow(null) + val result: StateFlow = _result.asStateFlow() + + /** The network the phone is on, when Android gives its name. */ + fun phoneNetwork(): String? { + val info = context.getSystemService(WifiManager::class.java)?.connectionInfo ?: return null + return info.ssid?.trim('"')?.takeIf { it.isNotBlank() && it != "" } + } + + fun refresh() = task { + _networks.value = wifi.saved() ?: _networks.value + } + + fun save(ssid: String, passphrase: String) = task { + _result.value = when (wifi.save(ssid, passphrase, open = false)) { + 0 -> WifiResult.SAVED + 6 -> WifiResult.WRONG_PASSWORD + else -> WifiResult.FAILED + } + wifi.saved()?.let { _networks.value = it } + } + + fun remove(ssid: String) = task { + _result.value = if (wifi.remove(ssid) == 0) WifiResult.REMOVED else WifiResult.FAILED + wifi.saved()?.let { _networks.value = it } + } + + private fun task(work: suspend () -> Unit) { + _busy.value = true + viewModelScope.launch { + try { + work() + } finally { + _busy.value = false + } + } + } +} + +/** The Wi-Fi networks a Display's glasses keep, with the means to add and remove them. */ +@Composable +fun WifiNetworksScreen(model: WifiNetworksModel = hiltViewModel()) { + val glasses by model.connection.state.collectAsStateWithLifecycle() + val ready = glasses.link == LinkState.READY + val networks by model.networks.collectAsStateWithLifecycle() + val busy by model.busy.collectAsStateWithLifecycle() + val result by model.result.collectAsStateWithLifecycle() + var adding by remember { mutableStateOf(false) } + LaunchedEffect(ready) { if (ready) model.refresh() } + + SettingsPage(stringResource(R.string.wifi_networks_title), light = 0f) { + item(key = "networks") { + Column { + val saved = networks.orEmpty() + Rows( + *saved.map { ssid -> + @Composable { + RowFrame(ssid, onClick = null, locked = !ready) { + ActionPill( + stringResource(R.string.wifi_remove), { model.remove(ssid) }, + quiet = true, modifier = Modifier.padding(start = 12.dp), enabled = ready && !busy, + ) + } + } + }.toTypedArray(), + { ValueRow(stringResource(R.string.wifi_add), null, onClick = { adding = true }, locked = !ready, busy = busy) }, + ) + result?.let { r -> + Footnote( + stringResource( + when (r) { + WifiResult.SAVED -> R.string.wifi_saved + WifiResult.WRONG_PASSWORD -> R.string.wifi_wrong_password + WifiResult.REMOVED -> R.string.wifi_removed + WifiResult.FAILED -> R.string.wifi_failed + }, + ), + ) + } + if (networks != null && saved.isEmpty()) Footnote(stringResource(R.string.wifi_none)) + if (networks == null && ready && !busy) Footnote(stringResource(R.string.wifi_unread)) + } + } + } + + if (adding) { + val phone = remember { model.phoneNetwork().orEmpty() } + var name by remember { mutableStateOf(phone) } + var password by remember { mutableStateOf("") } + VisorSheet({ adding = false }) { + SheetTitle(stringResource(R.string.wifi_add)) + SheetField(name, { name = it }, hint = stringResource(R.string.wifi_network_name), focus = name.isEmpty()) + Spacer(Modifier.height(10.dp)) + SheetField(password, { password = it }, hint = stringResource(R.string.wifi_password), secret = true, focus = name.isNotEmpty()) + Row(Modifier.fillMaxWidth().padding(horizontal = SIDE).padding(top = 16.dp), horizontalArrangement = Arrangement.End) { + ActionPill(stringResource(R.string.settings_save), { + model.save(name.trim(), password) + adding = false + }, enabled = name.isNotBlank() && password.isNotEmpty()) + } + } + } +} diff --git a/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt b/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt index cd507a7b..f0af7538 100644 --- a/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt +++ b/app/src/main/kotlin/com/macedonga/visor/setup/NearbyGlasses.kt @@ -69,9 +69,10 @@ class NearbyGlasses @Inject constructor(@ApplicationContext private val context: val data = record.getManufacturerSpecificData(META_COMPANY) if (data == null && record.serviceUuids?.any { it.uuid == META_LE_SERVICE } != true) return val advert = data?.let(::advert) - if (advert?.product == PRODUCT_BAND) return + if (advert?.product in PRODUCT_BANDS) return val device = result.device val name = record.deviceName ?: runCatching { device.name }.getOrNull() ?: productName(advert?.product) + if (BAND_NAME.containsMatchIn(name)) return val pair = NearbyPair(device.address, name, result.rssi, advert?.pairing == true, device.bondState == BluetoothDevice.BOND_BONDED) synchronized(heard) { heard[device.address] = pair to SystemClock.elapsedRealtime() } publish() @@ -99,7 +100,7 @@ class NearbyGlasses @Inject constructor(@ApplicationContext private val context: @SuppressLint("MissingPermission") fun bonded(): List = runCatching { adapter?.bondedDevices.orEmpty() - .filter { d -> d.name?.let { GLASSES_NAME.containsMatchIn(it) } == true } + .filter { d -> d.name?.let { GLASSES_NAME.containsMatchIn(it) && !BAND_NAME.containsMatchIn(it) } == true } .map { NearbyPair(it.address, it.name, null, pairing = false, bonded = true) } }.getOrDefault(emptyList()) @@ -134,6 +135,30 @@ class NearbyGlasses @Inject constructor(@ApplicationContext private val context: throw BondFailed(BondFailed.Why.NO_ANSWER) } + /** + * The Neural Band heard nearby, one in pairing mode before the strongest: its address, or null when none answers + * within [BAND_SCAN_MS]. Bonded by [bond], as the glasses are. + */ + @SuppressLint("MissingPermission") + suspend fun bandNearby(): String? { + val scanner = adapter?.takeIf { it.isEnabled }?.bluetoothLeScanner ?: throw BondFailed(BondFailed.Why.BLUETOOTH_OFF) + val heard = HashMap>() + val callback = object : ScanCallback() { + override fun onScanResult(callbackType: Int, result: ScanResult) { + val advert = result.scanRecord?.getManufacturerSpecificData(META_COMPANY)?.let(::advert) ?: return + if (advert.product !in PRODUCT_BANDS) return + synchronized(heard) { heard[result.device.address] = advert.pairing to result.rssi } + } + } + scanner.startScan(null, ScanSettings.Builder().setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY).build(), callback) + try { + delay(BAND_SCAN_MS) + } finally { + runCatching { scanner.stopScan(callback) } + } + return synchronized(heard) { heard.maxByOrNull { (_, v) -> (if (v.first) 1_000 else 0) + v.second }?.key } + } + private class Advert(val product: Int, val pairing: Boolean) /** Manufacturer data after the company id: legacy [product:2 LE][flags], or V2 [version][product:2 LE][reserved][flags]. */ @@ -146,7 +171,7 @@ class NearbyGlasses @Inject constructor(@ApplicationContext private val context: private fun productName(product: Int?) = when (product) { 257 -> "Ray-Ban Stories" - 259 -> "Meta Ray-Ban Display" + 259, 270, 271 -> "Meta Ray-Ban Display" 264 -> "Oakley Meta Vanguard" 265 -> "Oakley Meta HSTN" else -> context.getString(R.string.glasses_default_name) @@ -154,11 +179,15 @@ class NearbyGlasses @Inject constructor(@ApplicationContext private val context: private companion object { const val META_COMPANY = 0x01AB - const val PRODUCT_BAND = 1537 + val PRODUCT_BANDS = setOf(1537, 1538) const val FORGET_MS = 12_000L const val SWEEP_MS = 2_000L const val BOND_MS = 90_000L + const val BAND_SCAN_MS = 15_000L const val POLL_MS = 400L val GLASSES_NAME = Regex("ray-?ban|meta|oakley|stories", RegexOption.IGNORE_CASE) + + // The Neural Band, "Meta Band 0064": set up from the Display's settings, never as glasses. + val BAND_NAME = Regex("^meta band", RegexOption.IGNORE_CASE) } } diff --git a/app/src/main/kotlin/com/macedonga/visor/stage/Frames.kt b/app/src/main/kotlin/com/macedonga/visor/stage/Frames.kt index ff14b088..8522bf16 100644 --- a/app/src/main/kotlin/com/macedonga/visor/stage/Frames.kt +++ b/app/src/main/kotlin/com/macedonga/visor/stage/Frames.kt @@ -1,12 +1,19 @@ package com.macedonga.visor.stage +import com.macedonga.visor.core.identity.token.withToken import android.content.Context +import android.graphics.BitmapFactory +import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.graphics.asImageBitmap import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.identity.language.GlassesLook import com.macedonga.visor.core.identity.language.fetchGlassesLook +import com.macedonga.visor.core.identity.language.fetchPairPictures import com.macedonga.visor.log.EventLog import com.macedonga.visor.login.MetaLogin import dagger.hilt.android.qualifiers.ApplicationContext +import java.net.HttpURLConnection +import java.net.URL import java.util.Locale import javax.inject.Inject import javax.inject.Singleton @@ -36,6 +43,9 @@ enum class FrameModel(val asset: String, val styles: Set) { } } +/** Whether Visor has a model of [look]'s frame; an unknown look gets the one there is. */ +fun modelled(look: GlassesLook?): Boolean = look?.style?.lowercase()?.let { style -> FrameModel.entries.any { style in it.styles } } ?: true + /** * The model's material variants to try for [look], best first: one named for the exact colour and lens * (`shiny_transparent_grey_sapphire_transitions`), for a model that has them, then the finish and the lens tint every @@ -91,9 +101,16 @@ class Looks @Inject constructor( val asking: StateFlow = _asking.asStateFlow() val problem: StateFlow = _problem.asStateFlow() + private val _all = MutableStateFlow( + prefs.all.mapNotNull { (key, text) -> if (key.startsWith(LOOK) && text is String) decode(text)?.let { key.removePrefix(LOOK) to it } else null }.toMap(), + ) + + /** Every pair's look known so far, by serial: the home shows each pair as it is. */ + val all: StateFlow> = _all.asStateFlow() + init { scope.launch { - combine(connection.state.map { it.target?.serial ?: it.info?.serial }.distinctUntilChanged(), login.status) { serial, _ -> serial } + combine(connection.pair, connection.state.map { it.info?.serial }.distinctUntilChanged(), login.status) { pair, told, _ -> pair ?: told } .collect { serial -> if (serial != null) ensure(serial, force = false) } } } @@ -104,21 +121,84 @@ class Looks @Inject constructor( scope.launch { ensure(serial, force = true) } } + private val _pictures = MutableStateFlow>(emptyMap()) + + /** Each pair as Meta pictures it, by serial: only in memory, asked for again at each launch. */ + val pictures: StateFlow> = _pictures.asStateFlow() + private val picturing = Mutex() + + /** Ask Meta for the pictures of [serials] it has not given yet. */ + fun wantPictures(serials: List) { + scope.launch { + picturing.withLock { + val missing = serials.filter { it !in _pictures.value } + if (missing.isEmpty() || !login.source.hasCachedLogin) return@withLock + val where = runCatching { login.source.withToken { fetchPairPictures(it, missing) } } + .onFailure { log.add("Pictures: ${it.message}") }.getOrNull() ?: return@withLock + for ((serial, picture) in where) { + // The app is dark: the picture made for a dark background. + val bitmap = runCatching { download(picture.dark ?: picture.light) }.onFailure { log.add("Pictures: $serial: ${it.message}") }.getOrNull() + if (bitmap != null) _pictures.value = _pictures.value + (serial to bitmap) + } + } + } + } + + private fun download(url: String): ImageBitmap { + val connection = URL(url).openConnection() as HttpURLConnection + try { + connection.connectTimeout = 20_000 + connection.readTimeout = 20_000 + val bytes = connection.inputStream.use { it.readBytes() } + return checkNotNull(BitmapFactory.decodeByteArray(bytes, 0, bytes.size)) { "not an image" }.asImageBitmap() + } finally { + connection.disconnect() + } + } + + /** Debug builds: a demo pair's look and Meta's picture at [picture], in memory only; a null look takes them away. */ + fun demo(serial: String, look: GlassesLook?, picture: String?) { + if (look == null) { + _all.value = _all.value - serial + _pictures.value = _pictures.value - serial + return + } + _all.value = _all.value + (serial to look) + picture ?: return + scope.launch { + runCatching { download(picture) }.onSuccess { _pictures.value = _pictures.value + (serial to it) } + .onFailure { log.add("Pictures: demo: ${it.message}") } + } + } + + /** Ask for another pair's look, if it is not known yet; the current pair's stays as it is. */ + fun want(serial: String) { + if (serial in _all.value) return + scope.launch { lock.withLock { fetch(serial, force = false) } } + } + private suspend fun ensure(serial: String, force: Boolean) = lock.withLock { this.serial = serial - val kept = prefs.getString(LOOK + serial, null)?.let(::decode) + val kept = _all.value[serial] _look.value = kept if (kept != null) return@withLock + _asking.value = true + val look = fetch(serial, force) + // The pair may have changed while Meta answered. + if (this.serial == serial && look != null) _look.value = look + _asking.value = false + } + + private suspend fun fetch(serial: String, force: Boolean): GlassesLook? { if (!login.source.hasCachedLogin) { _problem.value = LookProblem.LOGGED_OUT - return@withLock + return null } val now = System.currentTimeMillis() - if (!force && now - prefs.getLong(NO_RECORD + serial, 0) < RETRY_MS) return@withLock - _asking.value = true + if (!force && now - prefs.getLong(NO_RECORD + serial, 0) < RETRY_MS) return null // Only language and country: Android adds the regional preferences to the default locale's name. val locale = Locale.getDefault().let { if (it.country.isEmpty()) "en_US" else "${it.language}_${it.country}" } - runCatching { fetchGlassesLook(login.source.get(), serial, locale) } + return runCatching { login.source.withToken { fetchGlassesLook(it, serial, locale) } } .onSuccess { look -> if (look == null) { log.add("Look: Meta has no record of $serial") @@ -127,7 +207,7 @@ class Looks @Inject constructor( } else { log.add("Look: ${look.style} ${look.color} ${look.lens}") prefs.edit().putString(LOOK + serial, encode(look)).putString(STYLE, look.style).apply() - _look.value = look + _all.value = _all.value + (serial to look) _problem.value = null } } @@ -135,7 +215,7 @@ class Looks @Inject constructor( log.add("Look: ${it.message}") _problem.value = LookProblem.FAILED } - _asking.value = false + .getOrNull() } private fun encode(l: GlassesLook) = JSONObject() diff --git a/app/src/main/kotlin/com/macedonga/visor/stage/GlassesRenderer.kt b/app/src/main/kotlin/com/macedonga/visor/stage/GlassesRenderer.kt index c0d5fe0b..604efc27 100644 --- a/app/src/main/kotlin/com/macedonga/visor/stage/GlassesRenderer.kt +++ b/app/src/main/kotlin/com/macedonga/visor/stage/GlassesRenderer.kt @@ -49,6 +49,8 @@ class GlassesPose { var tiltPitchDegrees = 0f var shiftX = 0f var shiftY = 0f + /** Sideways out of the light, in stage widths. */ + var slide = 0f /** Its own slow drift, so it never looks frozen. */ var idleYaw = 0f var idlePitch = 0f @@ -251,7 +253,7 @@ class GlassesRenderer(context: Context, private val light: LightFrame, private v } val signature = floatArrayOf( light.level, light.flash, light.core[0], light.core[1], light.core[2], - pose.foldLeft, pose.foldRight, pose.framing, pose.yawDegrees, pose.pitchDegrees, pose.tiltYawDegrees, pose.tiltPitchDegrees, pose.shiftX, pose.shiftY, + pose.foldLeft, pose.foldRight, pose.framing, pose.yawDegrees, pose.pitchDegrees, pose.tiltYawDegrees, pose.tiltPitchDegrees, pose.shiftX, pose.shiftY, pose.slide, pose.idleYaw, pose.idlePitch, pose.idleRoll, pose.idleLift, pose.gestureYaw, pose.led, pose.viewYaw, pose.viewPitch, pose.innerLed, pose.button, if (loaded) 1f else 0f, ) @@ -307,7 +309,8 @@ class GlassesRenderer(context: Context, private val light: LightFrame, private v val m = FloatArray(16) Matrix.setIdentityM(m, 0) // Folded they are flatter, so they settle a little lower into the light. - Matrix.translateM(m, 0, pose.shiftX, pose.shiftY + pose.idleLift - CLOSED_DROP * pose.framing, 0f) + // A stage width is twice the radius the camera fits. + Matrix.translateM(m, 0, pose.shiftX + pose.slide * 2 * FIT_RADIUS.toFloat(), pose.shiftY + pose.idleLift - CLOSED_DROP * pose.framing, 0f) Matrix.rotateM(m, 0, pose.idleRoll, 0f, 0f, 1f) Matrix.rotateM(m, 0, pose.pitchDegrees + pose.tiltPitchDegrees + pose.idlePitch + pose.viewPitch, 1f, 0f, 0f) Matrix.rotateM(m, 0, REST_YAW + pose.yawDegrees + pose.tiltYawDegrees + pose.idleYaw + pose.gestureYaw + pose.viewYaw, 0f, 1f, 0f) diff --git a/app/src/main/kotlin/com/macedonga/visor/stage/SharedStage.kt b/app/src/main/kotlin/com/macedonga/visor/stage/SharedStage.kt index c74cee81..21cf5c57 100644 --- a/app/src/main/kotlin/com/macedonga/visor/stage/SharedStage.kt +++ b/app/src/main/kotlin/com/macedonga/visor/stage/SharedStage.kt @@ -6,8 +6,10 @@ import androidx.compose.animation.core.animateFloatAsState import androidx.compose.animation.core.snap import androidx.compose.animation.core.spring import androidx.compose.animation.core.tween +import androidx.compose.foundation.Image import androidx.compose.foundation.gestures.detectHorizontalDragGestures import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.aspectRatio import androidx.compose.foundation.layout.BoxWithConstraints import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth @@ -28,13 +30,17 @@ import androidx.compose.runtime.setValue import androidx.compose.runtime.snapshotFlow import androidx.compose.runtime.withFrameNanos import androidx.compose.ui.Alignment +import androidx.compose.ui.BiasAlignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clipToBounds import androidx.compose.ui.geometry.Offset import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.ImageBitmap import androidx.compose.ui.graphics.TransformOrigin import androidx.compose.ui.graphics.graphicsLayer import androidx.compose.ui.input.pointer.pointerInput +import androidx.compose.ui.input.pointer.util.VelocityTracker +import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.layout.layout import androidx.compose.ui.unit.IntOffset import com.macedonga.visor.ui.animationsOff @@ -126,20 +132,69 @@ fun StageLight(look: LightLook) { } } -/** Dragging sideways turns the pair, and it springs back to rest. */ +/** The pairs a drag on the stage moves between: whether there is one [direction] (1 the next), and going there. */ +class StagePages(val can: (direction: Int) -> Boolean, val go: (direction: Int) -> Unit) + +/** + * Dragging sideways turns the pair, and it springs back to rest. With [pages] it does not turn: it follows the finger + * sideways, and a long drag or a fling takes it out of the light for the next one. + */ @Composable -fun Modifier.turnsThePair(onDetent: () -> Unit): Modifier { +fun Modifier.turnsThePair(pages: StagePages? = null, onDetent: () -> Unit): Modifier { val stage = LocalStage.current val scope = rememberCoroutineScope() val detent = rememberUpdatedState(onDetent) + val paging = rememberUpdatedState(pages) return pointerInput(stage) { var lastDetent = 0 + var travel = 0f + var past = false + val tracker = VelocityTracker() val back = spring(dampingRatio = 0.62f, stiffness = Spring.StiffnessLow) + fun settle() { + scope.launch { stage.yaw.animateTo(0f, back) } + scope.launch { stage.slide.animateTo(0f, back) } + } detectHorizontalDragGestures( - onDragEnd = { scope.launch { stage.yaw.animateTo(0f, back) } }, - onDragCancel = { scope.launch { stage.yaw.animateTo(0f, back) } }, + onDragStart = { + travel = 0f + past = false + tracker.resetTracking() + }, + onDragEnd = { + val pages = paging.value + val fling = tracker.calculateVelocity().x + val direction = when { + pages == null -> 0 + travel < -size.width * PAGE_AT || fling < -PAGE_FLING -> 1 + travel > size.width * PAGE_AT || fling > PAGE_FLING -> -1 + else -> 0 + } + if (direction != 0 && pages!!.can(direction)) { + if (!past) detent.value() + scope.launch { stage.page(direction) { pages.go(direction) } } + } else { + settle() + } + }, + onDragCancel = { settle() }, ) { change, dx -> change.consume() + tracker.addPosition(change.uptimeMillis, change.position) + travel += dx + val pages = paging.value + if (pages != null) { + // Towards a page it follows the finger; at the end of the row it barely moves. A tick says it will go. + val towards = if (travel < 0) 1 else -1 + val can = pages.can(towards) + scope.launch { stage.slide.snapTo(travel / size.width * if (can) FOLLOW else FOLLOW_AT_END) } + val nowPast = can && abs(travel) > size.width * PAGE_AT + if (nowPast != past) { + past = nowPast + if (nowPast) detent.value() + } + return@detectHorizontalDragGestures + } // Free for the first half turn either way, then it resists like a rubber band. val yaw = stage.yaw.value val next = (yaw + dx * DEGREES_PER_PX * (1f - (abs(yaw) / MAX_YAW).coerceIn(0f, 0.9f))).coerceIn(-MAX_YAW, MAX_YAW) @@ -181,6 +236,11 @@ fun SharedStage(stage: StageHolder, shown: Boolean) { } } LaunchedEffect(stage) { snapshotFlow { stage.yaw.value }.collect { stage.pose.yawDegrees = it; stage.renderer.invalidate() } } + LaunchedEffect(stage) { snapshotFlow { stage.slide.value }.collect { stage.pose.slide = it; stage.renderer.invalidate() } } + // At once, not faded: a page changes while the pair is out of sight, and a fade would show both as it slides in. + val picture = stage.picture.takeIf { !stage.vacant } + val pictured = if (picture != null) 1f else 0f + val vacant = if (stage.vacant) 1f else 0f val curtain by animateFloatAsState(if (shown && newest != null) 1f else 0f, if (shown) snap() else tween(CURTAIN_MS), label = "stage") BoxWithConstraints(Modifier.fillMaxSize()) { @@ -214,16 +274,45 @@ fun SharedStage(stage: StageHolder, shown: Boolean) { }, ) { LightPool(stage.light, Offset(0.5f, POOL_Y), Size(0.21f, 0.058f), Size(0.17f, 0.02f), Modifier.fillMaxSize()) - GlassesStage(liveliness = newest?.liveliness ?: 0.25f, active = curtain > 0f, modifier = Modifier.fillMaxSize()) + GlassesStage( + liveliness = newest?.liveliness ?: 0.25f, active = curtain > 0f, + modifier = Modifier.fillMaxSize().graphicsLayer { alpha = 1f - maxOf(pictured, vacant) }, + ) + picture?.let { PairPicture(it, stage, Modifier.fillMaxSize()) } } } } } +/** Meta's picture of the pair, front on where the model would stand, sliding with it. */ +@Composable +private fun PairPicture(picture: ImageBitmap, stage: StageHolder, modifier: Modifier) { + BoxWithConstraints(modifier) { + Image( + picture, null, + Modifier + .align(BiasAlignment(0f, PICTURE_Y)) + .fillMaxWidth(PICTURE_WIDTH) + // Its own shape, so it fills the width rather than drawing at its pixel size. + .aspectRatio(picture.width.toFloat() / picture.height) + .graphicsLayer { translationX = stage.slide.value * size.width / PICTURE_WIDTH }, + contentScale = ContentScale.Fit, + ) + } +} + /** How the pair travels from one screen's place to the next. */ internal val HANDOVER_EASE = CubicBezierEasing(0.2f, 0f, 0f, 1f) internal const val HANDOVER_NS = 640_000_000L private const val CURTAIN_MS = 180 +// As wide as the model looks on the home, and low enough that it ends in the light where the model's temples do. +private const val PICTURE_WIDTH = 0.56f +private const val PICTURE_Y = 0.1f private const val DEGREES_PER_PX = 0.32f private const val MAX_YAW = 160f private const val DETENT_DEGREES = 30f +// A third of the width, or a flick, goes to the next pair. +private const val PAGE_AT = 0.33f +private const val PAGE_FLING = 1_400f +private const val FOLLOW = 0.9f +private const val FOLLOW_AT_END = 0.12f diff --git a/app/src/main/kotlin/com/macedonga/visor/stage/StageHolder.kt b/app/src/main/kotlin/com/macedonga/visor/stage/StageHolder.kt index e4d90dc5..613772c0 100644 --- a/app/src/main/kotlin/com/macedonga/visor/stage/StageHolder.kt +++ b/app/src/main/kotlin/com/macedonga/visor/stage/StageHolder.kt @@ -2,14 +2,23 @@ package com.macedonga.visor.stage import android.content.Context import androidx.compose.animation.core.Animatable +import androidx.compose.animation.core.FastOutLinearInEasing +import androidx.compose.animation.core.spring +import androidx.compose.animation.core.tween import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableFloatStateOf import androidx.compose.runtime.mutableStateListOf +import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue import androidx.compose.runtime.staticCompositionLocalOf import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.ImageBitmap import androidx.compose.ui.platform.LocalContext +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch /** * The glasses, their light and their pose for the life of the activity, above navigation: the model loads once and @@ -31,6 +40,23 @@ class StageHolder(private val context: Context, markAtLaunch: Boolean) { var fromLive = false /** Turned by a finger; springs back to 0. */ val yaw = Animatable(0f) + /** Sideways out of the light, in stage widths: a pair leaving for the next one. */ + val slide = Animatable(0f) + /** A pair Visor has no model for, or one not connected, shows as Meta pictures it; null shows the model. */ + var picture by mutableStateOf(null) + /** Nothing under the light: the home's page for adding another pair. */ + var vacant by mutableStateOf(false) + + /** The pair slides out [direction] (1 to the left), [swap] puts the next one up, and it slides in from the far side. */ + suspend fun page(direction: Int, swap: () -> Unit) { + coroutineScope { + launch { yaw.animateTo(0f, tween(PAGE_OUT_MS, easing = FastOutLinearInEasing)) } + slide.animateTo(-direction.toFloat(), tween(PAGE_OUT_MS, easing = FastOutLinearInEasing)) + } + swap() + slide.snapTo(direction.toFloat()) + slide.animateTo(0f, spring(dampingRatio = 0.86f, stiffness = 260f)) + } internal val spots = mutableStateListOf() internal val looks = mutableStateListOf() @@ -125,3 +151,5 @@ fun rememberStageHolder(markAtLaunch: Boolean): StageHolder { DisposableEffect(stage) { onDispose { stage.destroy() } } return stage } + +private const val PAGE_OUT_MS = 200 diff --git a/app/src/main/kotlin/com/macedonga/visor/ui/Icons.kt b/app/src/main/kotlin/com/macedonga/visor/ui/Icons.kt index e2b8d58e..8dffb5ee 100644 --- a/app/src/main/kotlin/com/macedonga/visor/ui/Icons.kt +++ b/app/src/main/kotlin/com/macedonga/visor/ui/Icons.kt @@ -45,6 +45,9 @@ object VisorIcons { val Case = icon("M6 8h12a4 4 0 0 1 0 8H6a4 4 0 0 1 0-8z", "M2.5 12h19") + /** The Neural Band: the strap seen edge on, the pod on top. */ + val Band = icon("M5 12.5a7 4.5 0 1 0 14 0a7 4.5 0 1 0-14 0", "M9.8 6.3h4.4a1 1 0 0 1 1 1v1.2H8.8V7.3a1 1 0 0 1 1-1z") + /** A turn back round the clock: what just happened. */ val Replay = icon("M5.2 13.5A7 7 0 1 0 7 7.1", "M6.6 3.6l.4 3.5 3.5-.4") @@ -195,6 +198,8 @@ object VisorIcons { "M7.5 11h4", "M13.5 11h3", "M7.5 14h7", ) + val Mirror = icon("M4 5.5h16v11H4z", "M9.5 19.5h5", "M12 16.5v3") + val People = icon( "M9 9m-2.5 0a2.5 2.5 0 1 0 5 0a2.5 2.5 0 1 0-5 0", "M4 18.5a5 5 0 0 1 10 0", "M16 9.5m-2 0a2 2 0 1 0 4 0a2 2 0 1 0-4 0", "M15.5 14a4.2 4.2 0 0 1 4.5 4.5", diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 8bf34cfd..09a916b2 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -70,6 +70,8 @@ %d h fa %d g fa Importo %d%% + Visti %s + Non connessi Importo %d nuovo @@ -374,6 +376,9 @@ Visione live Privacy Accessi del telefono + Neural Band + Batteria + %1$d%% Cosa arriva agli occhiali, e cosa dicono ad alta voce. Cosa fanno il tasto di cattura e il touchpad sull\'asta destra. Come gli occhiali capiscono di essere indossati, e come risparmiano energia. @@ -422,6 +427,7 @@ Occhiali Occhiali di %s + %1$s, %2$s Occhiali Meta Montatura Colore @@ -450,9 +456,11 @@ Collegamento Collegati via Bluetooth LE Collegati via Bluetooth + Collegati via Wi-Fi Non collegati Scollega Collega + %1$d%% · custodia %2$d%% Account Meta Accesso a Meta fatto Nessun accesso a Meta @@ -549,6 +557,13 @@ Per capire le domande successive e Live AI con il riconoscimento del telefono. Il microfono del telefono ascolta solo mentre tieni premuto il microfono in Meta AI. Contatti Così gli occhiali possono dire da chi arriva un messaggio. + Foto + Video + Allegato + SMS + Così gli occhiali possono mostrare i tuoi SMS e inviarne uno. + Registro chiamate + Così gli occhiali possono mostrare le tue chiamate recenti. Posizione Per annotare dove gli occhiali hanno fatto ogni foto. Posizione, sempre @@ -928,10 +943,61 @@ Riprova Indicazioni per %1$s Riproduci %1$s + Teleprompter + Testo + Scrivi o incolla il testo + Nessun testo + + %d parola + %d parole + + Dimensione del testo + Piccolo + Medio + Grande + Molto grande + Velocità di scorrimento + Disposizione + Scorrimento + Pagine + Mostra sugli occhiali + Aggiorna + Chiudi + Sugli occhiali + Connetti gli occhiali per mostrarlo + Gli occhiali non l\'hanno aperto + La batteria degli occhiali è troppo scarica + Gli occhiali sono troppo caldi + Gli occhiali non hanno abbastanza energia + Gesti + Impara i gesti + Bracciale %1$d%% + Connessione al bracciale + Configura il Neural Band nelle Impostazioni + Seleziona + Tocca insieme pollice e indice + Indietro + Tocca insieme pollice e medio + Scorri + Fai scorrere il pollice lungo il lato dell\'indice + Meta AI + Tocca due volte il pollice sul lato dell\'indice + Riattiva e sospendi + Tocca due volte insieme pollice e medio + App + Tieni uniti pollice e medio + Vestibilità + Larga + Aderente + Indossato al contrario Wi-Fi creato dagli occhiali Una VPN tiene Visor fuori dalla rete degli occhiali. Escludi Visor dalla VPN. Lo streaming si interrompe spesso Usa il Wi-Fi degli occhiali + Occhiali + Non associato + Associato, collegato + Associato, non collegato Lingua in cui parli La lingua degli occhiali diff --git a/app/src/main/res/values-it/strings_accounts.xml b/app/src/main/res/values-it/strings_accounts.xml new file mode 100644 index 00000000..96b5791e --- /dev/null +++ b/app/src/main/res/values-it/strings_accounts.xml @@ -0,0 +1,23 @@ + + + Account + Account Meta + WhatsApp + Facebook + Instagram + Collega + Collegato + Non collegato + Scansiona il codice sugli occhiali da WhatsApp, Dispositivi collegati. + Fatto + Accedi prima a Meta + Meta non raggiungibile + Gli occhiali non hanno risposto + Gli occhiali lo hanno rifiutato + Collegato come %1$s + %1$s, disattivato per gli occhiali + Collegamento account + Collega %1$s ai tuoi occhiali + Ricollega %1$s sui tuoi occhiali + Te lo chiedono gli occhiali. Tocca per aprire Account. + diff --git a/app/src/main/res/values-it/strings_calls.xml b/app/src/main/res/values-it/strings_calls.xml new file mode 100644 index 00000000..bc1fef4a --- /dev/null +++ b/app/src/main/res/values-it/strings_calls.xml @@ -0,0 +1,5 @@ + + + Effettua e rispondi alle chiamate + Così gli occhiali possono effettuare e rispondere alle chiamate su questo telefono. + diff --git a/app/src/main/res/values-it/strings_captions.xml b/app/src/main/res/values-it/strings_captions.xml new file mode 100644 index 00000000..efb1ba28 --- /dev/null +++ b/app/src/main/res/values-it/strings_captions.xml @@ -0,0 +1,21 @@ + + + Sottotitoli + Modalità + Sottotitoli + Traduzione + Lingua + Tu parli + Gli altri parlano + Da %1$s a %2$s + Ascoltato + In ascolto… + Avvia sugli occhiali + Ferma + Sugli occhiali + Connetti gli occhiali per avviare + Avvia + Scarica · %1$s + Non installato: %1$s + Installazione sugli occhiali + diff --git a/app/src/main/res/values-it/strings_firmware.xml b/app/src/main/res/values-it/strings_firmware.xml new file mode 100644 index 00000000..548872e2 --- /dev/null +++ b/app/src/main/res/values-it/strings_firmware.xml @@ -0,0 +1,15 @@ + + + Firmware + Versione + Aggiornamento + Cerca aggiornamenti + Ricerca + Aggiornato + Installa + Installare %1$s? Gli occhiali si riavviano al termine. + Installazione + Installazione, %1$d%% + Installato. Gli occhiali si riavviano. + Errore: %1$s + diff --git a/app/src/main/res/values-it/strings_mirror.xml b/app/src/main/res/values-it/strings_mirror.xml new file mode 100644 index 00000000..3d4d8ab1 --- /dev/null +++ b/app/src/main/res/values-it/strings_mirror.xml @@ -0,0 +1,19 @@ + + + Specchio + Avvia + Ferma + Inattivo + Avvio… + In diretta + Connettiti prima agli occhiali. + Leggi prima la configurazione degli occhiali. + Questi occhiali non hanno l\'interruttore dello schermo. + Gli occhiali non hanno accettato l\'interruttore: %1$s + Gli occhiali non hanno risposto. + Lo schermo degli occhiali è spento. + Gli occhiali sono andati in timeout. + Gli occhiali hanno rifiutato lo schermo condiviso. + Riavvia gli occhiali una volta per attivare la proiezione. + Il Wi-Fi con gli occhiali non si è attivato: %1$s + diff --git a/app/src/main/res/values-it/strings_navigation.xml b/app/src/main/res/values-it/strings_navigation.xml new file mode 100644 index 00000000..d7a90c5d --- /dev/null +++ b/app/src/main/res/values-it/strings_navigation.xml @@ -0,0 +1,4 @@ + + + mappe + diff --git a/app/src/main/res/values-it/strings_requests.xml b/app/src/main/res/values-it/strings_requests.xml new file mode 100644 index 00000000..7507a677 --- /dev/null +++ b/app/src/main/res/values-it/strings_requests.xml @@ -0,0 +1,12 @@ + + + Richieste + Qui compare ciò che gli occhiali chiedono al telefono di scaricare. + Link copiato + in corso + origine %1$d + tunnel + Meta + posizione + chiamate + diff --git a/app/src/main/res/values-it/strings_transcript.xml b/app/src/main/res/values-it/strings_transcript.xml new file mode 100644 index 00000000..53428fbe --- /dev/null +++ b/app/src/main/res/values-it/strings_transcript.xml @@ -0,0 +1,4 @@ + + + Condividi + diff --git a/app/src/main/res/values-it/strings_webapps.xml b/app/src/main/res/values-it/strings_webapps.xml new file mode 100644 index 00000000..4091e2f9 --- /dev/null +++ b/app/src/main/res/values-it/strings_webapps.xml @@ -0,0 +1,21 @@ + + + App web + App web + Aggiungi + Apri + Rimuovi + Nome + Indirizzo (https) + Nessuna app web + Sfoglia + Sugli occhiali + Impossibile caricare il catalogo + Nessuna app web per questi occhiali + Aperta sugli occhiali + Gli occhiali non l\'hanno aperta + Usa un indirizzo https + Massimo 20 app web + Collega prima gli occhiali + Impossibile leggere le app web degli occhiali + diff --git a/app/src/main/res/values-it/strings_wifi.xml b/app/src/main/res/values-it/strings_wifi.xml new file mode 100644 index 00000000..69370622 --- /dev/null +++ b/app/src/main/res/values-it/strings_wifi.xml @@ -0,0 +1,15 @@ + + + Reti Wi-Fi + Reti Wi-Fi + Aggiungi rete + Rimuovi + Nome della rete + Password + Nessuna rete salvata + Non riesco a leggere le reti + Salvata + Password sbagliata + Rimossa + Non riuscito + diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index e97f41f7..7d8f9cc9 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -559,6 +559,13 @@ Чтобы распознавать уточняющие вопросы и Live AI средствами телефона. Собственный микрофон телефона слушает, только пока вы удерживаете микрофон в Meta AI. Контакты Чтобы очки могли сказать, от кого сообщение. + Фото + Видео + Вложение + SMS + Чтобы очки могли показывать ваши SMS и отправлять их. + Журнал звонков + Чтобы очки могли показывать ваши недавние звонки. Местоположение Чтобы отмечать, где очки сделали каждое фото. Местоположение всегда diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index fcdd7b67..b2ca0c2a 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -71,6 +71,8 @@ %d h ago %d d ago Importing %d%% + Seen %s + Not connected Importing %d new @@ -375,6 +377,9 @@ Live view Privacy Phone access + Neural Band + Battery + %1$d%% What reaches the glasses, and what they say out loud. What the capture button and the touchpad on the right temple do. How the glasses notice they are on, and how they save power. @@ -423,6 +428,7 @@ Glasses %s\'s glasses + %1$s, %2$s Meta glasses Frame Colour @@ -451,9 +457,11 @@ Connection Connected over Bluetooth LE Connected over Bluetooth + Connected over Wi-Fi Not connected Disconnect Connect + %1$d%% · case %2$d%% Meta account Logged in to Meta Not logged in to Meta @@ -550,6 +558,13 @@ To understand follow-ups and Live AI with the phone\'s recogniser. The phone\'s own microphone listens only while you hold the mic in Meta AI. Contacts So the glasses can say who a message is from. + Photo + Video + Attachment + Text messages + So the glasses can show your texts and send one. + Call history + So the glasses can show your recent calls. Location To note where the glasses took each photo. Location, all the time @@ -930,10 +945,61 @@ Try again Directions to %1$s Play %1$s + Teleprompter + Script + Write or paste your script + No script + + %d word + %d words + + Text size + Small + Medium + Large + Very large + Scroll speed + Layout + Scroll + Pages + Show on glasses + Update + Close + On the glasses + Connect the glasses to show it + The glasses did not open it + The glasses\' battery is too low + The glasses are too warm + The glasses are short of power + Gestures + Learn the gestures + Band %1$d%% + Connecting to the band + Set up the Neural Band in Settings + Select + Tap your thumb and index finger together + Back + Tap your thumb and middle finger together + Scroll + Swipe your thumb along the side of your index finger + Meta AI + Double tap your thumb on the side of your index finger + Wake and sleep + Double tap your thumb and middle finger together + Apps + Hold your thumb and middle finger together + Fit + Loose + Snug + Worn the wrong way round Glasses host the Wi-Fi A VPN keeps Visor off the glasses\' network. Exclude Visor from it. The stream keeps dropping Use the glasses\' Wi-Fi + Glasses + Not paired + Paired, connected + Paired, not connected Language you speak The glasses\' language diff --git a/app/src/main/res/values/strings_accounts.xml b/app/src/main/res/values/strings_accounts.xml new file mode 100644 index 00000000..6fa5c84c --- /dev/null +++ b/app/src/main/res/values/strings_accounts.xml @@ -0,0 +1,23 @@ + + + Accounts + Meta account + WhatsApp + Facebook + Instagram + Link + Linked + Not linked + Scan the code on the glasses from WhatsApp, Linked devices. + Done + Log in to Meta first + Meta could not be reached + The glasses did not answer + The glasses refused it + Linked as %1$s + %1$s, turned off for glasses + Account linking + Link %1$s to your glasses + Reconnect %1$s on your glasses + Your glasses asked for it. Tap to open Accounts. + diff --git a/app/src/main/res/values/strings_calls.xml b/app/src/main/res/values/strings_calls.xml new file mode 100644 index 00000000..173f0b7c --- /dev/null +++ b/app/src/main/res/values/strings_calls.xml @@ -0,0 +1,5 @@ + + + Place and answer calls + So the glasses can place and answer calls on this phone. + diff --git a/app/src/main/res/values/strings_captions.xml b/app/src/main/res/values/strings_captions.xml new file mode 100644 index 00000000..ddc6974c --- /dev/null +++ b/app/src/main/res/values/strings_captions.xml @@ -0,0 +1,21 @@ + + + Captions + Mode + Captions + Translation + Language + You speak + They speak + %1$s to %2$s + Heard + Listening… + Start on glasses + Stop + On the glasses + Connect the glasses to start + Start + Download · %1$s + Did not install: %1$s + Installing on the glasses + diff --git a/app/src/main/res/values/strings_firmware.xml b/app/src/main/res/values/strings_firmware.xml new file mode 100644 index 00000000..d3c7dcf6 --- /dev/null +++ b/app/src/main/res/values/strings_firmware.xml @@ -0,0 +1,15 @@ + + + Firmware + Version + Update + Check for update + Checking + Up to date + Install + Install %1$s? The glasses restart when it is done. + Installing + Installing, %1$d%% + Installed. The glasses restart. + Failed: %1$s + diff --git a/app/src/main/res/values/strings_mirror.xml b/app/src/main/res/values/strings_mirror.xml new file mode 100644 index 00000000..47726847 --- /dev/null +++ b/app/src/main/res/values/strings_mirror.xml @@ -0,0 +1,19 @@ + + + Mirror + Start + Stop + Idle + Starting… + Live + Connect to the glasses first. + Read the glasses\' configuration first. + These glasses have no screencast switch. + The glasses did not take the switch: %1$s + The glasses did not answer. + The glasses\' screen is asleep. + The glasses timed out. + The glasses refused the screencast. + Restart the glasses once to turn on mirroring. + Wi-Fi to the glasses did not come up: %1$s + diff --git a/app/src/main/res/values/strings_navigation.xml b/app/src/main/res/values/strings_navigation.xml new file mode 100644 index 00000000..186630f1 --- /dev/null +++ b/app/src/main/res/values/strings_navigation.xml @@ -0,0 +1,4 @@ + + + maps + diff --git a/app/src/main/res/values/strings_requests.xml b/app/src/main/res/values/strings_requests.xml new file mode 100644 index 00000000..f88feeed --- /dev/null +++ b/app/src/main/res/values/strings_requests.xml @@ -0,0 +1,13 @@ + + + Requests + What the glasses ask the phone to fetch shows here. + Link copied + running + source %1$d + tunnel + Meta + location + SMS + calls + diff --git a/app/src/main/res/values/strings_transcript.xml b/app/src/main/res/values/strings_transcript.xml new file mode 100644 index 00000000..96107217 --- /dev/null +++ b/app/src/main/res/values/strings_transcript.xml @@ -0,0 +1,4 @@ + + + Share + diff --git a/app/src/main/res/values/strings_webapps.xml b/app/src/main/res/values/strings_webapps.xml new file mode 100644 index 00000000..20ffb6c2 --- /dev/null +++ b/app/src/main/res/values/strings_webapps.xml @@ -0,0 +1,21 @@ + + + Web apps + Web apps + Add + Open + Remove + Name + Address (https) + No web apps + Browse + On the glasses + Could not load the catalogue + No web apps for these glasses + Opened on the glasses + The glasses did not open it + Use an https address + Up to 20 web apps + Connect the glasses first + Could not read the glasses\' web apps + diff --git a/app/src/main/res/values/strings_wifi.xml b/app/src/main/res/values/strings_wifi.xml new file mode 100644 index 00000000..c6aab527 --- /dev/null +++ b/app/src/main/res/values/strings_wifi.xml @@ -0,0 +1,15 @@ + + + Wi-Fi networks + Wi-Fi networks + Add network + Remove + Network name + Password + No saved networks + Could not read the networks + Saved + Wrong password + Removed + Failed + diff --git a/app/src/main/res/xml/shared_files.xml b/app/src/main/res/xml/shared_files.xml index 95a25138..ac6206ca 100644 --- a/app/src/main/res/xml/shared_files.xml +++ b/app/src/main/res/xml/shared_files.xml @@ -3,4 +3,6 @@ + + diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt index 1d5b584d..8c6fcea4 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Assistant.kt @@ -2,11 +2,13 @@ package com.macedonga.visor.core.assistant import android.content.Context import com.macedonga.visor.core.assistant.mcp.ToolAction +import com.macedonga.visor.core.assistant.mcp.ToolResult import com.macedonga.visor.core.connection.GlassesConnection import com.macedonga.visor.core.connection.GlassesLink import com.macedonga.visor.core.media.AirshipReceiver import com.macedonga.visor.core.media.ExtraFiles import com.macedonga.visor.core.media.NAMESPACE_SHARING +import com.macedonga.visor.core.media.Offer import com.macedonga.visor.core.media.Received import com.macedonga.visor.core.protocol.Thrift import com.macedonga.visor.core.protocol.ThriftWriter @@ -23,6 +25,7 @@ import java.time.LocalDate import java.util.Locale import java.util.UUID import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job @@ -46,6 +49,7 @@ import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import kotlinx.coroutines.withTimeout import kotlinx.coroutines.withTimeoutOrNull +import org.json.JSONObject enum class AssistantPhase { OFF, WAITING, REGISTERING, LISTENING, THINKING, SPEAKING } @@ -106,6 +110,8 @@ class Assistant( private val processingTimeoutS: () -> Long? = { null }, /** The phone's player, for pausing, resuming and skipping by voice. */ private val phone: Phone? = null, + /** The connected glasses have a display: each answer then comes with a card for it. */ + private val display: () -> Boolean = { false }, ) : Responder { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) @@ -176,7 +182,7 @@ class Assistant( } override fun note(text: String) = say(text) - }) + }, display) // The glasses' last language, kept so the speech model can be fetched at launch, before they connect. private val speechPrefs = context.getSharedPreferences("speech", Context.MODE_PRIVATE) @@ -287,6 +293,10 @@ class Assistant( // The text the glasses read in each question's photo, from their ev65. private val ocr = ConcurrentHashMap>() fun ocrOf(interaction: String): CompletableDeferred = ocr.getOrPut(interaction) { CompletableDeferred() } + // The glasses' tool calls waiting for their ev312, by JSON-RPC id; and device intents waiting for their ev381. + val calls = ConcurrentHashMap>() + val handovers = ConcurrentHashMap>>() + val callIds = AtomicInteger(1) } /** A photo this side is waiting for, for one interaction. */ @@ -316,8 +326,10 @@ class Assistant( init { // The assistant's photo comes over airship, in a namespace the media import refuses. - airship.extra = object : ExtraFiles { - override fun wants(namespace: String, size: Long): Boolean { + airship.extras += object : ExtraFiles { + override fun wants(offer: Offer): Boolean { + val namespace = offer.namespace + val size = offer.size val waiting = pending != null && size < 20_000_000 || namespace == NAMESPACE_SHARING && store.config.value.enabled && size < EARLY_MAX_BYTES if (waiting) say("Taking ${size / 1000} KB in namespace \"$namespace\" for the photo") @@ -594,10 +606,26 @@ class Assistant( say("The glasses took the photo: ${describe(event.body)}") } CustomWakeWord.EV_RESULT -> wakeWord.onResult(runCatching { Thrift.loads(event.body) }.getOrDefault(emptyMap())) + EV_MCP_REQUEST_ACK -> Unit + EV_MCP_RESPONSE -> glassesReply(l, event.body) + EV_REMOTE_INTENT_RESPONSE -> handoverReply(l, event.body) else -> say("Assistant: ${eventName(event.id)} (${event.id}), ${event.body.size} bytes ${describe(event.body)}") } } + /** A tool call's answer, ev312 {1 jsonContent}: handed to the call waiting for its JSON-RPC id. */ + private fun glassesReply(l: Link, body: ByteArray) { + val json = runCatching { Thrift.loads(body)[1].asText() }.getOrDefault("") + val id = runCatching { JSONObject(json).opt("id")?.toString() }.getOrNull() ?: return say("Assistant: an unreadable tool answer") + l.calls[id]?.complete(json) + } + + /** ev381 {1 requestId, 2 success, 3 message}: the glasses' answer to a device intent they were asked to run. */ + private fun handoverReply(l: Link, body: ByteArray) { + val f = runCatching { Thrift.loads(body) }.getOrNull() ?: return + l.handovers[f[1].asText()]?.complete((f[2] == true) to f[3].asText()) + } + /** * ev59 MwaVoiceInteractionCloseRequest {1 interactionId, 2 audioLogData {3 transcription}}: the glasses end a turn. One * they never asked about they did themselves, as "play something": it goes in the diary with its words. @@ -724,6 +752,9 @@ class Assistant( say("Group conversation: $intent") return } + // The Display's own tools for this turn, as Meta's server names them (ev57 field 54). + val advertised = (m[54] as? List<*>).orEmpty().flatMap { server -> (server.asStruct()[2] as? List<*>).orEmpty().map { it.asText() } } + val glasses = if (display()) glassesToolbox(advertised) { full, input -> glassesCall(l, full, input) } else null when { // The glasses' own words for giving up. waiting != null && intent in CANCEL_INTENTS -> { @@ -735,9 +766,13 @@ class Assistant( sendReply(l, interaction, transcript, waiting.conversation, slot(slots, "SL:CONTENT_EXACT").ifEmpty { transcript }) } intent == "IN:REPLY_MESSAGE" -> scope.launch { replyRequested(l, interaction, transcript, slot(slots, "SL:CONTENT_EXACT")) } + // A device action the glasses understood: they run it, and the model answers only if they do not. + display() && intent in DEVICE_INTENTS -> scope.launch { + if (!handOver(l, interaction, transcript, body, intent)) answer(l, interaction, transcript, visual, glasses) + } // Left to the server by the glasses: the phone's player does them at once. phone != null && intent in MEDIA_CONTROLS -> scope.launch { mediaControl(l, interaction, transcript, phone, MEDIA_CONTROLS.getValue(intent)) } - else -> scope.launch { answer(l, interaction, transcript, visual) } + else -> scope.launch { answer(l, interaction, transcript, visual, glasses) } } } @@ -781,6 +816,50 @@ class Assistant( ) } + /** + * A tool call on the glasses: ev311 {1 jsonContent} with a JSON-RPC id, answered by ev312 with the same id. Only + * while the turn is open, which is while the model is asked. + */ + private suspend fun glassesCall(l: Link, full: String, input: JSONObject): ToolResult { + val m0 = l.m0 ?: return ToolResult("the glasses are not connected", true) + val id = l.callIds.getAndIncrement().toString() + val reply = CompletableDeferred() + l.calls[id] = reply + try { + val params = JSONObject().put("name", full).put("arguments", input) + val request = JSONObject().put("jsonrpc", "2.0").put("id", id.toInt()).put("method", "tools/call").put("params", params) + l.mux.send(m0, netEvent(EV_MCP_REQUEST, ThriftWriter().str(1, request.toString()).done()), messageType = MT_NET_EVENT) + val json = withTimeoutOrNull(MCP_WAIT_MS) { reply.await() } + say(if (json == null) "The glasses did not answer $full" else "The glasses answered $full") + return json?.let(::glassesResult) ?: ToolResult("no answer from the glasses", true) + } finally { + l.calls.remove(id) + } + } + + /** + * A device intent the glasses understood, handed back as ev380 {1 requestId, 2 interactionId, 3 intent}, the intent + * as they sent it. True when they ran it and the wearer hears it done; false leaves the turn to the model. + */ + private suspend fun handOver(l: Link, interaction: String, transcript: String, body: ByteArray, intent: String): Boolean { + val m0 = l.m0 ?: return false + val raw = Thrift.rawStruct(body, 55) ?: return false + val requestId = UUID.randomUUID().toString() + val answer = CompletableDeferred>() + l.handovers[requestId] = answer + val (done, message) = try { + l.mux.send(m0, netEvent(EV_REMOTE_INTENT_REQUEST, ThriftWriter().str(1, requestId).str(2, interaction).struct(3, raw).done()), messageType = MT_NET_EVENT) + withTimeoutOrNull(INTENT_WAIT_MS) { answer.await() } ?: (false to "no answer") + } finally { + l.handovers.remove(requestId) + } + say("Glasses intent $intent handed back: ${if (done) "done" else "not done"} ($message)") + if (!done) return false + if (diary.turn(interaction) == null) record(TurnRecord(interaction, System.currentTimeMillis(), transcript)) + speakTurn(l, interaction, transcript, "Tell the wearer in two or three words that it is done.") + return true + } + /** Speak, as this interaction's answer, a sentence the model writes from [what] in the glasses' language. */ private suspend fun speakTurn(l: Link, interaction: String, transcript: String, what: String) { _phase.value = AssistantPhase.THINKING @@ -797,7 +876,7 @@ class Assistant( // --- answering ----------------------------------------------------------- - private suspend fun answer(l: Link, interaction: String, transcript: String, visual: Visual) { + private suspend fun answer(l: Link, interaction: String, transcript: String, visual: Visual, glasses: GlassesToolbox? = null) { val started = System.currentTimeMillis() record(TurnRecord(interaction, started, transcript)) _phase.value = AssistantPhase.THINKING @@ -827,7 +906,7 @@ class Assistant( // The glasses give up after their processing timeout; speech and streaming need the last seconds of it. val budget = (processingTimeoutS() ?: FACTORY_PROCESSING_S) * 1000 - DELIVERY_MS val left = (budget - (System.currentTimeMillis() - started)).coerceAtLeast(MIN_MODEL_MS) - withTimeout(left) { brain.answer(transcript, spoken(l), pastOf(interaction), image, note, canEnd = true) } + withTimeout(left) { brain.answer(transcript, spoken(l), pastOf(interaction), image, note, glasses = glasses, canEnd = true) } } catch (error: Exception) { problem = error.message ?: error::class.simpleName.orEmpty() Reply(fallback(spoken(l), store.config.value.apiKey.isBlank() && store.config.value.provider == Provider.CLAUDE)) @@ -1004,12 +1083,15 @@ class Assistant( preamble(l, interaction, transcript, text) _phase.value = AssistantPhase.SPEAKING val voiceId = glassesVoice() - val answer = netEvent(EV_VOICE_ACTION_RESPONSE, varm(interaction, transcript, text, onGlasses = true, voiceId = voiceId)) + val card = if (display() && text.isNotEmpty()) answerCard(text) else null + val answer = netEvent(EV_VOICE_ACTION_RESPONSE, varm(interaction, transcript, text, onGlasses = true, voiceId = voiceId, card = card != null)) // An answer that asks something has the glasses listen for the reply, without the wake word. Any other answer // offers a follow-up, as the VARM's own shouldSmartFollowUp does. val complete = netEvent( EV_MWA_VOICE_INTERACTION_COMPLETE, - interactionComplete(interaction, text, expectsAnswer = listen && ASKS.containsMatchIn(text), followUp = listen && text.isNotEmpty()), + interactionComplete( + interaction, text, expectsAnswer = listen && ASKS.containsMatchIn(text), followUp = listen && text.isNotEmpty(), card = card, + ), ) // Without simplified forwarding the VARM is acted on as it lands, so the text must be there before it. for (event in if (l.simplified) listOf(answer, complete) else listOf(complete, answer)) { @@ -1238,6 +1320,8 @@ class Assistant( val WAKE_WORD_ONLY = Regex("^(hey|hi|ehi|ei|ok|okay)?[ ,]*(meta|mata|metta|matta|meda)[.!?]*$", RegexOption.IGNORE_CASE) const val MIN_MODEL_MS = 6_000L const val PHOTO_WAIT_MS = 8_000L + const val MCP_WAIT_MS = 8_000L + const val INTENT_WAIT_MS = 5_000L // The "Sharing" photo came 0.2 s before its question (issue #18); a low-res one is 13 KB. const val EARLY_IMAGE_MS = 10_000L const val EARLY_MAX_BYTES = 2_000_000L @@ -1256,6 +1340,12 @@ class Assistant( /** 1026 is either the bare locale string or `{1: locale}`. */ private val CANCEL_INTENTS = setOf("IN:CANCEL", "IN:NEVER_MIND", "IN:STOP", "IN:NEGATE_INTENT", "IN:CANCEL_MESSAGE") +/** The glasses' own device intents Visor hands back to them (ev380) on a Display: volume and brightness only. */ +private val DEVICE_INTENTS = setOf( + "IN:SET_VOLUME", "IN:INCREASE_VOLUME", "IN:DECREASE_VOLUME", "IN:MUTE_VOLUME", "IN:UNMUTE_VOLUME", + "IN:SET_BRIGHTNESS", "IN:INCREASE_BRIGHTNESS", "IN:DECREASE_BRIGHTNESS", "IN:MAXIMIZE_BRIGHTNESS", +) + /** The text of [name] in the glasses' slot map (field 26): each slot is a list of {1 name, 4 text, 5 start, 6 end}. */ internal fun slot(slots: Any?, name: String): String = ((slots as? Map<*, *>)?.get(name) as? List<*>)?.firstOrNull().asStruct()[4].asText().trim() diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Brain.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Brain.kt index 9edb2b64..c58348d9 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Brain.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Brain.kt @@ -45,7 +45,7 @@ data class Reply( * What one answer may use and what it did with it; [photo] is the glasses' JPEG that came with the question, [canEnd] * whether the model may end the conversation. */ -private class Turn(val box: Toolbox, val photo: ByteArray? = null, val canEnd: Boolean = false) { +private class Turn(val box: Toolbox, val photo: ByteArray? = null, val glasses: GlassesToolbox? = null, val canEnd: Boolean = false) { val remembered = ArrayList() val actions = ArrayList() var ends = false @@ -94,12 +94,13 @@ class Brain( image: ByteArray? = null, note: String = "", audience: Audience = Audience.GLASSES, + glasses: GlassesToolbox? = null, canEnd: Boolean = false, ): Reply = withContext(Dispatchers.IO) { val c = config() check(c.apiKey.isNotBlank() || c.provider == Provider.OPENAI_COMPATIBLE) { "no API key is set" } // The tools never stand in the way of an answer: without them the model still answers. - val turn = Turn(hub?.let { runCatching { it.toolbox() }.getOrNull() } ?: Toolbox(emptyList(), ""), image, canEnd) + val turn = Turn(hub?.let { runCatching { it.toolbox() }.getOrNull() } ?: Toolbox(emptyList(), ""), image, glasses, canEnd) val system = (if (audience == Audience.PHONE) PHONE_PROMPT else SYSTEM_PROMPT) + " It is ${ZonedDateTime.now().format(DateTimeFormatter.ofPattern("EEEE d MMMM yyyy, HH:mm", Locale.ENGLISH))}." + (if (audience == Audience.GLASSES) " The glasses are set to ${locale.getDisplayLanguage(Locale.ENGLISH)}." else "") + @@ -181,6 +182,8 @@ class Brain( action?.let(turn.actions::add) return result } + val glasses = turn.glasses + if (glasses != null) glasses.tools.find { it.short == name }?.let { return glasses.call(it.full, input) } val tool = turn.box.find(name) ?: return ToolResult("unknown tool", true) val (result, action) = hub?.call(tool, input, turn.photo) ?: return ToolResult("unavailable", true) action?.let(turn.actions::add) @@ -221,6 +224,7 @@ class Brain( if (turn?.canEnd == true) tools.put(JSONObject().put("name", END_TOOL).put("description", END_HELP).put("input_schema", NO_INPUT)) if (turn != null) phoneTools?.tools?.forEach { (name, help, input) -> tools.put(JSONObject().put("name", name).put("description", help).put("input_schema", input)) } turn?.box?.tools?.forEach { t -> tools.put(JSONObject().put("name", t.name).put("description", t.help()).put("input_schema", t.tool.schema)) } + turn?.glasses?.tools?.forEach { t -> tools.put(JSONObject().put("name", t.short).put("description", t.help).put("input_schema", t.schema)) } var said = "" repeat(MAX_TOOL_ROUNDS) { // Current models think before they answer, and the thinking counts toward max_tokens. @@ -278,6 +282,7 @@ class Brain( if (turn != null) phoneTools?.tools?.forEach { (name, help, input) -> put(function(name, help, input)) } // Some servers behind this API refuse a schema that names its own dialect. turn?.box?.tools?.forEach { t -> put(function(t.name, t.help(), JSONObject(t.tool.schema.toString()).apply { remove("\$schema") })) } + turn?.glasses?.tools?.forEach { t -> put(function(t.short, t.help, t.schema)) } }.takeIf { it.length() > 0 } var said = "" repeat(MAX_TOOL_ROUNDS) { diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/CustomWakeWord.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/CustomWakeWord.kt index 98e12a6b..eca03b2f 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/CustomWakeWord.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/CustomWakeWord.kt @@ -173,6 +173,10 @@ class CustomWakeWord internal constructor( return withTimeoutOrNull(ENROLL_MS) { waiting.await() } } + /** Delete the enrolled phrase, as the Meta app does: an empty utterance under a new id. The glasses answer with a result. */ + suspend fun clear(): Boolean = + send(EV_UTTERANCE, ThriftWriter().i32(1, 0).i32(2, 1).str(3, "").str(4, ByteArray(0)).str(5, UUID.randomUUID().toString()).done()) + /** Close the session; false when the close could not go out. */ suspend fun end(session: String, reason: Int = 0): Boolean = send(EV_SESSION_END, ThriftWriter().str(1, session).i32(2, reason).done()) @@ -204,6 +208,7 @@ class CustomWakeWord internal constructor( private const val BYTES_PER_MS = 32 // A voice near the glasses peaks around a third of full scale. private const val LOUD = 12_000f + private const val EV_UTTERANCE = 365 private const val EV_BATCH_ENROLLMENT = 386 private const val EV_SESSION_BEGIN = 390 private const val EV_SESSION_END = 391 diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/GlassesTools.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/GlassesTools.kt new file mode 100644 index 00000000..758cdb8c --- /dev/null +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/GlassesTools.kt @@ -0,0 +1,98 @@ +package com.macedonga.visor.core.assistant + +import com.macedonga.visor.core.assistant.mcp.ToolResult +import org.json.JSONArray +import org.json.JSONObject + +/* + * The Display's own tools, offered to the model for one turn. Meta's server names them on ev57 field 54; a call goes + * out as ev311 tools/call and comes back on ev312, with ev387 only acknowledging it. Names and help are Visor's own. + */ + +/** One of the glasses' tools as the model sees it: the short name it calls, the glasses' full name, help and schema. */ +class GlassesTool(val short: String, val full: String, val help: String, val schema: JSONObject) + +/** The glasses' tools for one turn, and the call that runs one of them. */ +class GlassesToolbox(val tools: List, val call: suspend (full: String, input: JSONObject) -> ToolResult) + +private class Known(val help: String, val props: Map = emptyMap(), val required: List = emptyList()) + +/** Keyed by the method name. A prop is a JSON type, or "string:a|b" for an enum. */ +private val KNOWN: Map = mapOf( + "setVolume" to Known("Set the glasses' volume, 0 to 100.", mapOf("percent" to "integer"), listOf("percent")), + "getVolume" to Known("Read the glasses' volume, 0 to 100."), + "setDisplayBrightness" to Known("Set the display brightness, 0 to 100.", mapOf("percent" to "integer"), listOf("percent")), + "getDisplayBrightness" to Known("Read the display brightness, 0 to 100."), + "setDoNotDisturb" to Known("Turn do not disturb on or off.", mapOf("enabled" to "boolean"), listOf("enabled")), + "getDoNotDisturbState" to Known("Say whether do not disturb is on."), + "getBatteryLevel" to Known("Read the glasses' battery level."), + "getNowPlaying" to Known("Read what is playing."), + "pauseMusic" to Known("Pause the music."), + "resumeMusic" to Known("Resume the music."), + "stopMusic" to Known("Stop the music."), + "skipNext" to Known("Skip to the next track."), + "skipPrevious" to Known("Go back to the previous track."), + "setShuffle" to Known("Turn shuffle on or off.", mapOf("enabled" to "boolean"), listOf("enabled")), + "setRepeatMode" to Known("Set repeat to off, all or one.", mapOf("mode" to "string:off|all|one"), listOf("mode")), + "likeCurrentTrack" to Known("Like the track playing."), + "startApp" to Known("Open an app on the glasses by its Android package name.", mapOf("packageName" to "string"), listOf("packageName")), + "openUrl" to Known("Open a web address in the glasses' browser.", mapOf("url" to "string"), listOf("url")), + "openMap" to Known("Show a place on the glasses' map.", mapOf("query" to "string"), listOf("query")), + "searchPlaces" to Known("Search for places near the wearer.", mapOf("query" to "string", "maxResults" to "integer"), listOf("query", "maxResults")), + "navigateTo" to Known( + "Start directions to a place, by coordinates.", + mapOf( + "latitude" to "number", "longitude" to "number", "destinationName" to "string", + "travelMode" to "string:WALKING|DRIVING|BICYCLING|TRANSIT", + ), + listOf("latitude", "longitude", "destinationName", "travelMode"), + ), + "stopNavigation" to Known("Stop the navigation."), + "getNavStatus" to Known("Say whether navigation is running."), + "getNextManeuver" to Known("Read the next turn of the route."), + "camera_capture" to Known( + "Take a photo with the glasses' camera and return where it is.", + mapOf("width" to "integer", "height" to "integer", "airshipUpload" to "boolean"), + ), +) + +/** + * The glasses' tools named in [advertised] (the full names from ev57 field 54), with short names the model can send. + * Null when the glasses advertise none. + */ +internal fun glassesToolbox(advertised: List, call: suspend (String, JSONObject) -> ToolResult): GlassesToolbox? { + if (advertised.isEmpty()) return null + val tools = mutableListOf() + for (full in advertised.distinct()) { + val method = full.substringAfterLast('.') + val base = "glasses_" + method.replace(Regex("[^A-Za-z0-9_-]"), "_").take(50) + val short = if (tools.any { it.short == base }) "${base}_${tools.size}" else base + val known = KNOWN[method] + tools += if (known == null) { + GlassesTool(short, full, "The glasses' own tool $method. Pass the arguments it takes.", JSONObject().put("type", "object")) + } else { + GlassesTool(short, full, known.help, schema(known)) + } + } + return GlassesToolbox(tools, call) +} + +private fun schema(known: Known): JSONObject { + val props = JSONObject() + for ((name, spec) in known.props) { + val type = spec.substringBefore(':') + val values = spec.substringAfter(':', "").takeIf { it.isNotEmpty() }?.split('|') + props.put(name, JSONObject().put("type", type).apply { if (values != null) put("enum", JSONArray(values)) }) + } + return JSONObject().put("type", "object").put("properties", props).put("required", JSONArray(known.required)) +} + +/** A tools/call answer (ev312): a JSON-RPC error, or a result whose text the model reads. */ +internal fun glassesResult(json: String): ToolResult { + val reply = runCatching { JSONObject(json) }.getOrNull() ?: return ToolResult("unreadable answer from the glasses", true) + reply.optJSONObject("error")?.let { return ToolResult(it.optString("message").ifEmpty { "failed" }, true) } + val result = reply.optJSONObject("result") ?: return ToolResult("no result from the glasses", true) + val content = result.optJSONArray("content") ?: JSONArray() + val text = (0 until content.length()).mapNotNull { content.optJSONObject(it)?.optString("text")?.takeIf { s -> s.isNotEmpty() } } + return ToolResult(text.joinToString("\n").ifEmpty { "done" }, result.optBoolean("isError")) +} diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt index ec1ecf84..f90e08bf 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/LiveAi.kt @@ -37,7 +37,12 @@ enum class LiveAiPhase { OFF, STARTING, LISTENING, THINKING } * the microphone on s0 with their own speech recognition unloaded. The server acknowledges the interaction, * listens, and answers each utterance as a sub-interaction of it, with the camera's picture. */ -internal class LiveAi(private val context: Context, private val scope: CoroutineScope, private val host: LiveAiHost) { +internal class LiveAi( + private val context: Context, + private val scope: CoroutineScope, + private val host: LiveAiHost, + private val display: () -> Boolean, +) { private val _phase = MutableStateFlow(LiveAiPhase.OFF) val phase: StateFlow = _phase.asStateFlow() @@ -83,7 +88,9 @@ internal class LiveAi(private val context: Context, private val scope: Coroutine delay(20) host.event(EV_SERVER_CONNECTION_STATUS, ThriftWriter().str(1, id).bool(2, true).bool(3, false).i32(4, 2).done()) delay(20) - host.event(EV_KEEP_AIR_SESSION_ALIVE, ThriftWriter().str(1, "AsrStartListening").i32(2, 20_000).i32(3, nonce.getAndIncrement()).done()) + val keep = ThriftWriter().str(1, "AsrStartListening").i32(2, 20_000).i32(3, nonce.getAndIncrement()) + if (display()) keep.str(4, id) + host.event(EV_KEEP_AIR_SESSION_ALIVE, keep.done()) ears?.stop() ears = GlassesEars(context, locale, continuous = true, onUtterance = ::heard, onProblem = { host.note("Live AI: $it") }).also { it.start() } keepalive?.cancel() @@ -144,7 +151,9 @@ internal class LiveAi(private val context: Context, private val scope: Coroutine while (scope.isActive && active) { delay(KEEPALIVE_MS) host.event(EV_FRAME_KEEP_ALIVE, ThriftWriter().i64(1, System.currentTimeMillis()).i32(2, nonce.getAndIncrement()).done()) - host.event(EV_KEEP_AIR_SESSION_ALIVE, ThriftWriter().str(1, "LiveAiSession").i32(2, 20_000).i32(3, nonce.getAndIncrement()).done()) + val keep = ThriftWriter().str(1, "LiveAiSession").i32(2, 20_000).i32(3, nonce.getAndIncrement()) + if (display()) keep.str(4, interaction) + host.event(EV_KEEP_AIR_SESSION_ALIVE, keep.done()) } } diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt index f16b892e..9aec1e0a 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Oxide.kt @@ -62,6 +62,12 @@ internal const val EV_ON_PHONE_UNDERSTANDING = 342 internal const val EV_TTS_START_INFO = 388 internal const val EV_TTS_END_INFO = 389 internal const val EV_KEEP_AIR_SESSION_ALIVE = 392 +// The glasses' own tools and intents, run for a Display turn (see GlassesTools.kt). +internal const val EV_REMOTE_INTENT_REQUEST = 380 +internal const val EV_REMOTE_INTENT_RESPONSE = 381 +internal const val EV_MCP_REQUEST = 311 +internal const val EV_MCP_RESPONSE = 312 +internal const val EV_MCP_REQUEST_ACK = 387 /** The glasses' name for a NetEvent id. */ internal fun eventName(id: Int): String = EVENT_NAMES[id] ?: "event $id" @@ -129,7 +135,14 @@ private fun inflate(data: ByteArray): ByteArray { * request and interaction ids were replaced in the file with neutral ones of the same shape. * Splicing keeps every other byte; re-encoding it would change integer widths. */ -internal fun varm(interactionId: String, transcript: String, text: String, onGlasses: Boolean = false, voiceId: String? = null): ByteArray { +internal fun varm( + interactionId: String, + transcript: String, + text: String, + onGlasses: Boolean = false, + voiceId: String? = null, + card: Boolean = false, +): ByteArray { val raw = checkNotNull(Assistant::class.java.getResourceAsStream("varm_template.bin")) { "varm_template.bin is missing" } .use { it.readBytes() } val old = Thrift.loads(raw) @@ -144,19 +157,38 @@ internal fun varm(interactionId: String, transcript: String, text: String, onGla val spliced = Thrift.replaceStrings(raw, mapping) if (!onGlasses) return spliced // No audio follows: streamingBackTts (6) and isUsingPartialResponseStreaming (14) go false; ev60 then carries the text. - return listOf(6, 14).fold(spliced) { b, field -> Thrift.setBool(b, field, false) } + val quiet = listOf(6, 14).fold(spliced) { b, field -> Thrift.setBool(b, field, false) } + // expectedUxTemplate (24) tells the glasses a card is coming in ev60. + return if (card) Thrift.setBool(quiet, 24, true) else quiet +} + +/** + * A simple card for a Display, as ev60's onDeviceUxMessage carries it: base64 of a compact AssistantTemplate + * (type 0, a SimpleTemplate inside). The Display throws on a template without hideAfterMs. + */ +internal fun answerCard(text: String): String { + // The dialog card shows its title only, in a scrolling block: the answer. The Display shows the question itself. + val simple = ThriftWriter().str(1, text).done() + val hideAfterMs = (8_000L + 60L * text.length).coerceAtMost(30_000L) + val template = ThriftWriter().i32(1, 0).str(2, simple).str(9, text).i64(10, hideAfterMs).done() + return java.util.Base64.getEncoder().encodeToString(template) } /** * ev60 MwaVoiceInteractionComplete, which ends the turn: ttsMessage, fulfillment type 1 (server), requiresAnswer when * they should listen for a reply, shouldSmartFollowUp (11) when they may listen for another question. With [streamed] * false the glasses speak the text themselves; true marks it as already played from the stream (7 and 8). - * Status 1000 is success. + * Status 1000 is success. A [card] goes in onDeviceUxMessage (field 27) for the Display to draw. */ internal fun interactionComplete( - interactionId: String, text: String, expectsAnswer: Boolean = false, followUp: Boolean = false, streamed: Boolean = false, + interactionId: String, text: String, expectsAnswer: Boolean = false, followUp: Boolean = false, card: String? = null, + streamed: Boolean = false, ): ByteArray { - val result = ThriftWriter().str(1, text).i32(2, 1).bool(3, expectsAnswer).bool(4, false).bool(7, streamed).bool(8, streamed) - .bool(11, followUp).done() - return ThriftWriter().str(1, interactionId).i32(2, 1000).str(3, "").struct(4, result).done() + var result = ThriftWriter().str(1, text).i32(2, 1).bool(3, expectsAnswer).bool(4, false).bool(7, streamed).bool(8, streamed) + .bool(11, followUp) + if (card != null) { + val ux = ThriftWriter().listOfBinaries(1, listOf(card.toByteArray(Charsets.UTF_8))).str(2, interactionId).done() + result = result.struct(27, ux) + } + return ThriftWriter().str(1, interactionId).i32(2, 1000).str(3, "").struct(4, result.done()).done() } diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Captions.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Captions.kt new file mode 100644 index 00000000..7b87a409 --- /dev/null +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Captions.kt @@ -0,0 +1,175 @@ +package com.macedonga.visor.core.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.LinkState +import java.util.UUID +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch + +enum class CaptionsPhase { IDLE, STARTING, LIVE, STOPPING } + +enum class CaptionsMode { CAPTIONS, TRANSLATION } + +/** One speech turn as the glasses last sent it: [text] to read, [source] what was said when it is a translation. [time] is when it arrived. */ +data class CaptionLine(val id: String, val text: String, val source: String?, val done: Boolean, val time: Long) + +/** [refusal] is the app manager's code when the glasses would not start it. [earlier] is the turns of the session before. */ +data class CaptionsState( + val phase: CaptionsPhase = CaptionsPhase.IDLE, + val mode: CaptionsMode = CaptionsMode.CAPTIONS, + val refusal: Long? = null, + val lines: List = emptyList(), + val earlier: List = emptyList(), +) + +/** + * The Display's live captions and translation, started as the Meta app starts them: svc28 StartApp (2000) for app 31, + * TRANSLATION, with `action.lst_lc.action` and the two languages as intent data, closed with StopApp (2002) and the + * matching stop action. The glasses recognise the speech themselves and send each turn back on svc72 as a + * TwoWayTranslation; they say so when the wearer stops it there. + */ +class Captions(private val connection: GlassesConnection) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val _state = MutableStateFlow(CaptionsState()) + val state: StateFlow = _state.asStateFlow() + private var wait: Job? = null + + init { + scope.launch { + connection.serviceData.collect { data -> + when (data.serviceId) { + SVC_COMPANION -> fromAppManager(data.messageType, runCatching { Wire.parse(data.payload).varint(1) }.getOrDefault(0L)) + SVC_TRANSLATION -> runCatching { fromSession(data.messageType, Wire.parse(data.payload)) } + } + } + } + scope.launch { + connection.state.map { it.link == LinkState.READY }.distinctUntilChanged().collect { up -> + if (!up) _state.update { it.copy(phase = CaptionsPhase.IDLE) } + } + } + } + + /** Start [mode] for a wearer speaking [self] with others speaking [other], codes such as "en"; the same for captions. */ + fun start(mode: CaptionsMode, self: String, other: String) { + if (connection.state.value.link != LinkState.READY) return + val data = intent("action.lst_lc.self_lang", self) + intent("action.lst_lc.other_lang", other) + + intent("action.lst_lc.request_id", UUID.randomUUID().toString()) + intent("action.lst_lc.action", action(mode, "start")) + _state.value = _state.value.restart(CaptionsPhase.STARTING, mode) + connection.submit { it.sendOnService(SVC_COMPANION, MT_START_APP, Wire.fieldVarint(1, APP_TRANSLATION) + data) } + settleAfter { if (it.phase == CaptionsPhase.STARTING) it.copy(phase = CaptionsPhase.LIVE) else it } + } + + fun stop() { + val now = _state.value + if (now.phase == CaptionsPhase.IDLE) return + _state.update { it.copy(phase = CaptionsPhase.STOPPING) } + val data = intent("action.lst_lc.action", action(now.mode, "stop")) + intent("action.lst_lc.request_id", UUID.randomUUID().toString()) + connection.submit { it.sendOnService(SVC_COMPANION, MT_STOP_APP, Wire.fieldVarint(1, APP_TRANSLATION) + data) } + settleAfter { if (it.phase == CaptionsPhase.STOPPING) it.copy(phase = CaptionsPhase.IDLE) else it } + } + + // Other apps' starts and stops are answered here too; only the one being waited for counts. + private fun fromAppManager(type: Int, code: Long) { + val phase = _state.value.phase + if (type == MT_START_APP_RESPONSE && phase == CaptionsPhase.STARTING) { + wait?.cancel() + _state.update { + if (code == Teleprompter.SNAPP_SUCCESS || code == Teleprompter.SNAPP_ALREADY_STARTED) it.copy(phase = CaptionsPhase.LIVE) + else it.copy(phase = CaptionsPhase.IDLE, refusal = code) + } + } else if (type == MT_STOP_APP_RESPONSE && phase == CaptionsPhase.STOPPING) { + wait?.cancel() + _state.update { it.copy(phase = CaptionsPhase.IDLE) } + } + } + + private fun fromSession(type: Int, fields: Wire.Fields) { + when (type) { + MT_TWO_WAY_TRANSLATION -> turn(fields, _state.value.mode)?.let { line -> + _state.update { now -> + val at = now.lines.indexOfLast { it.id == line.id } + val lines = if (at >= 0) now.lines.toMutableList().also { it[at] = line.copy(time = now.lines[at].time) } else (now.lines + line).takeLast(MAX_LINES) + now.copy(phase = if (now.phase == CaptionsPhase.IDLE) CaptionsPhase.LIVE else now.phase, lines = lines) + } + } + // Also how a session the wearer started on the glasses shows here. Its category is 1 for captions as well. + MT_INVITE_TO_JOIN -> _state.update { + if (it.phase == CaptionsPhase.IDLE) it.restart(CaptionsPhase.LIVE) else it.copy(phase = CaptionsPhase.LIVE) + } + MT_SESSION_STATUS -> if (fields.varint(1) == STATUS_IDLE) ended() + MT_END_FROM_FRAME, MT_STOPPING_FROM_FRAME -> ended() + } + } + + // The turns so far become the earlier session's, so a restart keeps the transcript. + private fun CaptionsState.restart(phase: CaptionsPhase, mode: CaptionsMode = CaptionsMode.CAPTIONS) = + CaptionsState(phase, mode, earlier = lines.ifEmpty { earlier }) + + // A start makes the glasses end the session before it first; that end is not this one's. + private fun ended() { + if (_state.value.phase == CaptionsPhase.STARTING) return + wait?.cancel() + _state.update { it.copy(phase = CaptionsPhase.IDLE) } + } + + /** + * Fields 12 and 11 are what was heard, final then partial, as it is recognised; 10 and 9 (and 4 and 5) what the + * Display shows, formatted or translated, with "..." while that catches up. A partial goes on with its own space. + */ + private fun turn(f: Wire.Fields, mode: CaptionsMode): CaptionLine? { + fun s(fn: Int) = f.bytes(fn).decodeToString() + fun shown(done: Int, partial: Int) = (s(done) + s(partial).takeUnless { it.trim() == PENDING }.orEmpty()).trim() + val heard = (s(12) + s(11)).trim() + val shown = shown(10, 9).ifEmpty { shown(4, 5) }.ifEmpty { s(2).trim() } + // As the glasses show it; what was heard stands in until their text comes. + val text = shown.ifEmpty { if (mode == CaptionsMode.CAPTIONS) heard else PENDING } + if (text.isEmpty() || text == PENDING && heard.isEmpty()) return null + val id = s(1).ifEmpty { "t${f.varint(6)}" } + val done = f.varint(13) != 0L || s(11).isEmpty() && s(9).isEmpty() && s(5).isEmpty() + return CaptionLine(id, text, heard.takeIf { mode == CaptionsMode.TRANSLATION && it.isNotEmpty() && it != text }, done, System.currentTimeMillis()) + } + + private fun settleAfter(change: (CaptionsState) -> CaptionsState) { + wait?.cancel() + wait = scope.launch { + delay(ANSWER_MS) + _state.update(change) + } + } + + private fun action(mode: CaptionsMode, verb: String) = + if (mode == CaptionsMode.TRANSLATION) "action.speech_translation.$verb" else "action.speech_caption.$verb" + + private fun intent(key: String, value: String): ByteArray = + Wire.fieldBytes(3, Wire.fieldBytes(1, key.encodeToByteArray()) + Wire.fieldBytes(5, value.encodeToByteArray())) + + private companion object { + const val SVC_COMPANION = 28 + const val SVC_TRANSLATION = 72 + const val MT_START_APP = 2000 + const val MT_START_APP_RESPONSE = 2001 + const val MT_STOP_APP = 2002 + const val MT_STOP_APP_RESPONSE = 2003 + const val MT_TWO_WAY_TRANSLATION = 1 + const val MT_INVITE_TO_JOIN = 2 + const val MT_END_FROM_FRAME = 4 + const val MT_SESSION_STATUS = 17 + const val MT_STOPPING_FROM_FRAME = 24 + const val STATUS_IDLE = 0L + const val APP_TRANSLATION = 31L + const val MAX_LINES = 300 + const val PENDING = "..." + const val ANSWER_MS = 7_000L + } +} diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/DeviceInfo.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/DeviceInfo.kt index 6443841c..6e014b2b 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/DeviceInfo.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/DeviceInfo.kt @@ -8,6 +8,8 @@ data class DeviceInfo( val serial: String, val deviceUuid: String = "", val model: String = "", + /** The wearables release version (field 9), e.g. 129.0.0.218.272: the store asks by it. */ + val wearablesRelease: String = "", ) private fun uuidShaped(text: String): Boolean = @@ -22,5 +24,6 @@ fun decodePeerDeviceInfo(payload: ByteArray): DeviceInfo { serial = String(fields.bytes(2), Charsets.ISO_8859_1), deviceUuid = if (uuidShaped(deviceUuid)) deviceUuid else "", model = String(fields.bytes(8), Charsets.ISO_8859_1), + wearablesRelease = String(fields.bytes(9), Charsets.ISO_8859_1), ) } diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt index ad7d537c..1dd90ab0 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesConnection.kt @@ -35,6 +35,8 @@ class GlassesConnection( private val hosts: SessionHostFactory, // Injectable so a test can keep the store reads on its own virtual clock. private val ioDispatcher: CoroutineDispatcher = Dispatchers.IO, + // False for a quick look at another pair, which leaves the choice of pair as it was. + private val chooses: Boolean = true, ) { private val _state = MutableStateFlow(GlassesState()) @@ -64,6 +66,11 @@ class GlassesConnection( private var job: Job? = null + private val _pair = MutableStateFlow(runCatching { store.currentSerial() }.getOrNull()) + + /** The serial of the pair Visor is for: the one connected, or chosen last. What Visor keeps per pair follows it. */ + val pair: StateFlow = _pair.asStateFlow() + private val _running = MutableStateFlow(false) /** True from [start] until [stop] has let go of the socket. */ @@ -81,31 +88,78 @@ class GlassesConnection( private val judged = AtomicBoolean(false) private var deadline: Job? = null + // svc28 state that came before svc100 said which model this is. Let through after a while if svc100 never decodes. + private val heldStatus = mutableListOf() + private var holdingStatus = true + private var statusTimer: Job? = null + + // While a lease holds the session on Wi-Fi, the Bluetooth target it came from. + @Volatile + private var bluetooth: GlassesTarget? = null + + /** True while the session runs on Wi-Fi for a lease, with Bluetooth to come back to. */ + val onWifi: Boolean get() = bluetooth != null + + /** True from [pause] until the session opens again: the Live or a media import has the glasses on Wi-Fi of its own. */ + @Volatile + var paused = false + private set + /** Choose a target, open a session on it, and keep the state current until [stop]. */ fun start(scope: CoroutineScope, override: GlassesTarget? = null) { + bluetooth = null + open(scope, override) + } + + /** + * Move the session to [target], a Wi-Fi link, as the Meta app's link manager does for a lease: the glasses take one + * session at a time, so it moves rather than opening a second, and every service moves with it. + */ + suspend fun moveToWifi(target: GlassesTarget) { + val current = _state.value.target ?: return + if (bluetooth == null) bluetooth = current.takeIf { it.link is GlassesLink.Bluetooth } ?: return + reopen(target) + } + + /** Back on the Bluetooth link the session left for [moveToWifi]. */ + suspend fun backToBluetooth() { + val target = bluetooth ?: return + bluetooth = null + reopen(target) + } + + private suspend fun reopen(target: GlassesTarget) { + val scope = lastScope ?: return + if (!_running.value) return + host?.stop() + host = null + job?.cancelAndJoin() + job = null + open(scope, target) + } + + private fun open(scope: CoroutineScope, override: GlassesTarget?) { + paused = false lastScope = scope lastOverride = override _running.value = true job = scope.launch { - // Both reads move together, not one withContext each: see the class doc for why - // they leave the caller's dispatcher at all. - val (bundles, bundle) = withContext(ioDispatcher) { - val bundles = store.all() - val bundle = if (override != null) { - bundles.firstOrNull { it.serial == override.serial } - } else { - store.load() - } - bundles to bundle + // The reads and the choice move together, not one withContext each: see the class doc for why + // they leave the caller's dispatcher at all. The pair connected to is the one chosen from now on. + val bundle = withContext(ioDispatcher) { + val bundle = if (override != null) store.all().firstOrNull { it.serial == override.serial } else store.chosen() + if (chooses) (override?.serial ?: bundle?.serial)?.let(store::choose) + bundle } val target = override ?: bundle?.let { GlassesTarget(GlassesLink.Bluetooth(it.glassesAddress), it.serial) } if (target == null) { - _state.value = GlassesState(reason = idleReason(bundles)) + _state.value = GlassesState(reason = "No glasses are set up on this phone yet. Set up glasses to start.") return@launch } + if (chooses) _pair.value = target.serial _state.value = GlassesState(target = target) val opened = hosts.create(target, bundle) host = opened @@ -116,6 +170,10 @@ class GlassesConnection( /** Let go of the link without ending the connection, so another session can have the glasses. */ suspend fun pause(why: String) { + // The session that takes over runs on Wi-Fi of its own; this one comes back on Bluetooth. + bluetooth?.let { lastOverride = it } + bluetooth = null + paused = true host?.stop() host = null job?.cancelAndJoin() @@ -126,7 +184,7 @@ class GlassesConnection( /** Reconnect after [pause], unless [stop] ran in between. */ fun resume() { val scope = lastScope ?: return - if (_running.value && job == null) start(scope, lastOverride) + if (_running.value && job == null) open(scope, lastOverride) } /** Ask svc28 again on the session's own status channel: the glasses answer with a fresh full snapshot. */ @@ -149,6 +207,7 @@ class GlassesConnection( * has stopped. */ suspend fun stop() { + bluetooth = null host?.stop() host = null job?.cancelAndJoin() @@ -199,13 +258,13 @@ class GlassesConnection( ) { canaryAnswered = true decodeInto(event) { copy(info = decodePeerDeviceInfo(it.payload)) } + heldStatus.forEach(::applyStatus) + heldStatus.clear() judge(host, bundle) } if (event.serviceId == SVC_STATUS && isDeviceStateType(event.messageType)) { - decodeInto(event) { - val update = decodeDeviceStateUpdate(it.messageType, it.payload) - copy(device = mergeDeviceState(device ?: DeviceState(), update.state)) - } + // How it is framed depends on the model, which svc100 names: what comes first waits for it. + if (holdingStatus && _state.value.info == null && heldStatus.size < HELD_STATUS) heldStatus += event else applyStatus(event) } // Airship's file chunks and the live stream's frames are their responders' business, // and thousands of them would push everything else out of this buffer. @@ -218,6 +277,11 @@ class GlassesConnection( } } + private fun applyStatus(event: ServiceData) = decodeInto(event) { + val update = decodeDeviceStateUpdate(it.messageType, it.payload, display = isDisplayBuild(info?.build)) + copy(device = mergeDeviceState(device ?: DeviceState(), update.state)) + } + /** Fold a decoded service message into the state, or report why it would not decode. */ private fun decodeInto(event: ServiceData, apply: GlassesState.(ServiceData) -> GlassesState) { try { @@ -228,6 +292,10 @@ class GlassesConnection( } private fun newAttempt() { + heldStatus.clear() + holdingStatus = true + statusTimer?.cancel() + statusTimer = null canaryAnswered = false judged.set(false) deadline?.cancel() @@ -249,6 +317,13 @@ class GlassesConnection( statusChannel = subscribeOn(connection, null) syncClock(connection) } + statusTimer = scope.launch { + delay(STATUS_HOLD_MILLIS) + if (canaryAnswered) return@launch + holdingStatus = false + heldStatus.forEach(::applyStatus) + heldStatus.clear() + } if (canaryAnswered) { judge(host, bundle) return @@ -295,19 +370,15 @@ class GlassesConnection( _state.value = _state.value.copy(diagnosis = checked.diagnosis) } - /** Why there is nothing to connect to, in words a person can act on. */ - private fun idleReason(bundles: List): String = when { - bundles.isEmpty() -> - "No glasses are set up on this phone yet. Set up glasses to start." - else -> - "More than one pair of glasses is set up on this phone. Choose which one to use." - } - companion object { /** 20 seconds, from `__main__.py`'s `_DEFAULT_TIMEOUT_SECONDS`. */ const val CANARY_TIMEOUT_MILLIS: Long = 20_000 + /** How long svc28 waits for svc100 before it is applied anyway. */ + private const val STATUS_HOLD_MILLIS: Long = 5_000 + private const val SVC_AIRSHIP = 20 private const val SVC_MEDIA_STREAMING_CONTROL = 51350 + private const val HELD_STATUS = 64 } } diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt new file mode 100644 index 00000000..3a2edf54 --- /dev/null +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/GlassesWifi.kt @@ -0,0 +1,131 @@ +package com.macedonga.visor.core.connection + +import com.macedonga.visor.core.protocol.Wire +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.onSubscription +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withTimeoutOrNull + +/** svc47, cloud configuration: the Wi-Fi networks the glasses keep. Its answers come on [GlassesConnection.serviceData]. */ +const val SVC_CLOUD_CONFIG = 47 +private const val MT_REMOVE_NETWORK = 3 +private const val MT_REMOVE_NETWORK_RESULT = 4 +private const val MT_QUERY_NETWORKS = 5 +private const val MT_QUERY_NETWORKS_RESULT = 6 +private const val MT_SAVE_NETWORK = 1 +private const val MT_SAVE_NETWORK_LIST = 18 +private const val MT_SAVE_NETWORK_RESULT = 2 + +private const val SVC_APPS = 28 +private const val MT_START_APP = 2000 +private const val MT_START_APP_RESPONSE = 2001 +private const val APP_CLOUD_CONFIGURATION = 6L + +private const val SECURITY_OPEN = 1L +private const val SECURITY_WPA_PSK = 3L +private const val RANDOMIZATION_PUBLIC = 1L +private const val REQUEST_CONNECT = 2L + +private const val ANSWER_MS = 15_000L + +/** Lists, saves and removes the Wi-Fi networks the glasses keep. One request at a time. */ +class GlassesWifi(private val connection: GlassesConnection) { + + private val lock = Mutex() + + // The glasses' cloud configuration app, started before the first request of a connection. + @Volatile + private var started = false + + private val _notes = MutableSharedFlow(extraBufferCapacity = 64) + + /** What was asked and answered, for the event log. */ + val notes: SharedFlow = _notes.asSharedFlow() + + /** The saved network names, or null without an answer. */ + suspend fun saved(): List? = lock.withLock { + val answer = ask(MT_QUERY_NETWORKS, ByteArray(0), MT_QUERY_NETWORKS_RESULT) + if (answer == null) null else readList(answer) + } + + /** The passphrase the glasses keep for [ssid] (QueryNetworksResult.Result 11), or null. A secret: never logged. */ + suspend fun keyOf(ssid: String): String? = lock.withLock { + val answer = ask(MT_QUERY_NETWORKS, ByteArray(0), MT_QUERY_NETWORKS_RESULT) ?: return null + val fields = Wire.parse(answer) + if (fields.varint(1) != 0L) return null + fields.repeatedBytes(2).map(Wire::parse).firstOrNull { it.bytes(1).decodeToString() == ssid }?.takeIf { it.has(11) }?.bytes(11)?.decodeToString() + } + + private fun readList(answer: ByteArray): List? { + val fields = Wire.parse(answer) + if (fields.varint(1) != 0L) { + note("list refused, status ${fields.varint(1)}") + return null + } + return fields.repeatedBytes(2).mapIndexedNotNull { i, entry -> + val network = Wire.parse(entry) + if (i == 0) note("first network has fields ${network.fieldNumbers()}") + if (network.has(1)) network.bytes(1).decodeToString().takeIf { it.isNotBlank() } else null + } + } + + /** Save a WPA2 network and have the glasses check it. Its status, 0 for saved; null without an answer. */ + suspend fun save(ssid: String, passphrase: String, open: Boolean): Int? = lock.withLock { + val network = Wire.fieldBytes(1, ssid.toByteArray()) + + (if (open) ByteArray(0) else Wire.fieldBytes(2, passphrase.toByteArray())) + + Wire.fieldVarint(3, if (open) SECURITY_OPEN else SECURITY_WPA_PSK) + + Wire.fieldVarint(5, 1) + + Wire.fieldVarint(6, RANDOMIZATION_PUBLIC) + + Wire.fieldVarint(9, REQUEST_CONNECT) + // The Display's firmware knows only the single SaveNetwork (type 1) and answers the list (18) with 0xD001. + val answer = ask(MT_SAVE_NETWORK, network, MT_SAVE_NETWORK_RESULT) + ?: ask(MT_SAVE_NETWORK_LIST, Wire.fieldBytes(1, network), MT_SAVE_NETWORK_RESULT) + answer?.let { Wire.parse(it).varint(1).toInt().also { status -> note("save $ssid: status $status") } } + } + + /** Forget a saved network. Its status, 0 for removed; null without an answer. */ + suspend fun remove(ssid: String): Int? = lock.withLock { + val answer = ask(MT_REMOVE_NETWORK, Wire.fieldBytes(1, ssid.toByteArray()), MT_REMOVE_NETWORK_RESULT) + answer?.let { Wire.parse(it).varint(1).toInt().also { status -> note("remove $ssid: status $status") } } + } + + /** Sends one svc47 message and returns the answer's payload, or null after [ANSWER_MS]. */ + private suspend fun ask(type: Int, payload: ByteArray, answer: Int): ByteArray? { + if (!started && !startApp()) return null + val reply = withTimeoutOrNull(ANSWER_MS) { + connection.serviceData + .onSubscription { connection.submit { it.sendOnService(SVC_CLOUD_CONFIG, type, payload) } } + .first { it.serviceId == SVC_CLOUD_CONFIG && it.messageType == answer } + .payload + } + if (reply == null) { + // The app may have stopped with the link; the next request starts it again. + started = false + note("no answer to type $type") + } + return reply + } + + /** Starts the cloud configuration app on svc28, as the Meta app does first. True when it runs. */ + private suspend fun startApp(): Boolean { + val code = withTimeoutOrNull(ANSWER_MS) { + connection.serviceData + .onSubscription { + connection.submit { it.sendOnService(SVC_APPS, MT_START_APP, Wire.fieldVarint(1, APP_CLOUD_CONFIGURATION)) } + } + .first { it.serviceId == SVC_APPS && it.messageType == MT_START_APP_RESPONSE } + .let { Wire.parse(it.payload).varint(1) } + } + started = code == 1L + if (!started) note("cloud configuration did not start, code ${code ?: "none"}") + return started + } + + private fun note(text: String) { + _notes.tryEmit(text) + } +} diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt index 9ad77803..628169a4 100644 --- a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Status.kt @@ -152,7 +152,7 @@ data class Led( val privacyLedActive: Boolean, ) -data class Tampering(val tampered: Boolean, val lux: Long, val indoor: Boolean) +data class Tampering(val tampered: Boolean, val lux: Float, val indoor: Boolean) data class Thermal( val health: String, @@ -206,6 +206,11 @@ data class DeviceState( val odc: String? = null, val overhead: Boolean? = null, val interactionKey: String? = null, + // The Display's Neural Band, as the Display reports it: battery (23), how tight it sits (30), which way round (34). + val bandBattery: Int? = null, + val bandCharging: Boolean? = null, + val bandFit: String? = null, + val bandOrientation: String? = null, ) { val worn: Boolean? get() = mount?.let { it == "DON" } } @@ -264,6 +269,10 @@ fun mergeDeviceState(base: DeviceState, update: DeviceState): DeviceState = Devi odc = update.odc ?: base.odc, overhead = update.overhead ?: base.overhead, interactionKey = update.interactionKey ?: base.interactionKey, + bandBattery = update.bandBattery ?: base.bandBattery, + bandCharging = update.bandCharging ?: base.bandCharging, + bandFit = update.bandFit ?: base.bandFit, + bandOrientation = update.bandOrientation ?: base.bandOrientation, ) fun isDeviceStateType(messageType: Int): Boolean = messageType in DEVICE_STATE_TYPES @@ -288,9 +297,20 @@ fun decodeDeviceState(payload: ByteArray): DeviceStatus { ) } -/** Decode any svc28 DeviceState message, full or partial, keeping only what it carries. */ -fun decodeDeviceStateUpdate(messageType: Int, payload: ByteArray): DeviceStateUpdate = - DeviceStateUpdate(messageType, DEVICE_STATE_TYPES.getValue(messageType), decodeDeviceStateFull(payload)) +/** + * Decode any svc28 DeviceState message, full or partial, keeping only what it carries. The Display sends it wrapped, + * HNDeviceState {1: stateCommon, 2: deviceLockState}, and Meta's app unwraps it by device type: [display]. + */ +fun decodeDeviceStateUpdate(messageType: Int, payload: ByteArray, display: Boolean = false): DeviceStateUpdate = + DeviceStateUpdate(messageType, DEVICE_STATE_TYPES.getValue(messageType), decodeDeviceStateFull(if (display) Wire.parse(payload).bytes(1) else payload)) + +/** Whether a build fingerprint, "Meta/greatwhite/greatwhite:14/...", is a Meta Ray-Ban Display's. */ +fun isDisplayBuild(build: String?): Boolean = build?.split('/')?.getOrNull(2)?.substringBefore(':')?.lowercase() in DISPLAY_DEVICES + +private val DISPLAY_DEVICES = setOf("greatwhite") + +private val BAND_FIT = mapOf(0L to "OFF", 1L to "LOOSE", 2L to "TIGHT", 3L to "UNKNOWN") +private val BAND_ORIENTATION = mapOf(0L to "UNKNOWN", 1L to "INCONCLUSIVE", 2L to "CORRECT", 3L to "INCORRECT") private fun text(fields: Wire.Fields, number: Int): String = String(fields.bytes(number), Charsets.UTF_8) @@ -361,7 +381,7 @@ fun decodeDeviceStateFull(payload: ByteArray): DeviceState { if (top.has(12)) { val t = Wire.parse(top.bytes(12)) state = state.copy(tampering = Tampering( - tampered = bool(t.varint(1)), lux = t.varint(2), indoor = bool(t.varint(3)), + tampered = bool(t.varint(1)), lux = t.float(2), indoor = bool(t.varint(3)), )) } if (top.has(13)) { @@ -400,6 +420,13 @@ fun decodeDeviceStateFull(payload: ByteArray): DeviceState { if (top.has(22)) { state = state.copy(registeredApps = top.repeatedBytes(22).map { Wire.parse(it).varint(1) }) } + if (top.has(23)) { + // EmgBatteryState {1 level, 2 charge: CHARGING 1, NOT_CHARGING 2}. + val b = Wire.parse(top.bytes(23)) + state = state.copy(bandBattery = b.varint(1).toInt(), bandCharging = b.varint(2) == 1L) + } + if (top.has(30)) state = state.copy(bandFit = name(BAND_FIT, Wire.parse(top.bytes(30)).varint(1))) + if (top.has(34)) state = state.copy(bandOrientation = name(BAND_ORIENTATION, Wire.parse(top.bytes(34)).varint(1))) if (top.has(25)) { val v = Wire.parse(top.bytes(25)) state = state.copy(volume = Volume( diff --git a/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Teleprompter.kt b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Teleprompter.kt new file mode 100644 index 00000000..16e94767 --- /dev/null +++ b/core/connection/src/main/kotlin/com/macedonga/visor/core/connection/Teleprompter.kt @@ -0,0 +1,131 @@ +package com.macedonga.visor.core.connection + +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.LinkState +import java.util.UUID +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch + +enum class PrompterPhase { IDLE, STARTING, SHOWING, CLOSING } + +/** [refusal] is the app manager's code when the glasses would not open it; [shown] and [look] what they have. */ +data class PrompterState( + val phase: PrompterPhase = PrompterPhase.IDLE, + val refusal: Long? = null, + val shown: String? = null, + val look: PrompterLook? = null, +) + +/** How the script shows: its size in sp on the glasses, its scroll speed (the glasses' default is 32), paged or scrolled. */ +data class PrompterLook(val sizeSp: Double = 35.0, val speed: Double = 32.0, val paged: Boolean = false, val followVoice: Boolean = false) + +/** + * The Display's teleprompter, run from the phone as the Meta app runs it: svc28 StartApp (2000) for app 35 with the + * script and its look as intent data, answered by 2001; a start while it shows replaces the text. StopApp (2002) + * closes it, answered by 2003. Following the voice is the glasses' own `asr_enabled`, which the Meta app never sends. + * The glasses do not say when the wearer closes it there; captions opening is seen, on svc72. + */ +class Teleprompter(private val connection: GlassesConnection) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val _state = MutableStateFlow(PrompterState()) + val state: StateFlow = _state.asStateFlow() + private var wait: Job? = null + + init { + scope.launch { + connection.serviceData.collect { data -> + // Captions or translation opening on the glasses takes the screen from it. + if (data.serviceId == SVC_TRANSLATION && data.messageType == MT_INVITE_TO_JOIN && _state.value.phase == PrompterPhase.SHOWING) { + wait?.cancel() + _state.value = PrompterState() + } + if (data.serviceId != SVC_COMPANION) return@collect + val code = runCatching { Wire.parse(data.payload).varint(1) }.getOrDefault(0L) + when (data.messageType) { + // Other apps' starts are answered here too; only the one being waited for counts. + MT_START_APP_RESPONSE -> if (_state.value.phase == PrompterPhase.STARTING) { + wait?.cancel() + _state.update { + if (code == SNAPP_SUCCESS || code == SNAPP_ALREADY_STARTED) it.copy(phase = PrompterPhase.SHOWING, refusal = null) + else PrompterState(refusal = code) + } + } + MT_STOP_APP_RESPONSE -> if (_state.value.phase == PrompterPhase.CLOSING) { + wait?.cancel() + _state.value = PrompterState() + } + } + } + } + scope.launch { + connection.state.map { it.link == LinkState.READY }.distinctUntilChanged().collect { up -> if (!up) _state.value = PrompterState() } + } + } + + /** Put [text] on the glasses, or replace what they show. */ + fun show(text: String, look: PrompterLook) { + if (connection.state.value.link != LinkState.READY) return + val data = intent("text", text) + intent("layout_type", if (look.paged) LAYOUT_PAGED else LAYOUT_SCROLL) + + intent("text_size_sp", look.sizeSp) + intent("scroll_speed", look.speed) + + intent("session_id", UUID.randomUUID().toString()) + intent("app_type", APP_TYPE_NATIVE) + + (if (look.followVoice) intent("asr_enabled", 1) else ByteArray(0)) + _state.update { it.copy(phase = PrompterPhase.STARTING, refusal = null, shown = text, look = look) } + connection.submit { it.sendOnService(SVC_COMPANION, MT_START_APP, Wire.fieldVarint(1, APP_TELEPROMPTER) + data) } + // No answer: the start went out, and the glasses may still be drawing it. + settleAfter { if (it.phase == PrompterPhase.STARTING) it.copy(phase = PrompterPhase.SHOWING) else it } + } + + fun close() { + if (_state.value.phase == PrompterPhase.IDLE) return + _state.update { it.copy(phase = PrompterPhase.CLOSING) } + connection.submit { it.sendOnService(SVC_COMPANION, MT_STOP_APP, Wire.fieldVarint(1, APP_TELEPROMPTER)) } + settleAfter { if (it.phase == PrompterPhase.CLOSING) PrompterState() else it } + } + + private fun settleAfter(change: (PrompterState) -> PrompterState) { + wait?.cancel() + wait = scope.launch { + delay(ANSWER_MS) + _state.update(change) + } + } + + /** One SNIntentData: the key, then the value in its type's field (3 int, 4 double, 5 bytes). */ + private fun intent(key: String, value: Int): ByteArray = entry(key, Wire.fieldVarint(3, value.toLong())) + + private fun intent(key: String, value: Double): ByteArray = entry(key, Wire.fieldDouble(4, value)) + + private fun intent(key: String, value: String): ByteArray = entry(key, Wire.fieldBytes(5, value.encodeToByteArray())) + + private fun entry(key: String, value: ByteArray): ByteArray = Wire.fieldBytes(3, Wire.fieldBytes(1, key.encodeToByteArray()) + value) + + companion object { + const val SNAPP_SUCCESS = 1L + const val SNAPP_NOT_ENOUGH_BATTERY = 6L + const val SNAPP_THERMAL_THROTTLING = 7L + const val SNAPP_ALREADY_STARTED = 12L + const val SNAPP_DENIED_BY_PEAK_POWER = 15L + private const val SVC_COMPANION = 28 + private const val SVC_TRANSLATION = 72 + private const val MT_INVITE_TO_JOIN = 2 + private const val MT_START_APP = 2000 + private const val MT_START_APP_RESPONSE = 2001 + private const val MT_STOP_APP = 2002 + private const val MT_STOP_APP_RESPONSE = 2003 + private const val APP_TELEPROMPTER = 35L + private const val LAYOUT_SCROLL = 0 + private const val LAYOUT_PAGED = 1 + private const val APP_TYPE_NATIVE = 0 + private const val ANSWER_MS = 7_000L + } +} diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/IdentityStore.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/IdentityStore.kt index 0aaa0925..a98665f8 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/IdentityStore.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/IdentityStore.kt @@ -21,6 +21,23 @@ class IdentityStore(directory: File, private val sealer: Sealer) { return bundles.firstOrNull { it.serial == serial } } + /** The pair last chosen, else the only one, else the newest set up. */ + fun chosen(): IdentityBundle? { + val bundles = all() + return bundles.firstOrNull { it.serial == currentSerial() } ?: bundles.singleOrNull() ?: bundles.maxByOrNull { it.createdAt } + } + + /** The serial last chosen, kept beside the store and unsealed: it is no secret, and reading it needs no key. */ + fun currentSerial(): String? = currentPath.takeIf { it.isFile }?.readText()?.trim()?.takeIf { it.isNotEmpty() } + + fun choose(serial: String) { + if (serial == currentSerial()) return + directory.mkdirs() + currentPath.writeText(serial) + } + + private val currentPath: File get() = File(directory, CURRENT_FILE) + /** The one owner key this account uses, or null before the first import. */ fun ownerKeyPem(): ByteArray? = all().firstOrNull()?.privateKeyPem @@ -41,6 +58,7 @@ class IdentityStore(directory: File, private val sealer: Sealer) { val bundles = all().filter { it.serial != serial } directory.mkdirs() writeAtomically(sealer.seal(bundlesToJson(bundles).toByteArray(Charsets.UTF_8))) + if (currentSerial() == serial) currentPath.delete() } /** DIVERGENCE: `store.py` calls `Path.write_bytes`, which truncates first. */ @@ -56,6 +74,7 @@ class IdentityStore(directory: File, private val sealer: Sealer) { companion object { const val STORE_FILE = "identities.sealed" + const val CURRENT_FILE = "current" } // DIVERGENCE: `migrate_legacy` does not port. diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/KeyHint.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/KeyHint.kt new file mode 100644 index 00000000..1b131035 --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/KeyHint.kt @@ -0,0 +1,46 @@ +package com.macedonga.visor.core.identity + +import java.io.ByteArrayInputStream +import java.security.KeyFactory +import java.security.MessageDigest +import java.security.cert.CertificateFactory +import java.security.interfaces.ECPublicKey +import java.security.spec.X509EncodedKeySpec + +/** The 64 raw bytes, X then Y, of the EC key in a device certificate, as svc36's IdentityResponse carries it. */ +fun devicePublicKey(certDer: ByteArray): ByteArray? = runCatching { + val cert = CertificateFactory.getInstance("X.509").generateCertificate(ByteArrayInputStream(certDer)) + point64(cert.publicKey as ECPublicKey) +}.getOrNull() + +/** A P-256 public key as 64 raw bytes, from raw, 0x04-prefixed or DER (SubjectPublicKeyInfo) form. */ +fun rawPublicKey(encoded: ByteArray): ByteArray? = when { + encoded.size == 64 -> encoded + encoded.size == 65 && encoded[0] == 0x04.toByte() -> encoded.copyOfRange(1, 65) + else -> runCatching { + point64(KeyFactory.getInstance("EC").generatePublic(X509EncodedKeySpec(encoded)) as ECPublicKey) + }.getOrNull() +} + +private fun point64(key: ECPublicKey): ByteArray = fixed32(key.w.affineX.toByteArray()) + fixed32(key.w.affineY.toByteArray()) + +/** + * Whether [hint] names [publicKey]: AirShield's key hint is a 4-byte salt then SHA256(SHA256(key) ‖ salt)[:8] + * (generateKeyHint, research PROTOCOL-REFERENCE). + */ +fun keyHintMatches(hint: ByteArray, publicKey: ByteArray): Boolean { + if (hint.size != 12 || publicKey.size != 64) return false + val salt = hint.copyOfRange(0, 4) + val sha = MessageDigest.getInstance("SHA-256") + val inner = sha.digest(publicKey) + sha.update(inner) + sha.update(salt) + return sha.digest().copyOfRange(0, 8).contentEquals(hint.copyOfRange(4, 12)) +} + +// A BigInteger's bytes carry a sign byte or lose leading zeros; a coordinate is always 32. +private fun fixed32(b: ByteArray): ByteArray = when { + b.size == 32 -> b + b.size > 32 -> b.copyOfRange(b.size - 32, b.size) + else -> ByteArray(32 - b.size) + b +} diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/appstore/AppCatalogue.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/appstore/AppCatalogue.kt new file mode 100644 index 00000000..ff16f2f7 --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/appstore/AppCatalogue.kt @@ -0,0 +1,54 @@ +package com.macedonga.visor.core.identity.appstore + +import com.macedonga.visor.core.identity.HttpPost +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.language.graphPost +import com.macedonga.visor.core.identity.token.Token +import com.macedonga.visor.core.identity.token.text +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put + +/** A web app in Meta's catalogue for the glasses. [id] is Meta's; the glasses key it when it is written. */ +data class CatalogueApp( + val id: String, + val name: String, + val iconUrl: String, + val url: String, + /** Base64 of Meta's SignedWebAppManifest. Null when the entry has none. */ + val manifest: String?, +) + +private const val CATALOGUE_DOC = "162073133818421355301019414873" + +/** + * WearablesAppStoreCatalogQuery, the store's list for [serial] running [osVersion], the glasses' wearables release + * version. Only web apps come back: an app with a web address; the others name a package and are not listed here. + */ +fun fetchAppCatalogue( + token: Token, + serial: String, + osVersion: String, + http: HttpPost = UrlConnectionPost, +): List { + val variables = buildJsonObject { + put("osVersion", osVersion) + put("serial", serial) + } + val (_, json) = graphPost(token, "WearablesAppStoreCatalogQuery", CATALOGUE_DOC, variables, http) + val edges = json.obj("data")?.obj("viewer")?.obj("user")?.obj("discovery_applications")?.get("edges") as? JsonArray + ?: return emptyList() + return edges.mapNotNull { edge -> + val node = (edge as? JsonObject)?.obj("node") ?: return@mapNotNull null + CatalogueApp( + id = node.text("id") ?: return@mapNotNull null, + name = node.text("display_name") ?: return@mapNotNull null, + iconUrl = node.text("pdp_icon_url").orEmpty(), + url = node.text("web_app_url")?.takeIf { it.isNotEmpty() } ?: return@mapNotNull null, + manifest = node.text("signed_web_app_manifest")?.takeIf { it.isNotEmpty() }, + ) + } +} + +private fun JsonObject.obj(key: String): JsonObject? = this[key] as? JsonObject diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/BandOffer.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/BandOffer.kt new file mode 100644 index 00000000..af0afb0b --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/BandOffer.kt @@ -0,0 +1,96 @@ +package com.macedonga.visor.core.identity.firmware + +import com.macedonga.visor.core.identity.HttpPost +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.language.graphPost +import com.macedonga.visor.core.identity.token.Token +import com.macedonga.visor.core.identity.token.TokenError +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.longOrNull +import kotlinx.serialization.json.put + +/** A Neural Band firmware image Meta offers: the zip's address, size, SHA-256 and the KEK it is wrapped with. */ +data class BandOffer( + val baseVersion: String, + val targetVersion: String, + val displayName: String, + val url: String, + val size: Long, + val checksum: String, + val kekId: Long?, +) + +/** The band's wrapped KEK as Meta gives it, base64, and the algorithm it names. */ +data class BandKek(val encrypted: String, val algorithm: String?) + +private const val SYSTEM_UPDATES_DOC = "29744781653741296994778085945" +private const val KEK_DOC = "313699925716490204858433749875" +private const val BAND_DEVICE_TYPE = "ota.ceres.user" + +/** + * FetchSystemUpdates, the Meta app's own query for the band's firmware: the full image newer than [firmware] (the + * band's version as a decimal string), or null when there is none. [token] is the AR token from the login. + */ +fun fetchBandOffer( + token: Token, + serial: String, + firmware: String, + http: HttpPost = UrlConnectionPost, +): BandOffer? { + val variables = buildJsonObject { + put("fetch_request", buildJsonObject { + put("topology_fetch_update_info", JsonArray(listOf(buildJsonObject { + put("custom_ota_fetch_type", "CERES_USER_INSIGHTS_NO_DEVICE_IDENTITY") + put("full_update_only", true) + put("device_serial_number", serial) + put("device_type", BAND_DEVICE_TYPE) + put("fw_version", firmware) + }))) + }) + } + val (status, json) = graphPost(token, "FetchSystemUpdates", SYSTEM_UPDATES_DOC, variables, http) + val root = json["data"]?.jsonObject?.get("xar_fetch_constellation_system_updates") as? JsonObject + ?: throw TokenError("Meta answered $status without updates") + val answers = (root["fetch_update_info_list"] as? JsonArray) ?: emptyList() + val offer = answers.firstNotNullOfOrNull { (it as? JsonObject)?.get("fetch_update_response") as? JsonObject } ?: return null + val url = offer.text("file_uri") ?: throw TokenError("Meta's band update has no file") + return BandOffer( + baseVersion = offer.text("base_version").orEmpty(), + targetVersion = offer.text("target_version").orEmpty(), + displayName = offer.text("build_number_display_name").orEmpty(), + url = url, + size = offer.long("file_size") ?: 0, + checksum = offer.text("file_checksum").orEmpty(), + kekId = offer.long("kek_id"), + ) +} + +/** FetchEncryptedKEKInfoByKEKId: the band's KEK for [kekId], wrapped for the band with [serial]. */ +fun fetchBandKek( + token: Token, + serial: String, + kekId: Long, + http: HttpPost = UrlConnectionPost, +): BandKek { + val variables = buildJsonObject { + put("request", buildJsonObject { + put("device_serial_number", serial) + put("kek_id", kekId) + }) + } + val (status, json) = graphPost(token, "FetchEncryptedKEKInfoByKEKId", KEK_DOC, variables, http) + val node = json["data"]?.jsonObject?.get("xar_fetch_encrypted_kek_info_by_kek_id") as? JsonObject + ?: throw TokenError("Meta answered $status without a key") + val encrypted = node.text("encrypted_kek") ?: throw TokenError("Meta's key has no value") + return BandKek(encrypted, node.text("encryption_algo")) +} + +private fun JsonObject.text(key: String): String? = (this[key] as? JsonPrimitive)?.contentOrNull + +private fun JsonObject.long(key: String): Long? = + (this[key] as? JsonPrimitive)?.let { it.longOrNull ?: it.contentOrNull?.toLongOrNull() } diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/FirmwareOffer.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/FirmwareOffer.kt new file mode 100644 index 00000000..11e897e6 --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/firmware/FirmwareOffer.kt @@ -0,0 +1,99 @@ +package com.macedonga.visor.core.identity.firmware + +import com.macedonga.visor.core.identity.HttpPost +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.language.constellationPost +import com.macedonga.visor.core.identity.token.Token +import com.macedonga.visor.core.identity.token.TokenError +import java.util.UUID +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.longOrNull +import kotlinx.serialization.json.put + +/** + * A full firmware image Meta offers for the glasses. The glasses fetch [url] themselves over their own Wi-Fi; the + * install message carries the fields the Meta app puts in it. + */ +data class FirmwareOffer( + val baseVersion: String, + val targetVersion: String, + /** Meta's name for the build; empty when Meta gives none. */ + val displayName: String, + val url: String, + val size: Long, + val checksum: String, + /** From the offer's install options: the payload's headers and offset inside the image. */ + val metaHeaders: String, + val payloadOffset: Long, + val kekId: Long?, + val kekHash: String, + /** The session the offer was asked under; the install names its update by it. */ + val sessionId: String, +) + +private const val UPDATES_DOC = "155904535318377635033579516369" + +/** + * FetchConstellationUpdates, the Meta app's own query for the glasses' firmware: the full image newer than + * [firmware], or null when there is none. [token] is the AR token from the login. + */ +fun fetchFirmwareOffer( + token: Token, + serial: String, + deviceType: String, + firmware: String, + http: HttpPost = UrlConnectionPost, +): FirmwareOffer? { + val sessionId = UUID.randomUUID().toString() + val info = buildJsonObject { + put("device_type", deviceType) + put("device_serial", serial) + put("device_firmware_version", firmware) + put("device_ota_session_id", sessionId) + put("device_feature_list", JsonArray(emptyList())) + put("device_artifact_list", JsonArray(emptyList())) + put("device_firmware_full_update_only", true) + put("device_firmware_rollout_token", JsonNull) + put("device_manage_mode", JsonNull) + // No constellation device is paired, so the app sends all three as null. + put("constellation_device_type", JsonNull) + put("constellation_device_firmware_version", JsonNull) + put("constellation_device_serial", JsonNull) + } + val (status, json) = constellationPost(token, "FetchConstellationUpdates", UPDATES_DOC, info, http) + val responses = json.path("data", "xar_constellation_updates")?.get("constellation_updates_response_info") as? JsonArray + ?: throw TokenError("Meta answered $status without updates") + val offer = responses.firstNotNullOfOrNull { (it.jsonObject["firmware_update_info"] as? JsonObject) } ?: return null + val url = offer.text("file_uri") ?: throw TokenError("Meta's firmware offer has no file") + // The options come as an object or as the JSON text of one. + val options = when (val raw = offer["extra_install_options"]) { + is JsonObject -> raw + is JsonPrimitive -> runCatching { Json.parseToJsonElement(raw.content).jsonObject }.getOrNull() + else -> null + } + return FirmwareOffer( + baseVersion = offer.text("base_version").orEmpty(), + targetVersion = offer.text("target_version").orEmpty(), + displayName = offer.text("build_number_display_name").orEmpty(), + url = url, + size = (offer["file_size"] as? JsonPrimitive)?.longOrNull ?: offer.text("file_size")?.toLongOrNull() ?: 0, + checksum = offer.text("file_checksum").orEmpty(), + metaHeaders = options?.text("headers").orEmpty(), + payloadOffset = (options?.get("payloadOffset") as? JsonPrimitive)?.longOrNull ?: 0, + kekId = (offer["kek_id"] as? JsonPrimitive)?.longOrNull, + kekHash = offer.text("kek_hash").orEmpty(), + sessionId = sessionId, + ) +} + +private fun JsonObject.path(vararg keys: String): JsonObject? = + keys.fold(this as JsonObject?) { o, k -> o?.get(k) as? JsonObject } + +private fun JsonObject.text(key: String): String? = (this[key] as? JsonPrimitive)?.contentOrNull diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/GlassesLook.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/GlassesLook.kt index 9cd87670..06603e44 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/GlassesLook.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/GlassesLook.kt @@ -1,5 +1,6 @@ package com.macedonga.visor.core.identity.language +import com.macedonga.visor.core.identity.token.checkSession import com.macedonga.visor.core.identity.HttpPost import com.macedonga.visor.core.identity.UrlConnectionPost import com.macedonga.visor.core.identity.pyUrlencode @@ -58,6 +59,7 @@ fun fetchGlassesLook(token: Token, serial: String, locale: String = "en_US", htt } val json = runCatching { Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject } .getOrElse { throw TokenError("Meta answered ${reply.status} with no JSON") } + checkSession(json) json["errors"]?.let { throw TokenError("Meta refused: ${it.toString().take(200)}") } val edges = (json["data"] as? JsonObject)?.get("xar_ar_ent_wearable_device_sku_query_by_serial_number") ?.let { it as? JsonObject }?.get("edges") as? JsonArray ?: return null diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt index f6e6834d..92a19876 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/LanguagePacks.kt @@ -1,5 +1,6 @@ package com.macedonga.visor.core.identity.language +import com.macedonga.visor.core.identity.token.checkSession import com.macedonga.visor.core.identity.HttpPost import com.macedonga.visor.core.identity.UrlConnectionPost import com.macedonga.visor.core.identity.pyUrlencode @@ -28,6 +29,10 @@ data class PackOffer( val url: String, val size: Long, val sha256: String, + /** What the glasses' OTA install names it by, e.g. `7506485332707669`. */ + val appId: String = "", + /** e.g. `RESOURCE_PACK`. */ + val type: String = "", ) const val LANGUAGE_FEATURE = "com.meta.constellation.feature.air.language_" @@ -35,7 +40,6 @@ const val LANGUAGE_FEATURE = "com.meta.constellation.feature.air.language_" /** Ray-Ban Meta Gen 2 on a user build, as the OTA service names it. */ const val GEN2_DEVICE_TYPE = "ota.hammerhead.user" -private const val GRAPH_URL = "https://ar-genai.graph.meta.com/graphql" private const val FETCH_UPDATES_DOC = "10649732309712548951633872342" /** @@ -65,13 +69,46 @@ fun fetchPackOffers( put("device_firmware_rollout_token", JsonNull) put("device_manage_mode", JsonNull) } + val (status, json) = constellationPost(token, "FetchConstellationFeatureUpdates", FETCH_UPDATES_DOC, info, http) + val responses = json.path("data", "xar_constellation_feature_updates")?.get("constellation_updates_response_info") as? JsonArray + ?: throw TokenError("Meta answered $status without updates") + return responses.flatMap { r -> + (r.jsonObject["resource_updates_info"] as? JsonArray).orEmpty().flatMap { u -> + val feature = u.jsonObject.path("feature")?.text("feature_name").orEmpty() + (u.jsonObject["feature_resource_updates_info"] as? JsonArray).orEmpty().mapNotNull { e -> + val o = e.jsonObject + PackOffer( + feature = feature, + resource = o.text("resource_name") ?: return@mapNotNull null, + version = o.text("resource_version").orEmpty(), + url = o.text("file_uri") ?: return@mapNotNull null, + size = (o["file_size"] as? JsonPrimitive)?.longOrNull ?: o.text("file_size")?.toLongOrNull() ?: 0, + sha256 = o.text("file_checksum").orEmpty(), + appId = o.text("app_id").orEmpty(), + type = o.text("resource_type").orEmpty(), + ) + } + } + } +} + +/** + * One FetchConstellation query as the Meta app sends it, with [info] as its `constellation_updates_request_info`. + * Returns the HTTP status and the answer's JSON. + */ +internal fun constellationPost(token: Token, friendlyName: String, docId: String, info: JsonObject, http: HttpPost): Pair { val variables = buildJsonObject { put("fetch_request", buildJsonObject { put("constellation_updates_request_info", JsonArray(listOf(info))) }) } + return graphPost(token, friendlyName, docId, variables, http) +} + +/** One persisted query on Meta's AR endpoint, as the Meta app sends it: the HTTP status and the answer's JSON. */ +internal fun graphPost(token: Token, friendlyName: String, docId: String, variables: JsonObject, http: HttpPost): Pair { val body = pyUrlencode( linkedMapOf( - "fb_api_req_friendly_name" to "FetchConstellationFeatureUpdates", - "client_doc_id" to FETCH_UPDATES_DOC, + "fb_api_req_friendly_name" to friendlyName, + "client_doc_id" to docId, "variables" to variables.toString(), "method" to "post", "locale" to "en_US", @@ -90,27 +127,13 @@ fun fetchPackOffers( } val json = runCatching { Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject } .getOrElse { throw TokenError("Meta answered ${reply.status} with no JSON") } + checkSession(json) json["errors"]?.let { throw TokenError("Meta refused: ${it.toString().take(200)}") } - val responses = json.path("data", "xar_constellation_feature_updates")?.get("constellation_updates_response_info") as? JsonArray - ?: throw TokenError("Meta answered ${reply.status} without updates") - return responses.flatMap { r -> - (r.jsonObject["resource_updates_info"] as? JsonArray).orEmpty().flatMap { u -> - val feature = u.jsonObject.path("feature")?.text("feature_name").orEmpty() - (u.jsonObject["feature_resource_updates_info"] as? JsonArray).orEmpty().mapNotNull { e -> - val o = e.jsonObject - PackOffer( - feature = feature, - resource = o.text("resource_name") ?: return@mapNotNull null, - version = o.text("resource_version").orEmpty(), - url = o.text("file_uri") ?: return@mapNotNull null, - size = (o["file_size"] as? JsonPrimitive)?.longOrNull ?: o.text("file_size")?.toLongOrNull() ?: 0, - sha256 = o.text("file_checksum").orEmpty(), - ) - } - } - } + return reply.status to json } +private const val GRAPH_URL = "https://ar-genai.graph.meta.com/graphql" + private fun JsonObject.path(vararg keys: String): JsonObject? = keys.fold(this as JsonObject?) { o, k -> o?.get(k) as? JsonObject } diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/PairPicture.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/PairPicture.kt new file mode 100644 index 00000000..1ad6dbbc --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/language/PairPicture.kt @@ -0,0 +1,72 @@ +package com.macedonga.visor.core.identity.language + +import com.macedonga.visor.core.identity.token.checkSession +import com.macedonga.visor.core.identity.HttpPost +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.pyUrlencode +import com.macedonga.visor.core.identity.token.Token +import com.macedonga.visor.core.identity.token.TokenError +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.intOrNull +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.putJsonArray + +/** Where Meta keeps a pair's picture: the frame, colour and lenses as sold, front on, on a clear background. */ +data class PairPicture(val light: String, val dark: String?) + +private const val GRAPH_URL = "https://ar-genai.graph.meta.com/graphql" +private const val ASSETS_BY_SERIAL_DOC = "256686669512680180239290821066" +// Asset category 2 is the device image the Meta app shows for each pair. +private const val DEVICE_IMAGE = 2 + +/** + * WearableDeviceAssetsBySerialNumbers, the Meta app's own query: each pair's picture by serial. The picture for the + * pair's SKU comes first; the one for its device type, a stock frame, only when there is none. + */ +fun fetchPairPictures(token: Token, serials: List, http: HttpPost = UrlConnectionPost): Map { + val body = pyUrlencode( + linkedMapOf( + "fb_api_req_friendly_name" to "WearableDeviceAssetsBySerialNumbers", + "client_doc_id" to ASSETS_BY_SERIAL_DOC, + "variables" to buildJsonObject { putJsonArray("serial_numbers") { serials.forEach { add(JsonPrimitive(it)) } } }.toString(), + "method" to "post", + "pretty" to "false", + "format" to "json", + ), + ) + val reply = try { + http.post( + GRAPH_URL, + mapOf("Authorization" to "OAuth ${token.value}", "Content-Type" to "application/x-www-form-urlencoded"), + body.toByteArray(Charsets.US_ASCII), + ) + } catch (error: Exception) { + throw TokenError("Meta could not be reached: ${error.message}") + } + val json = runCatching { Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject } + .getOrElse { throw TokenError("Meta answered ${reply.status} with no JSON") } + checkSession(json) + json["errors"]?.let { throw TokenError("Meta refused: ${it.toString().take(200)}") } + val categories = ((json["data"] as? JsonObject)?.get("xar_wearable_device_assets_by_serial_numbers") as? JsonObject)?.get("data") as? JsonArray + ?: return emptyMap() + val images = categories.mapNotNull { it as? JsonObject }.firstOrNull { (it["asset_category"] as? JsonPrimitive)?.intOrNull == DEVICE_IMAGE } + ?: return emptyMap() + return (images["serial_specific"] as? JsonArray).orEmpty().mapNotNull { entry -> + val o = entry as? JsonObject ?: return@mapNotNull null + val serial = o.text("serial_number") ?: return@mapNotNull null + val picture = o.image("sku_specific") ?: o.image("device_specific") ?: return@mapNotNull null + serial to picture + }.toMap() +} + +private fun JsonObject.image(group: String): PairPicture? = + ((this[group] as? JsonObject)?.get("assets") as? JsonArray).orEmpty().mapNotNull { it as? JsonObject } + .firstOrNull { it.text("asset_type") == "IMAGE" && it.text("status") == "ACTIVE" && it.text("asset_manager_url") != null } + ?.let { PairPicture(it.text("asset_manager_url")!!, it.text("asset_manager_url_dark")) } + +private fun JsonObject.text(key: String): String? = (this[key] as? JsonPrimitive)?.contentOrNull?.takeIf { it.isNotBlank() } diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/Provision.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/Provision.kt index 26d00f6a..da2970b4 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/Provision.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/Provision.kt @@ -61,12 +61,12 @@ class Receipt(val signature: ByteArray, val bodyJson: ByteArray) { } } -/** IdentityResponse fields 1 and 2. */ -class DeviceIdentity(val certDer: ByteArray, val serial: String) { +/** IdentityResponse {1: certificate, 2: serial, 3: registration challenge, 4: DRK present, 5: secondary certificate}. */ +class DeviceIdentity(val certDer: ByteArray, val serial: String, val secondaryCert: ByteArray = ByteArray(0)) { companion object { fun decode(data: ByteArray): DeviceIdentity { val fields = Wire.parse(data) - return DeviceIdentity(certDer = fields.bytes(1), serial = String(fields.bytes(2), Charsets.US_ASCII)) + return DeviceIdentity(certDer = fields.bytes(1), serial = String(fields.bytes(2), Charsets.US_ASCII), secondaryCert = fields.bytes(5)) } } } @@ -134,7 +134,7 @@ class PairingBackend( ) } - fun pairRequest(certDer: ByteArray, serial: String, deviceNonce: ByteArray, appPubkey: ByteArray): Receipt { + fun pairRequest(certDer: ByteArray, serial: String, deviceNonce: ByteArray, appPubkey: ByteArray, secondaryCert: ByteArray = ByteArray(0)): Receipt { val fields = common() fields["device_cert"] = Base64.getEncoder().encodeToString(certDer) fields["serial_number"] = serial @@ -142,7 +142,8 @@ class PairingBackend( buildJsonObject { put("device_nonce", Base64.getEncoder().encodeToString(deviceNonce)) put("app_pubkey", Base64.getEncoder().encodeToString(appPubkey)) - put("secondary_cert", "") + // IdentityResponse field 5, as the Meta app sends it; glasses have none. + put("secondary_cert", if (secondaryCert.isEmpty()) "" else Base64.getEncoder().encodeToString(secondaryCert)) }, ) return post("pair_request", fields, "pending_ownership_receipt") @@ -189,6 +190,10 @@ class ProvisioningFlow( private val channel = connection.openChannel(ServiceId.IDENTITY.value) private var first = true + /** The device's certificate from its IdentityResponse, once [run] has it. */ + var deviceCert: ByteArray? = null + private set + private suspend fun send(messageType: IdentityMsgType, payload: ByteArray = ByteArray(0)) { // A peer names a channel's service only on that channel's first message; later ones // carry service_id 0. @@ -224,6 +229,7 @@ class ProvisioningFlow( onStep(ProvisionStep.IDENTITY) send(IdentityMsgType.IDENTITY_REQUEST) val identity = DeviceIdentity.decode(await(IdentityMsgType.IDENTITY_RESPONSE)) + deviceCert = identity.certDer onStep(ProvisionStep.SKIP_CHALLENGE) send(IdentityMsgType.SKIP_CHALLENGE) @@ -231,7 +237,10 @@ class ProvisioningFlow( onStep(ProvisionStep.PAIR_REQUEST) val pending = withContext(Dispatchers.IO) { - backend.pairRequest(certDer = identity.certDer, serial = identity.serial, deviceNonce = nonce, appPubkey = appPubkey) + backend.pairRequest( + certDer = identity.certDer, serial = identity.serial, deviceNonce = nonce, appPubkey = appPubkey, + secondaryCert = identity.secondaryCert, + ) } onStep(ProvisionStep.START_CHANGE_OWNER) diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/ProvisionSession.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/ProvisionSession.kt index 17266091..e4057635 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/ProvisionSession.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/provision/ProvisionSession.kt @@ -12,6 +12,7 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import kotlinx.serialization.json.jsonObject /** Drive one provisioning connection: link up, change owner, hand back the key. */ class ProvisioningResult( @@ -20,6 +21,9 @@ class ProvisioningResult( /** DIVERGENCE: the reference returns the path it wrote. The key itself comes back here. */ val keyPem: ByteArray, val reusedKey: Boolean, + val deviceCert: ByteArray? = null, + /** additional_data.device_ec_public_key in Meta's final receipt, as the Meta app stores it; lease hints name it. */ + val deviceEcPublicKey: ByteArray? = null, ) /** Run change owner over [transport] and return the key it installed. */ @@ -37,6 +41,7 @@ suspend fun provision( onStep(ProvisionStep.CONNECT) transport.connect() + var deviceCert: ByteArray? = null val final = try { coroutineScope { onStep(ProvisionStep.SECURE_LINK) @@ -45,7 +50,8 @@ suspend fun provision( val connection = DataXConnection(link, ChannelRegistry()) val reader = connection.startReader(this) try { - ProvisioningFlow(connection, backend, key.publicKey64, timeoutMillis, onStep).run() + val flow = ProvisioningFlow(connection, backend, key.publicKey64, timeoutMillis, onStep) + flow.run().also { deviceCert = flow.deviceCert } } finally { // Close first: a read blocked on the socket does not answer cancellation, and // the reader would hold this scope open until it did. @@ -63,6 +69,13 @@ suspend fun provision( userId = info.textOf("user_id"), keyPem = key.pem, reusedKey = existingKeyPem != null, + deviceCert = deviceCert, + deviceEcPublicKey = runCatching { + // The Meta app reads it with getString, which takes a string or an object. + val extra = (info["additional_data"] as? kotlinx.serialization.json.JsonObject) + ?: kotlinx.serialization.json.Json.parseToJsonElement(info.textOf("additional_data")).jsonObject + com.macedonga.visor.core.identity.rawPublicKey(java.util.Base64.getDecoder().decode(extra.textOf("device_ec_public_key"))) + }.getOrNull(), ) } diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/AccountToken.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/AccountToken.kt new file mode 100644 index 00000000..a3c3e268 --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/AccountToken.kt @@ -0,0 +1,82 @@ +package com.macedonga.visor.core.identity.token + +import com.macedonga.visor.core.identity.HttpPost +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.pyUrlencode +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.jsonObject + +/** The glasses' accounts app: the Meta account the Display keeps is minted for this app id and package. */ +const val ACCOUNTS_APP_ID = "1463123261063354" +const val ACCOUNTS_PACKAGE = "com.meta.smartglass.app.accounts" + +/** What the token exchange answers for the Meta account. The access token is a secret: never log it. */ +class AccountIdentity( + val accessToken: String, + val userId: String, + val appId: String, + val accountId: String, + val name: String, + val profilePictureUrl: String, + val isEmployee: Boolean, + val isTestUser: Boolean, + val analyticsClaim: String, +) + +/** + * POST /app_token_exchange: an access token for [appId] and [packageName], minted from the user's own Meta login. The + * user id and the account fields come back with it. + */ +fun exchangeAccountToken( + userToken: Token, + appId: String = ACCOUNTS_APP_ID, + packageName: String = ACCOUNTS_PACKAGE, + http: HttpPost = UrlConnectionPost, +): AccountIdentity { + val body = pyUrlencode( + linkedMapOf( + "target_app_id" to appId, + "package_name" to packageName, + "extras" to "account_id,is_employee,is_test_user,name,profile_picture_url,analytics_claim", + ), + ).toByteArray(Charsets.US_ASCII) + val reply = try { + http.post( + "https://$DEFAULT_AR_HOST/app_token_exchange", + mapOf( + "Authorization" to "OAuth ${userToken.value}", + "Content-Type" to "application/x-www-form-urlencoded", + "User-Agent" to META_USER_AGENT, + ), + body, + ) + } catch (error: Exception) { + throw TokenError("app_token_exchange: ${error.javaClass.simpleName}: ${error.message}", unreachable = true) + } + val payload: JsonObject = try { + Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject + } catch (error: Exception) { + throw TokenError("app_token_exchange: response was not JSON") + } + checkSession(payload) + payload["error"]?.let { throw TokenError("token exchange refused: ${(it as? JsonPrimitive)?.content ?: it}") } + val token = payload.text("access_token") + val userId = payload.text("user_id") + if (token.isNullOrEmpty() || userId.isNullOrEmpty()) { + throw TokenError("no access_token or user_id in the answer; keys: ${payload.sortedKeys()}") + } + return AccountIdentity( + accessToken = token, + userId = userId, + appId = payload.text("app_id") ?: appId, + accountId = payload.text("account_id").orEmpty(), + name = payload.text("name").orEmpty(), + profilePictureUrl = payload.text("profile_picture_url").orEmpty(), + isEmployee = (payload["is_employee"] as? JsonPrimitive)?.booleanOrNull == true, + isTestUser = (payload["is_test_user"] as? JsonPrimitive)?.booleanOrNull == true, + analyticsClaim = payload.text("analytics_claim").orEmpty(), + ) +} diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Frla.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Frla.kt index cd4bbcdc..21c288c5 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Frla.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Frla.kt @@ -32,7 +32,7 @@ private fun post(url: String, fields: Map, http: HttpPost): Json } catch (error: Exception) { // The interesting answers arrive as 4xx/5xx with a JSON body and are read, not thrown; // only failing to reach the server at all lands here. - throw TokenError("$url: ${error.javaClass.simpleName}: ${error.message}") + throw TokenError("$url: ${error.javaClass.simpleName}: ${error.message}", unreachable = true) } return try { Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/LinkedAccounts.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/LinkedAccounts.kt new file mode 100644 index 00000000..da2e8245 --- /dev/null +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/LinkedAccounts.kt @@ -0,0 +1,95 @@ +package com.macedonga.visor.core.identity.token + +import com.macedonga.visor.core.identity.HttpPost +import com.macedonga.visor.core.identity.UrlConnectionPost +import com.macedonga.visor.core.identity.pyUrlencode +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.jsonObject + +/** A provider account of the Meta account: its svc89 account type, the Graph name and the app id its token is minted for. */ +enum class LinkedAccountType(val wire: Int, val graphName: String, val appId: String) { + FACEBOOK(1, "FACEBOOK", "1053491986377452"), + INSTAGRAM(2, "INSTAGRAM", "1313693057438013"), + THREADS(3, "THREADS", "1590413528774085"), +} + +/** A linked account as the Meta account lists it: the id the Graph queries take, its name, and whether glasses may use it. */ +class LinkedAccountRef(val type: LinkedAccountType, val obfuscatedId: String, val name: String, val forGlasses: Boolean) + +/** A linked account's access token and, when Meta gives one, its first-party token. Secrets: never log them. */ +class LinkedTokens(val accessToken: String, val firstPartyToken: String?) + +private const val GRAPH_URL = "https://ar-genai.graph.meta.com/graphql" +private const val LINKED_LIST_DOC = "3214186391731893797068997674" +private const val LINKED_TOKENS_DOC = "254156320718283261599350148705" +private const val LINKED_TOKENS_ROOT = "fx_ar_linked_access_and_first_party_token_from_obid" +private const val TOKEN_PACKAGE = "com.facebook.stella" + +/** + * The provider accounts linked to the Meta account: data.wearable_linked_accounts [{obfuscated_id, account_type, + * display_name, is_enabled_for_wearable}], as Meta answered it on 2026-10-23. + */ +fun fetchLinkedAccounts(accessToken: String, http: HttpPost = UrlConnectionPost): List { + val json = graphql(accessToken, "MetaLinkedAccounts", LINKED_LIST_DOC, "{}", http) + val list = (json["data"] as? JsonObject)?.get("wearable_linked_accounts") as? JsonArray ?: return emptyList() + return list.mapNotNull { element -> + val node = element as? JsonObject ?: return@mapNotNull null + val id = node.text("obfuscated_id") ?: return@mapNotNull null + val type = node.text("account_type")?.let { name -> LinkedAccountType.entries.firstOrNull { it.graphName == name.uppercase() } } + ?: return@mapNotNull null + LinkedAccountRef(type, id, node.text("display_name").orEmpty(), (node["is_enabled_for_wearable"] as? JsonPrimitive)?.content == "true") + }.distinctBy { it.type } +} + +/** The tokens of one linked account, from the accounts app's token; null when Meta gives no access token for it. */ +fun fetchLinkedTokens( + accountsToken: String, + account: LinkedAccountRef, + deviceId: String, + http: HttpPost = UrlConnectionPost, +): LinkedTokens? { + val variables = JsonObject( + mapOf( + "obfuscated_account_id" to JsonPrimitive(account.obfuscatedId), + "account_type" to JsonPrimitive(account.type.graphName), + "app_id" to JsonPrimitive(account.type.appId), + "package_name" to JsonPrimitive(TOKEN_PACKAGE), + "device_id" to JsonPrimitive(deviceId), + "service" to JsonPrimitive("WEARABLES_LINKED_ACCOUNTS"), + ), + ).toString() + val json = graphql(accountsToken, "MetaLinkedAccountAccessAndFirstPartyTokensQuery", LINKED_TOKENS_DOC, variables, http) + val node = (json["data"] as? JsonObject)?.get(LINKED_TOKENS_ROOT) as? JsonObject ?: return null + val token = node.text("token")?.takeIf { it.isNotEmpty() } ?: return null + return LinkedTokens(token, node.text("first_party_token")?.takeIf { it.isNotEmpty() }) +} + +/** One persisted query on Meta's ar GraphQL, as the Meta app's own calls send it. */ +private fun graphql(accessToken: String, name: String, doc: String, variables: String, http: HttpPost): JsonObject { + val body = pyUrlencode( + linkedMapOf( + "fb_api_req_friendly_name" to name, + "client_doc_id" to doc, + "variables" to variables, + "method" to "post", + "pretty" to "false", + "format" to "json", + ), + ).toByteArray(Charsets.US_ASCII) + val reply = try { + http.post( + GRAPH_URL, + mapOf("Authorization" to "OAuth $accessToken", "Content-Type" to "application/x-www-form-urlencoded"), + body, + ) + } catch (error: Exception) { + throw TokenError("Meta could not be reached: ${error.message}", unreachable = true) + } + val json = runCatching { Json.parseToJsonElement(String(reply.body, Charsets.UTF_8)).jsonObject } + .getOrElse { throw TokenError("Meta answered ${reply.status} with no JSON") } + if (json["errors"] != null) throw TokenError("Meta refused the query $name") + return json +} diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Token.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Token.kt index eea7d6ef..69110452 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Token.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/Token.kt @@ -3,9 +3,11 @@ package com.macedonga.visor.core.identity.token import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.intOrNull /** Where the provisioning token comes from. */ -class TokenError(message: String) : Exception(message) +/** [unreachable]: Meta was not reached at all, so the login itself may well be fine. */ +open class TokenError(message: String, val unreachable: Boolean = false) : Exception(message) class Token( val value: String, @@ -18,6 +20,29 @@ class Token( fun interface TokenSource { fun get(): Token + + /** Meta ended the session behind [dropped]: the next [get] logs in for a new token. */ + fun forget(dropped: Token) {} +} + +/** Meta answered OAuthException 190 to the token: the session behind it has ended. */ +class SessionEnded : TokenError("Meta ended the login session; log in again if this repeats") + +/** Throws [SessionEnded] when a Meta answer carries error 190. */ +fun checkSession(json: JsonObject) { + val code = (json["error"] as? JsonObject)?.get("code") as? JsonPrimitive + if (code?.intOrNull == 190) throw SessionEnded() +} + +/** Runs [block] with the token. When Meta ended its session, forgets the token and runs [block] once more with a fresh one. */ +inline fun TokenSource.withToken(block: (Token) -> T): T { + val token = get() + return try { + block(token) + } catch (error: SessionEnded) { + forget(token) + block(get()) + } } /** The user agent both Meta auth calls send. */ diff --git a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt index d26bd7ec..76fd6e04 100644 --- a/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt +++ b/core/identity/src/main/kotlin/com/macedonga/visor/core/identity/token/WebLogin.kt @@ -63,7 +63,7 @@ fun exchangeCode(code: String, http: HttpPost = UrlConnectionPost): String { body.toByteArray(Charsets.US_ASCII), ) } catch (error: Exception) { - throw TokenError("token-exchange: ${error.javaClass.simpleName}: ${error.message}") + throw TokenError("token-exchange: ${error.javaClass.simpleName}: ${error.message}", unreachable = true) } val payload = try { @@ -98,17 +98,30 @@ class WebLoginSource( /** True when an FRLA is cached. Says nothing about whether Meta still accepts it. */ val hasCachedLogin: Boolean get() = readCache() != null + // One token for the run: each exchange opens a new Meta session, and Meta ends the older ones, with the + // requests still in flight on them. + @Volatile private var token: Token? = null + override fun get(): Token { - val cached = readCache() ?: throw TokenError("not logged in to Meta") - try { - return exchangeFrla(cached, http = http) - } catch (error: TokenError) { - // An FRLA that no longer works is the one case where the browser has to open - // again. Say why, so this does not look like a network fault. - throw TokenError("the saved Meta login no longer works (${error.message}); log in again") + token?.let { return it } + synchronized(this) { + token?.let { return it } + val cached = readCache() ?: throw TokenError("not logged in to Meta") + try { + return exchangeFrla(cached, http = http).also { token = it } + } catch (error: TokenError) { + // No network is not a dead login: say so, and keep "log in again" for an FRLA Meta turned down. + if (error.unreachable) throw TokenError("Meta could not be reached (${error.message})", unreachable = true) + throw TokenError("the saved Meta login no longer works (${error.message}); log in again") + } } } + /** Meta no longer takes [dropped]: the next [get] exchanges the login for a new token. */ + override fun forget(dropped: Token) { + synchronized(this) { if (token == dropped) token = null } + } + fun begin(): AuthorizeRequest = authorizeUrl() /** Take the redirect the WebView caught, exchange its code, and cache the FRLA. */ @@ -137,6 +150,7 @@ class WebLoginSource( /** Forget the cached login. The WebView's cookies are the app's to clear. */ fun logOut() { + token = null cacheFile?.delete() } @@ -152,6 +166,7 @@ class WebLoginSource( } private fun writeCache(frla: String) { + token = null if (cacheFile == null || sealer == null) return cacheFile.parentFile?.mkdirs() cacheFile.writeBytes(sealer.seal(frla.toByteArray(Charsets.UTF_8))) diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Airship.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Airship.kt index c83bd349..55369941 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Airship.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Airship.kt @@ -5,6 +5,7 @@ import com.macedonga.visor.core.session.DataXMessage import com.macedonga.visor.core.session.Responder import java.io.File import java.io.FileOutputStream +import java.util.concurrent.CopyOnWriteArrayList import java.util.zip.Adler32 import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.Flow @@ -61,7 +62,7 @@ data class Transfer(val uuid: String, val received: Long, val size: Long, val by /** Files for someone other than the media import, such as the assistant's photo. */ interface ExtraFiles { /** Whether to take an offer outside the media and bug report namespaces. */ - fun wants(namespace: String, size: Long): Boolean + fun wants(offer: Offer): Boolean /** A file this side wanted, whole. The receiver deletes it afterwards. */ fun take(received: Received) @@ -77,10 +78,10 @@ class AirshipReceiver(private val spool: File) : Responder { @Volatile var describe: (String) -> String = { "" } - @Volatile - var extra: ExtraFiles? = null + /** Takers of other files, asked in order; the first that wants an offer gets it. */ + val extras = CopyOnWriteArrayList() - private class Live(val offer: Offer, val file: File) { + private class Live(val offer: Offer, val file: File, val taker: ExtraFiles?) { var out = FileOutputStream(file) val adler = Adler32() var received = 0L @@ -172,10 +173,11 @@ class AirshipReceiver(private val spool: File) : Responder { baseName: String, file: java.io.File, chunk: Int = 1024, + uuid: ByteArray? = null, onProgress: (Long) -> Unit = {}, ): String? = java.io.RandomAccessFile(file, "r").use { raf -> - send(connection, namespace, baseName, file.length(), chunk, onProgress) { at, n -> ByteArray(n).also { raf.seek(at); raf.readFully(it) } } + send(connection, namespace, baseName, file.length(), chunk, onProgress, uuid) { at, n -> ByteArray(n).also { raf.seek(at); raf.readFully(it) } } } private suspend fun send( @@ -262,13 +264,14 @@ class AirshipReceiver(private val spool: File) : Responder { val f = fields(payload) val offer = Offer(f.bytes(1), f.long(2), f.bytes(3).decodeToString()) live.remove(channel)?.discard() + val taker = if (offer.namespace in ACCEPTED) null else extras.firstOrNull { runCatching { it.wants(offer) }.getOrDefault(false) } // Refusing still needs the drop, or the glasses' queue stalls behind this offer. - if (offer.namespace !in ACCEPTED && extra?.wants(offer.namespace, offer.size) != true) { + if (offer.namespace !in ACCEPTED && taker == null) { if (offer.namespace !in QUIET) _notes.trySend("Refused ${sizeWords(offer.size)} in namespace \"${offer.namespace}\"") return MT_DROP to fileId(offer.raw) } spool.mkdirs() - live[channel] = Live(offer, File(spool, offer.uuid)) + live[channel] = Live(offer, File(spool, offer.uuid), taker) _notes.trySend("Receiving ${sizeWords(offer.size)}${describe(offer.uuid)}") publish() // Offset 0 with the adler32 of nothing: what FileReceiver::acceptOffer sends. @@ -298,10 +301,10 @@ class AirshipReceiver(private val spool: File) : Responder { val verified = f.long(1) == t.received && f.long(2) == t.adler.value val whole = Received(t.offer, t.file, verified) if (t.offer.namespace in ACCEPTED) { - extra?.let { runCatching { it.peek(whole) } } + extras.forEach { runCatching { it.peek(whole) } } _received.trySend(whole) } else { - extra?.let { runCatching { it.take(whole) } } + t.taker?.let { runCatching { it.take(whole) } } t.file.delete() } publish() diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Broadcast.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Broadcast.kt index 9b8e7e53..5f9d7a1f 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Broadcast.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Broadcast.kt @@ -83,16 +83,19 @@ class Broadcast(private val scope: CoroutineScope) { var transcoder: AvcTranscoder? = null try { rtmp.connect(scope) - val sps = hevcParameterSets(config)[33]?.firstOrNull()?.let(::parseHevcSps) - val width = sps?.width ?: 1080 - val height = sps?.height ?: 1920 + // H.264 from the glasses goes out as it is; HEVC goes out as HEVC, or is re-encoded to H.264. + val input = configCodec(config) + val out = if (input == VideoCodec.AVC) BroadcastCodec.H264 else target.codec + val (width, height) = videoSize(config) ?: (1080 to 1920) val audio = audioFormat() - rtmp.metadata(width, height, frameRate, target.codec, audio) + rtmp.metadata(width, height, frameRate, out, audio) if (audio != null) rtmp.audioHeader(audio) var base = -1L fun ms(pts: Long) = ((pts - base) / 1000).coerceAtLeast(0) - if (target.codec == BroadcastCodec.HEVC) { + if (out == BroadcastCodec.HEVC) { rtmp.videoHeader(BroadcastCodec.HEVC, checkNotNull(hvcC(config)) { "the stream's parameter sets do not parse" }) + } else if (input == VideoCodec.AVC) { + rtmp.videoHeader(BroadcastCodec.H264, checkNotNull(avcC(config)) { "the stream's parameter sets do not parse" }) } else { var headerSent = false transcoder = AvcTranscoder(width, height, config, bitrate, frameRate, @@ -119,8 +122,8 @@ class Broadcast(private val scope: CoroutineScope) { check(!transcoder.failed) { "the phone's H.264 encoder stopped" } transcoder.feed(item.frame) } else { - val units = annexBToLengthPrefixed(item.frame.data, dropParameterSets = true) - rtmp.video(BroadcastCodec.HEVC, ms(item.frame.ptsUs), item.frame.isKey, units) + val units = annexBToLengthPrefixed(item.frame.data, dropParameterSets = out == BroadcastCodec.HEVC) + rtmp.video(out, ms(item.frame.ptsUs), item.frame.isKey, units) } sent += item.frame.data.size } diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt index 58cf23dd..bf1f9ba8 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Captures.kt @@ -28,6 +28,7 @@ const val KIND_STILL = 2 const val KIND_IMU = 3 const val KIND_FRAMES = 4 +// The Display's kinds are the wire CaptureType plus 0xC00 (inferred). Any other kind is classed by its assets. private const val CAPTURE_VIDEO = 3074L private const val ASSET_DELIVERED = 0x100000L diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/FrameGrab.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/FrameGrab.kt index b87bb6b0..5c0de51a 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/FrameGrab.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/FrameGrab.kt @@ -17,12 +17,13 @@ import kotlinx.coroutines.withContext * it works with no screen showing the stream. Null if the decoder gave nothing. */ suspend fun grabJpeg(picture: Replay, quality: Int = 92): ByteArray? = withContext(Dispatchers.Default) { - val sps = hevcParameterSets(picture.config)[33]?.firstOrNull()?.let(::parseHevcSps) ?: return@withContext null - val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_HEVC, sps.width, sps.height).apply { + val (width, height) = videoSize(picture.config) ?: return@withContext null + val videoCodec = picture.video.firstOrNull()?.codec ?: configCodec(picture.config) + val format = MediaFormat.createVideoFormat(videoCodec.mime, width, height).apply { setByteBuffer("csd-0", ByteBuffer.wrap(picture.config)) setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible) } - val codec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_HEVC) + val codec = MediaCodec.createDecoderByType(videoCodec.mime) try { codec.configure(format, null, null, 0) codec.start() @@ -49,7 +50,7 @@ suspend fun grabJpeg(picture: Replay, quality: Int = 92): ByteArray? = withConte } val o = codec.dequeueOutputBuffer(info, 5_000) if (o >= 0) { - if (info.size > 0) codec.getOutputImage(o)?.use { last = jpegOf(it, sps.width, sps.height, quality) } + if (info.size > 0) codec.getOutputImage(o)?.use { last = jpegOf(it, width, height, quality) } codec.releaseOutputBuffer(o, false) if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) ended = true } diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/GlassesConfig.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/GlassesConfig.kt index 52283407..2a4c86ef 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/GlassesConfig.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/GlassesConfig.kt @@ -24,6 +24,8 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex @@ -66,7 +68,12 @@ class GlassesConfig( private val airship: AirshipReceiver, private val media: MediaImport, ) { - private val dir = File(context.filesDir, "mobileconfig").apply { mkdirs() } + private val root = File(context.filesDir, "mobileconfig").apply { mkdirs() } + + // Each pair's own folder: their parameter maps differ with their firmware, and a table built on one pair's values + // must never reach another. + @Volatile + private var dir = File(root, connection.pair.value ?: NO_PAIR) private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private val _state = MutableStateFlow(ConfigState()) val state: StateFlow = _state.asStateFlow() @@ -81,7 +88,29 @@ class GlassesConfig( init { media.bugReportReader = { take(it) } - scope.launch { runCatching { load() } } + scope.launch { + connection.pair.filterNotNull().distinctUntilChanged().collect { serial -> + writes.withLock { runCatching { switchTo(serial) } } + } + } + } + + /** The pair's own values, or none yet; what the last pair had stays in its folder. */ + private fun switchTo(serial: String) { + dir = File(root, serial) + // Before Visor kept pairs apart, the one pair's files sat in the root. + val loose = root.listFiles { f -> f.isFile }.orEmpty() + if (loose.isNotEmpty() && !dir.exists()) { + dir.mkdirs() + loose.forEach { it.renameTo(File(dir, it.name)) } + } + dir.mkdirs() + schemaHash = "" + counts = IntArray(4) + sessionId = "" + boot = null + _state.value = ConfigState() + load() } /** Ask the glasses for a bug report and read their configuration out of it. */ @@ -304,6 +333,7 @@ class GlassesConfig( const val SENT = "sent_at.txt" const val TIMESTAMP = "current_timestamp.txt" const val LAST_STAMP = "last_stamp.txt" + private const val NO_PAIR = "none" const val STAMP = "MMM-dd-yyyy-HH-mm-ss" const val STALE_MARGIN_MS = 15 * 60_000L const val READ_TIMEOUT_MS = 10 * 60_000L diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Hevc.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Hevc.kt index d8df0378..906af747 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Hevc.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Hevc.kt @@ -122,7 +122,7 @@ fun hvcC(config: ByteArray): ByteArray? { } /** Drops the emulation-prevention bytes (00 00 03) from a NAL unit's payload. */ -private fun unescape(unit: ByteArray): ByteArray { +internal fun unescape(unit: ByteArray): ByteArray { val out = ByteArrayOutputStream(unit.size) var zeros = 0 for (b in unit) { @@ -136,7 +136,7 @@ private fun unescape(unit: ByteArray): ByteArray { return out.toByteArray() } -private class BitReader(private val data: ByteArray, startByte: Int) { +internal class BitReader(private val data: ByteArray, startByte: Int) { private var pos = startByte * 8 fun bits(n: Int): Int { diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveDecoder.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveDecoder.kt index b38b2874..fac08553 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveDecoder.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveDecoder.kt @@ -12,7 +12,7 @@ import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.TimeUnit /** - * Hardware HEVC decoding onto a Surface, one decoder per stream, asynchronous so a decoded frame is + * Hardware HEVC or H.264 decoding onto a Surface, one decoder per stream, asynchronous so a decoded frame is * shown the moment it exists rather than when the next one arrives. It is configured for the size * asked for with room up to 1920 either way, because a decoder fed a stream larger than its * configured size draws garbage on many phones. @@ -50,7 +50,7 @@ class LiveDecoder( fun feed(frame: VideoFrame, config: ByteArray?, show: Boolean = true): Boolean { val c = codec ?: run { if (!frame.isKey) return false - start(config) + start(config, frame.codec) } if (frame.data.size > MAX_INPUT) return false return queue(c, frame.data, frame.ptsUs, 0).also { if (it && !show) hidden.add(frame.ptsUs) } @@ -83,8 +83,8 @@ class LiveDecoder( false } - private fun start(config: ByteArray?): MediaCodec { - val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_HEVC, width, height) + private fun start(config: ByteArray?, videoCodec: VideoCodec): MediaCodec { + val format = MediaFormat.createVideoFormat(videoCodec.mime, width, height) format.setInteger(MediaFormat.KEY_MAX_WIDTH, MAX_SIDE) format.setInteger(MediaFormat.KEY_MAX_HEIGHT, MAX_SIDE) format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, MAX_INPUT) @@ -93,7 +93,7 @@ class LiveDecoder( if (config != null) format.setByteBuffer("csd-0", ByteBuffer.wrap(config)) val t = HandlerThread("live-decoder").also { it.start(); thread = it } - return MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_HEVC).also { + return MediaCodec.createDecoderByType(videoCodec.mime).also { it.setCallback(object : MediaCodec.Callback() { override fun onInputBufferAvailable(codec: MediaCodec, index: Int) { freeInputs.offer(index) diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveOutputs.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveOutputs.kt index 467a9a42..41848858 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveOutputs.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveOutputs.kt @@ -70,7 +70,7 @@ class LiveOutputs(private val context: Context) { } fun onVideo(input: VideoFrame) { - val frame = VideoFrame(input.seq, rebase(input.ptsUs), input.data, input.arrivedNs) + val frame = VideoFrame(input.seq, rebase(input.ptsUs), input.data, input.arrivedNs, input.codec) replay.add(frame) mp4?.let { r -> runCatching { synchronized(r) { r.video(frame) } } } broadcast.video(frame) diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveRecorder.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveRecorder.kt index 51a1bd62..48ff254a 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveRecorder.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveRecorder.kt @@ -17,7 +17,7 @@ class AudioPacket(val ptsUs: Long, val data: ByteArray) class AacFormat(val sampleRate: Int, val channels: Int, val asc: ByteArray) /** - * One MP4 in Movies/Visor, written as the frames come: the glasses' HEVC as it is, the audio as AAC. The file + * One MP4 in Movies/Visor, written as the frames come: the glasses' video as it is (HEVC or H.264), the audio as AAC. The file * stays pending, so the gallery does not list it, until [finish]. Audio before the first video frame is dropped, * and everything is timed from that frame. */ @@ -33,7 +33,6 @@ class Mp4Recording(private val context: Context, config: ByteArray, audio: AacFo private val info = MediaCodec.BufferInfo() init { - val sps = hevcParameterSets(config)[33]?.firstOrNull()?.let(::parseHevcSps) val values = ContentValues().apply { put(MediaStore.MediaColumns.DISPLAY_NAME, name) put(MediaStore.MediaColumns.MIME_TYPE, "video/mp4") @@ -44,8 +43,10 @@ class Mp4Recording(private val context: Context, config: ByteArray, audio: AacFo uri = checkNotNull(context.contentResolver.insert(MediaStore.Video.Media.EXTERNAL_CONTENT_URI, values)) { "MediaStore refused the file" } fd = checkNotNull(context.contentResolver.openFileDescriptor(uri, "rw")) { "no file to write" } muxer = MediaMuxer(fd.fileDescriptor, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) - val video = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_HEVC, sps?.width ?: 1080, sps?.height ?: 1920) - video.setByteBuffer("csd-0", ByteBuffer.wrap(onlyParameterSets(config))) + val codec = configCodec(config) + val (width, height) = videoSize(config) ?: (1080 to 1920) + val video = MediaFormat.createVideoFormat(codec.mime, width, height) + video.setByteBuffer("csd-0", ByteBuffer.wrap(onlyParameterSets(codec, config))) videoTrack = muxer.addTrack(video) audioTrack = if (audio != null) { val a = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, audio.sampleRate, audio.channels) @@ -64,7 +65,7 @@ class Mp4Recording(private val context: Context, config: ByteArray, audio: AacFo firstPts = frame.ptsUs } if (frame.ptsUs < lastPts) return - val data = withoutParameterSets(frame.data) + val data = withoutParameterSets(frame.codec, frame.data) info.set(0, data.size, frame.ptsUs - firstPts, if (frame.isKey) MediaCodec.BUFFER_FLAG_KEY_FRAME else 0) muxer.writeSampleData(videoTrack, ByteBuffer.wrap(data), info) lastPts = frame.ptsUs @@ -103,7 +104,7 @@ class ReplayRing(@Volatile var windowUs: Long) { @Synchronized fun config(next: ByteArray) { - val size = { c: ByteArray? -> c?.let { hevcParameterSets(it)[33]?.firstOrNull()?.let(::parseHevcSps) }?.let { it.width to it.height } } + val size = { c: ByteArray? -> c?.let(::videoSize) } if (size(config) != size(next)) { video.clear() audio.clear() @@ -155,14 +156,14 @@ class ReplayRing(@Volatile var windowUs: Long) { class Replay(val config: ByteArray, val video: List, val audio: List) -/** Only the VPS, SPS and PPS of [config], Annex-B, as csd-0 wants them. */ -private fun onlyParameterSets(config: ByteArray): ByteArray = annexB(annexBUnits(config).filter { hevcNalType(it) in 32..34 }) +/** Only the parameter sets of [config], Annex-B, as csd-0 wants them. */ +private fun onlyParameterSets(codec: VideoCodec, config: ByteArray): ByteArray = annexB(annexBUnits(config).filter { codec.isParameterSet(it) }) -/** A frame with any parameter sets in it taken out: MP4's hvc1 keeps them in the sample entry only. */ -private fun withoutParameterSets(data: ByteArray): ByteArray { +/** A frame with any parameter sets in it taken out: MP4 keeps them in the sample entry only. */ +private fun withoutParameterSets(codec: VideoCodec, data: ByteArray): ByteArray { val units = annexBUnits(data) - if (units.none { hevcNalType(it) in 32..34 }) return data - return annexB(units.filter { hevcNalType(it) !in 32..34 }) + if (units.none { codec.isParameterSet(it) }) return data + return annexB(units.filter { !codec.isParameterSet(it) }) } private fun annexB(units: List): ByteArray { diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt index 331e530a..7a6840da 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/LiveStream.kt @@ -14,6 +14,7 @@ import com.macedonga.visor.core.connection.SessionHostFactory import com.macedonga.visor.core.connection.subscribeToDeviceStatus import com.macedonga.visor.core.connection.DeviceState import com.macedonga.visor.core.connection.decodeDeviceStateUpdate +import com.macedonga.visor.core.connection.isDisplayBuild import com.macedonga.visor.core.connection.isDeviceStateType import com.macedonga.visor.core.connection.mergeDeviceState import com.macedonga.visor.core.identity.IdentityStore @@ -49,6 +50,7 @@ import kotlinx.coroutines.flow.SharedFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asSharedFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch @@ -92,8 +94,14 @@ enum class StreamQuality(val width: Int, val height: Int, val label: String) { * One reassembled frame: the glasses' presentation time in microseconds, then Annex-B video. [seq] * counts frames, so a reader can tell one was lost and wait for the next keyframe. */ -class VideoFrame(val seq: Long, val ptsUs: Long, val data: ByteArray, val arrivedNs: Long = System.nanoTime()) { - val isKey: Boolean by lazy { nalTypes(data).any { it in HEVC_KEY_NALS } } +class VideoFrame( + val seq: Long, + val ptsUs: Long, + val data: ByteArray, + val arrivedNs: Long = System.nanoTime(), + val codec: VideoCodec = VideoCodec.HEVC, +) { + val isKey: Boolean by lazy { codec.isKeyFrame(data) } } /** @@ -133,6 +141,11 @@ class StreamReceiver : Responder { private val _codecConfig = MutableStateFlow(null) val codecConfig: StateFlow = _codecConfig.asStateFlow() + // HEVC until the parameter sets or a picture say otherwise. + private var codec = VideoCodec.HEVC + private val _videoCodec = MutableStateFlow(null) + val videoCodec: StateFlow = _videoCodec.asStateFlow() + private val audioChunks = ByteArrayOutputStream() private val _audio = MutableSharedFlow(extraBufferCapacity = 256, onBufferOverflow = BufferOverflow.DROP_OLDEST) val audio: SharedFlow = _audio.asSharedFlow() @@ -182,9 +195,19 @@ class StreamReceiver : Responder { fpsCount = 0 gapMax = 0 _codecConfig.value = null + codec = VideoCodec.HEVC + _videoCodec.value = null _stats.value = StreamStats() } + private fun noteCodec(data: ByteArray) { + if (_videoCodec.value != null) return + detectVideoCodec(data)?.let { + codec = it + _videoCodec.value = it + } + } + override suspend fun respond(message: DataXMessage, connection: DataXConnection) { val payload = message.payload when (message.info.messageType) { @@ -200,6 +223,7 @@ class StreamReceiver : Responder { // Field 2 says which; failing that, video parameter sets are Annex-B and an AudioSpecificConfig is not. if (type == CodecType.VIDEO || (type != CodecType.AUDIO && nalTypes(config).isNotEmpty())) { _codecConfig.value = config + noteCodec(config) recording?.write(config) } else { aacFormatOf(config)?.let { _aac.value = it } @@ -274,7 +298,8 @@ class StreamReceiver : Responder { maxGapMs = if (gap >= 0) gap else it.maxGapMs, ) } - _frames.tryEmit(VideoFrame(seq++, pts, data)) + noteCodec(data) + _frames.tryEmit(VideoFrame(seq++, pts, data, codec = codec)) } } @@ -353,6 +378,10 @@ class LiveStream( private val _wifiSession = MutableStateFlow(null) val wifiSession: StateFlow = _wifiSession.asStateFlow() + // What the glasses send on the Wi-Fi session, for work run there with overWifi. + private val _wifiData = kotlinx.coroutines.flow.MutableSharedFlow(extraBufferCapacity = 256) + val wifiData: kotlinx.coroutines.flow.SharedFlow = _wifiData + /** The picture now as a JPEG, decoded from the replay ring; null when no Live runs. */ suspend fun frameNow(): ByteArray? = if (_running.value) outputs.replay.lastPicture()?.let { grabJpeg(it, quality = 88) } else null @@ -444,6 +473,7 @@ class LiveStream( scope.launch { receiver.control.collect(::onControl) } scope.launch { receiver.frames.collect(outputs::onVideo) } scope.launch { receiver.codecConfig.collect { if (it != null) outputs.onConfig(it) } } + scope.launch { receiver.videoCodec.filterNotNull().collect { say("Video codec: ${it.label}") } } scope.launch { receiver.audio.collect(outputs::onAudio) } scope.launch { receiver.aac.collect { if (it != null) outputs.onAudioFormat(it) } } scope.launch { @@ -849,6 +879,7 @@ class LiveStream( } is ServiceData -> { messageAt = System.currentTimeMillis() + _wifiData.tryEmit(event) onWifiMessage(event) } is ProtocolError -> say("Wi-Fi: ${event.detail}") @@ -878,11 +909,13 @@ class LiveStream( event.serviceId == 28 && event.messageType == MT_START_APP_RESPONSE -> startAnswer?.complete(Wire.parse(event.payload).varint(1)) event.serviceId == 28 && isDeviceStateType(event.messageType) -> runCatching { - val update = decodeDeviceStateUpdate(event.messageType, event.payload).state + val update = decodeDeviceStateUpdate(event.messageType, event.payload, display = isDisplayBuild(connection.state.value.info?.build)).state _device.update { it?.let { base -> mergeDeviceState(base, update) } ?: update } } // Airship, whose receiver reports for itself; everything else goes in the log. event.serviceId == 28 || event.serviceId == SVC_AIRSHIP -> Unit + // The Display's screencast, which reads its own. + event.serviceId == SVC_HN_SCREENCAST || event.serviceId == SVC_HN_SCREENCAST_MANAGER -> Unit else -> say( "Wi-Fi: service ${event.serviceId} type ${event.messageType}, ${event.payload.size} bytes " + event.payload.take(16).joinToString("") { "%02x".format(it) }, @@ -1087,9 +1120,6 @@ private fun snappWords(code: Long): String = when (code) { else -> "error $code" } -// HEVC NAL unit types: 16-21 are IRAP (keyframes), 32-34 VPS/SPS/PPS. -private val HEVC_KEY_NALS = setOf(16, 17, 18, 19, 20, 21) - /** The HEVC NAL unit types in an Annex-B buffer. */ fun nalTypes(data: ByteArray): List { val out = ArrayList() diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/MediaImport.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/MediaImport.kt index d6db1206..b4759436 100644 --- a/core/media/src/main/kotlin/com/macedonga/visor/core/media/MediaImport.kt +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/MediaImport.kt @@ -97,6 +97,9 @@ class MediaImport( // Asked for while another import ran: they go next. Guarded by this, with running and inFlight's additions. private val queued = LinkedHashSet() + // Captures that are not media, logged once each. + private val noticed = java.util.Collections.synchronizedSet(HashSet()) + // The import running, and when it last said or received anything. @Volatile private var running: Job? = null @@ -499,6 +502,9 @@ class MediaImport( private fun onMedia(data: ServiceData) { when (data.messageType) { MT_CAPTURE -> parseCapture(data.payload)?.let { capture -> + if (!capture.isMedia && noticed.add(capture.handle)) { + say("Skipped a capture of kind ${capture.kindCode}: no photo or video in it, assets ${capture.assets.map { it.kind }}") + } _captures.update { list -> (list.filter { it.handle != capture.handle } + capture).sortedByDescending { it.timestampNs } } diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt new file mode 100644 index 00000000..d33a2dd8 --- /dev/null +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/Screencast.kt @@ -0,0 +1,204 @@ +package com.macedonga.visor.core.media + +import com.macedonga.visor.core.connection.GlassesConnection +import com.macedonga.visor.core.protocol.Wire +import com.macedonga.visor.core.session.LinkState +import com.macedonga.visor.core.session.ServiceData +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull + +const val SVC_HN_SCREENCAST_MANAGER = 23464 +const val SVC_HN_SCREENCAST = 23463 + +// The MobileConfig switch the glasses check before they run the screencast service. +const val SCREENCAST_FLAG = "hn_demos.enable_hn_screencast" + +enum class ScreencastPhase { IDLE, STARTING, LIVE, FAILED } + +enum class ScreencastFault { NOT_CONNECTED, NOT_READ, NO_FLAG, FLAG_NOT_SENT, NEEDS_RESTART, NO_WIFI, NO_ANSWER, ASLEEP, GLASSES_TIMEOUT, UNKNOWN } + +data class ScreencastState(val phase: ScreencastPhase = ScreencastPhase.IDLE, val fault: ScreencastFault? = null, val detail: String = "") + +/** One whole H.264 frame, its parameter sets in band. [flags] are MediaCodec's: key frame, codec config. */ +class ScreencastFrame(val timestampMs: Long, val flags: Int, val data: ByteArray) { + val key get() = flags and FLAG_KEY != 0 + val config get() = flags and FLAG_CONFIG != 0 + + companion object { + const val FLAG_KEY = 1 + const val FLAG_CONFIG = 2 + } +} + +/** + * The Display's own screen, streamed to the phone on svc23463 and driven on svc23464, over Wi-Fi as the Meta app runs + * it: the manager opened and answered, then the encoder settings and their answer, then the frames opened. The service + * is started by the glasses' init on the MobileConfig switch's sysprop, so a new switch needs the glasses restarted. + */ +class Screencast(private val connection: GlassesConnection, private val config: GlassesConfig, private val live: LiveStream) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val _state = MutableStateFlow(ScreencastState()) + val state: StateFlow = _state.asStateFlow() + + private val _frames = MutableSharedFlow(extraBufferCapacity = 8, onBufferOverflow = BufferOverflow.DROP_OLDEST) + val frames: SharedFlow = _frames.asSharedFlow() + + private var job: Job? = null + private var answer: CompletableDeferred>? = null + private var stopping = CompletableDeferred() + + // The frame being assembled: its time, flags and the chunks still owed. + private var timestamp = 0L + private var flags = 0 + private var chunksLeft = 0 + private var frame = ByteArrayOutputStream() + + init { + scope.launch { connection.serviceData.collect { onData(it) } } + } + + /** Switch the glasses' screencast on and wait for their answer; the state says how it went. */ + @Synchronized + fun start() { + if (job?.isActive == true) return + job = scope.launch { run() } + } + + /** Stop the screencast on the glasses and forget the state. */ + @Synchronized + fun stop() { + // Live, the run sends the stop itself and lets Wi-Fi go; still starting, it is cut short. + if (_state.value.phase != ScreencastPhase.LIVE) job?.cancel() + stopping.complete(Unit) + _state.value = ScreencastState() + } + + private suspend fun run() { + fun fail(fault: ScreencastFault, detail: String = "") { + _state.value = ScreencastState(ScreencastPhase.FAILED, fault, detail) + } + _state.value = ScreencastState(ScreencastPhase.STARTING) + if (connection.state.value.link != LinkState.READY) return fail(ScreencastFault.NOT_CONNECTED) + val cfg = config.state.value + if (cfg.params.isEmpty()) return fail(ScreencastFault.NOT_READ) + if (SCREENCAST_FLAG !in cfg.params) return fail(ScreencastFault.NO_FLAG) + if (cfg.values[SCREENCAST_FLAG] != true) { + config.send(mapOf(SCREENCAST_FLAG to true))?.let { return fail(ScreencastFault.FLAG_NOT_SENT, it) } + return fail(ScreencastFault.NEEDS_RESTART) + } + stopping = CompletableDeferred() + val why = live.overWifi { mux -> + coroutineScope { + val listening = launch { live.wifiData.collect { onData(it) } } + try { + // The Meta app waits for the manager's answer to the open, then to the settings, then opens the frames. + // An open left unanswered goes on to the settings, as Visor did before. + val opened = ask(OPEN_MS) { mux.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONTROL, byteArrayOf(0)) } + val got = if (opened == null || opened.first) { + ask { mux.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONFIGURE, configure()) } + } else opened + if (got?.first == true) { + mux.sendOnService(SVC_HN_SCREENCAST, MT_CONTROL, byteArrayOf(0)) + _state.value = ScreencastState(ScreencastPhase.LIVE) + stopping.await() + mux.sendOnService(SVC_HN_SCREENCAST_MANAGER, MT_CONTROL, byteArrayOf(3)) + delay(STOP_MS) + } else refused(got) + } finally { + listening.cancel() + } + } + null + } + if (why != null) fail(ScreencastFault.NO_WIFI, why) + } + + /** + * Send, and wait for the manager's answer: success, and why not. A 0xC001 from the glasses also answers, as + * "not running": the switch is on but the glasses have not started the service since it was set. + */ + private suspend fun ask(waitMs: Long = ANSWER_MS, send: suspend () -> Unit): Pair? = coroutineScope { + val reply = CompletableDeferred>().also { answer = it } + val absent = launch(start = CoroutineStart.UNDISPATCHED) { + live.notes.first { it.contains("service $SVC_HN_SCREENCAST_MANAGER answered status 0xc001") } + reply.complete(Pair(false, NOT_RUNNING)) + } + send() + withTimeoutOrNull(waitMs) { reply.await() }.also { + absent.cancel() + answer = null + } + } + + private fun refused(got: Pair?) { + _state.value = ScreencastState( + ScreencastPhase.FAILED, + when { + got == null -> ScreencastFault.NO_ANSWER + got.second == NOT_RUNNING -> ScreencastFault.NEEDS_RESTART + got.second == 2 -> ScreencastFault.ASLEEP + got.second == 3 -> ScreencastFault.GLASSES_TIMEOUT + else -> ScreencastFault.UNKNOWN + }, + ) + } + + private fun onData(m: ServiceData) { + when (m.serviceId) { + SVC_HN_SCREENCAST_MANAGER -> { + val reply = runCatching { Wire.parse(m.payload) }.getOrNull() ?: return + answer?.complete(Pair(reply.varint(1) != 0L, reply.varint(2).toInt())) + } + SVC_HN_SCREENCAST -> chunk(m.payload) + } + } + + // A 32-byte message opens a frame; every other one is the next chunk of the open frame. + private fun chunk(p: ByteArray) { + if (p.size == HEADER_BYTES) { + val b = ByteBuffer.wrap(p).order(ByteOrder.LITTLE_ENDIAN) + timestamp = b.getLong(0) + flags = b.getInt(8) + chunksLeft = b.getInt(24) + frame = ByteArrayOutputStream() + return + } + if (chunksLeft <= 0) return + frame.write(p) + if (--chunksLeft == 0) _frames.tryEmit(ScreencastFrame(timestamp, flags, frame.toByteArray())) + } + + // Five little-endian int32s: bitrate, I-frame interval, 2^profile, 2^level, framerate. These are the Meta app's defaults. + private fun configure(): ByteArray = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN) + .putInt(100_000).putInt(1_000_000).putInt(1).putInt(1).putInt(5) + .array() + + private companion object { + const val MT_CONTROL = 129 + const val MT_CONFIGURE = 1 + const val HEADER_BYTES = 32 + const val ANSWER_MS = 15_000L + const val STOP_MS = 300L + const val OPEN_MS = 3_000L + const val NOT_RUNNING = -1 + } +} diff --git a/core/media/src/main/kotlin/com/macedonga/visor/core/media/VideoCodec.kt b/core/media/src/main/kotlin/com/macedonga/visor/core/media/VideoCodec.kt new file mode 100644 index 00000000..0169eaa4 --- /dev/null +++ b/core/media/src/main/kotlin/com/macedonga/visor/core/media/VideoCodec.kt @@ -0,0 +1,93 @@ +package com.macedonga.visor.core.media + +import android.media.MediaFormat + +/** The video codecs a glasses stream may carry: HEVC from the Gen 2, H.264 (AVC) possibly from the Display. */ +enum class VideoCodec(val mime: String, val label: String) { + HEVC(MediaFormat.MIMETYPE_VIDEO_HEVC, "HEVC"), + AVC(MediaFormat.MIMETYPE_VIDEO_AVC, "H.264"), +} + +/** + * The codec of Annex-B bytes, from an HEVC VPS, SPS or PPS (0x40, 0x42, 0x44 then 0x01) or an H.264 SPS + * (nal_ref_idc set, type 7). HEVC base-layer units never start with a byte that reads as the H.264 SPS, so the + * two cannot be confused. Null when the bytes show neither, such as slices with no parameter sets. + */ +fun detectVideoCodec(data: ByteArray): VideoCodec? { + val units = annexBUnits(data) + if (units.any { it.size >= 2 && (it[0].toInt() and 0xFF) in setOf(0x40, 0x42, 0x44) && it[1].toInt() == 1 }) return VideoCodec.HEVC + if (units.any { isAvcSpsHeader(it[0].toInt() and 0xFF) }) return VideoCodec.AVC + return null +} + +private fun isAvcSpsHeader(b: Int): Boolean = (b and 0x80) == 0 && (b and 0x1F) == 7 && (b and 0x60) != 0 + +/** The codec of a config, HEVC unless it shows H.264. */ +fun configCodec(config: ByteArray): VideoCodec = detectVideoCodec(config) ?: VideoCodec.HEVC + +/** Width and height from the first SPS in [config], whichever codec it is in. */ +fun videoSize(config: ByteArray): Pair? = when (configCodec(config)) { + VideoCodec.HEVC -> hevcParameterSets(config)[33]?.firstOrNull()?.let(::parseHevcSps)?.let { it.width to it.height } + VideoCodec.AVC -> annexBUnits(config).firstOrNull { (it[0].toInt() and 0x1F) == 7 }?.let(::parseAvcSps) +} + +/** Parameter sets: HEVC VPS, SPS, PPS (32 to 34); H.264 SPS and PPS (7, 8). */ +fun VideoCodec.isParameterSet(unit: ByteArray): Boolean = when (this) { + VideoCodec.HEVC -> hevcNalType(unit) in 32..34 + VideoCodec.AVC -> (unit[0].toInt() and 0x1F) in 7..8 +} + +/** A keyframe: HEVC IRAP units (16 to 21), H.264 IDR slices (5). */ +fun VideoCodec.isKeyFrame(data: ByteArray): Boolean = when (this) { + VideoCodec.HEVC -> nalTypes(data).any { it in 16..21 } + VideoCodec.AVC -> annexBUnits(data).any { (it[0].toInt() and 0x1F) == 5 } +} + +private val HIGH_PROFILES = setOf(100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135) + +/** Width and height of an H.264 SPS, cropping applied. Streams with scaling matrices are not read. */ +fun parseAvcSps(unit: ByteArray): Pair? = runCatching { + val r = BitReader(unescape(unit), 1) + val profile = r.bits(8) + r.bits(16) // constraint flags and level + r.ue() // seq_parameter_set_id + var chroma = 1 + if (profile in HIGH_PROFILES) { + chroma = r.ue() + if (chroma == 3) r.bits(1) + r.ue() + r.ue() // bit depths + r.bits(1) // qpprime_y_zero_transform_bypass + check(r.bits(1) == 0) { "scaling matrices" } + } + r.ue() // log2_max_frame_num_minus4 + when (r.ue()) { + 0 -> r.ue() + 1 -> { + r.bits(1) + r.ue() + r.ue() + repeat(r.ue()) { r.ue() } + } + } + r.ue() // max_num_ref_frames + r.bits(1) // gaps_in_frame_num_value_allowed_flag + val widthMbs = r.ue() + 1 + val mapUnits = r.ue() + 1 + val frameMbsOnly = r.bits(1) + if (frameMbsOnly == 0) r.bits(1) // mb_adaptive_frame_field_flag + r.bits(1) // direct_8x8_inference_flag + var left = 0 + var right = 0 + var top = 0 + var bottom = 0 + if (r.bits(1) == 1) { + left = r.ue() + right = r.ue() + top = r.ue() + bottom = r.ue() + } + val cropX = if (chroma == 1 || chroma == 2) 2 else 1 + val cropY = (if (chroma == 1) 2 else 1) * (2 - frameMbsOnly) + (widthMbs * 16 - cropX * (left + right)) to ((2 - frameMbsOnly) * mapUnits * 16 - cropY * (top + bottom)) +}.getOrNull() diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt new file mode 100644 index 00000000..e0914b4b --- /dev/null +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Band.kt @@ -0,0 +1,227 @@ +package com.macedonga.visor.core.protocol + +/** + * The Neural Band's RPC on svc52822, as Meta AI 294.1 speaks it (com.oculus.wearableinputservice). A request goes out + * as DataX type 0x0314, whose low byte is the band's REQUEST; what the band sends carries its own type in the low byte. + */ +object Band { + const val REQUEST_TYPE = 0x0314 + const val RESPONSE = 21 + const val STREAM_UPDATE = 22 + + /** The answer's code when the band did what it was asked. */ + const val OK = 0 + + /** The PSM the band's GATT names; the glasses' is 128. */ + const val PSM = 129 + + class Battery(val percent: Int, val charging: Boolean) + + /** RpcResponse {1: seq, 2: code, 3: BatteryInfoResp, 4: DeviceInfoResp, 6: ConfigResp, 18: HapticsResp, ...}, as far as Visor reads it. */ + class Response( + val seq: Int, + val code: Int, + val battery: Battery?, + val leftHanded: Boolean?, + val protocol: Int? = null, + val haptics: Int? = null, + ) + + /** + * A gesture, `EmgImu$GestureEvent`, sent as stream type 13 or a granular type from 60 to 200: {1 seq, 3 finger, 4 action, + * 5 derived action}. Fingers: THUMB 1, INDEX 2, MIDDLE 3. Actions: PRESS 1, RELEASE 2, TAP 3, DOUBLETAP 4, CLICK 5, UP 6, + * DOWN 7, LEFT 8, RIGHT 9. Derived: SINGLE_TAP 1, DOUBLE_TAP 2, BUTTON_HOLD 3, BUTTON_RELEASE 4, the four directions 5-8. + */ + class Gesture(val type: Int, val seq: Int, val finger: Int, val action: Int, val derived: Int) + + /** RpcRequest {1: seq, 2: BatteryInfoReq {}}. */ + fun batteryRequest(seq: Int): ByteArray = request(seq, 2, ByteArray(0)) + + /** RpcRequest {1: seq, 3: DeviceInfoReq {}}: the answer names the band's protocol version (field 15). */ + fun deviceInfoRequest(seq: Int): ByteArray = request(seq, 3, ByteArray(0)) + + /** + * RpcRequest {1: seq, 4: StreamControlReq}: the gestures on or off, as the Meta app asks. From protocol 2 a map by stream + * type (field 36) with priority 90 (35) and the use case "GenericC50" (37); before it the one bool, enableGestures (3). + */ + fun gestureStreams(seq: Int, on: Boolean, protocol: Int?): ByteArray { + val map = protocol == null || protocol >= 2 + val entries = if (map) GESTURE_STREAMS.fold(ByteArray(0)) { out, type -> + out + Wire.fieldBytes(36, Wire.fieldVarint(1, type.toLong()) + Wire.fieldVarint(2, if (on) 1 else 0)) + } + Wire.fieldVarint(35, PRIORITY_GENERIC_C50) + Wire.fieldBytes(37, "GenericC50".encodeToByteArray()) else ByteArray(0) + val bool = if (!map || protocol == null) Wire.fieldVarint(3, if (on) 1 else 0) else ByteArray(0) + return request(seq, 4, bool + entries) + } + + /** RpcRequest {1: seq, 11: GestureAckReq {1 the gesture's seq, 2 source MWA_ANDROID, 3 topology DIRECT}}, one per gesture. */ + fun gestureAck(seq: Int, gesture: Int): ByteArray = + request(seq, 11, Wire.fieldVarint(1, gesture.toLong()) + Wire.fieldVarint(2, SOURCE_MWA_ANDROID) + Wire.fieldVarint(3, TOPOLOGY_DIRECT)) + + /** RpcRequest {1: seq, 17: HapticsReq {1: VibrateReq {1 duration, 2 frequency, 3 amplitude}}}; the units are not known. */ + fun vibrate(seq: Int, duration: Long, frequency: Float, amplitude: Int): ByteArray = + request(seq, 17, Wire.fieldBytes(1, Wire.fieldVarint(1, duration) + Wire.fieldFloat(2, frequency) + Wire.fieldVarint(3, amplitude.toLong()))) + + /** RpcRequest {1: seq, 17: HapticsReq {3: StopReq {}}}. */ + fun stopHaptics(seq: Int): ByteArray = request(seq, 17, Wire.fieldBytes(3, ByteArray(0))) + + // The firmware update: Ota on RpcRequest 7 (answers in RpcResponse 8), the bulk file transfer on 9 (answers in 10). + const val OTA_ITEM_FIRMWARE = 0 + const val OTA_ITEM_ML_MODEL = 2 + const val OTA_UNAVAILABLE = 0 + // otaUnavailableReason when nothing stands in the way. + const val OTA_NO_REASON = 0 + const val OTA_IN_PROGRESS = 2 + const val OTA_RESULT_OK = 0 + const val TRANSFER_OTA = 1 + const val TRANSFER_NEW = 1 + const val TRANSFER_PARTIAL = 2 + const val TRANSFER_FULL = 3 + const val CHUNK_BYTES = 950 + + /** The band's OTA device info: the firmware and ML versions by item, the KEK id, the OTA state and why it is unavailable. */ + class OtaInfo(val firmware: Long, val ml: Long, val kek: Long, val state: Int, val unavailable: Int) + + /** BulkDataTransferResp: the info answer, a chunk's ack or a nack. */ + sealed interface Bulk { + class Info(val state: Int, val memory: Long, val resume: Long, val transferId: Int) : Bulk + class Ack(val transferId: Int, val offset: Long) : Bulk + class Nack(val transferId: Int, val reason: String) : Bulk + } + + /** RpcRequest {1: seq, 7: OtaReq {1: DeviceInfoRequest {}}}. */ + fun otaDeviceInfo(seq: Int): ByteArray = ota(seq, 1, ByteArray(0)) + + /** OtaReq {2: StartItemUpdateRequest {1: ItemInfo, 2: size, 4: kekVersion}}; the version check (3) is left as the band has it. */ + fun otaStartItem(seq: Int, itemType: Int, version: Long, size: Long): ByteArray = + ota(seq, 2, Wire.fieldBytes(1, itemInfo(itemType, version)) + Wire.fieldVarint(2, size) + Wire.fieldVarint(4, 0)) + + /** OtaReq {3: VerifyItemUpdateRequest {1: ItemInfo}}. */ + fun otaVerifyItem(seq: Int, itemType: Int, version: Long): ByteArray = ota(seq, 3, Wire.fieldBytes(1, itemInfo(itemType, version))) + + /** OtaReq {4: CommitUpdateRequest {}}: one commit for all items, the reboot not deferred. */ + fun otaCommit(seq: Int): ByteArray = ota(seq, 4, ByteArray(0)) + + /** OtaReq {5: AbortUpdateRequest {}}. */ + fun otaAbort(seq: Int): ByteArray = ota(seq, 5, ByteArray(0)) + + /** OtaReq {6: InstallKEKRequest {1: kekVersion, 3: encryptedKek}}; the key type is 0, the default, so not sent. */ + fun otaInstallKek(seq: Int, kekVersion: Long, encryptedKek: ByteArray): ByteArray = + ota(seq, 6, Wire.fieldVarint(1, kekVersion) + Wire.fieldBytes(3, encryptedKek)) + + /** RpcRequest {1: seq, 9: BulkDataTransferReq {1: InfoRequest {1 MD5, 2 size, 3 type, 4 transferID}}}. */ + fun bulkInfo(seq: Int, md5: ByteArray, size: Long, transferId: Int): ByteArray = request( + seq, 9, + Wire.fieldBytes(1, Wire.fieldBytes(1, md5) + Wire.fieldVarint(2, size) + Wire.fieldVarint(3, TRANSFER_OTA.toLong()) + Wire.fieldVarint(4, transferId.toLong())), + ) + + /** RpcRequest {1: seq, 9: BulkDataTransferReq {2: ChunkData {1 transferID, 2 chunkOffset, 3 chunkData}}}. */ + fun bulkChunk(seq: Int, transferId: Int, offset: Long, data: ByteArray): ByteArray = request( + seq, 9, Wire.fieldBytes(2, Wire.fieldVarint(1, transferId.toLong()) + Wire.fieldVarint(2, offset) + Wire.fieldBytes(3, data)), + ) + + /** The seq an answer echoes. */ + fun seqOf(payload: ByteArray): Int = Wire.parse(payload).varint(1).toInt() + + /** The OTA device info in an answer (OtaResp 2), or null when it is another answer. */ + fun otaInfo(payload: ByteArray): OtaInfo? { + val ota = otaAnswer(payload) ?: return null + if (!ota.has(2)) return null + val info = Wire.parse(ota.bytes(2)) + val items = info.repeatedBytes(5).map { Wire.parse(it) }.associate { it.varint(1).toInt() to it.varint(2) } + return OtaInfo( + firmware = items[OTA_ITEM_FIRMWARE] ?: 0, + ml = items[OTA_ITEM_ML_MODEL] ?: 0, + kek = info.varint(6), + state = info.varint(4).toInt(), + unavailable = info.varint(7).toInt(), + ) + } + + /** The result code of a GeneralResponse (1), VerifyItemUpdateResponse (3) or CommitUpdateResponse (4); null for other answers. */ + fun otaResult(payload: ByteArray): Int? { + val ota = otaAnswer(payload) ?: return null + val field = listOf(1, 3, 4).firstOrNull { ota.has(it) } ?: return null + return Wire.parse(Wire.parse(ota.bytes(field)).bytes(1)).varint(1).toInt() + } + + /** The bulk answer in an RpcResponse 10, or null for another answer. */ + fun bulk(payload: ByteArray): Bulk? { + val rpc = Wire.parse(payload) + if (!rpc.has(10)) return null + val answer = Wire.parse(rpc.bytes(10)) + return when { + answer.has(1) -> Wire.parse(answer.bytes(1)).let { Bulk.Info(it.varint(1).toInt(), it.varint(2), it.varint(3), it.varint(5).toInt()) } + answer.has(2) -> Wire.parse(answer.bytes(2)).let { Bulk.Ack(it.varint(1).toInt(), it.varint(2)) } + answer.has(3) -> Wire.parse(answer.bytes(3)).let { Bulk.Nack(it.varint(1).toInt(), it.bytes(2).decodeToString()) } + else -> null + } + } + + private fun otaAnswer(payload: ByteArray): Wire.Fields? = Wire.parse(payload).let { if (it.has(8)) Wire.parse(it.bytes(8)) else null } + + private fun ota(seq: Int, field: Int, body: ByteArray): ByteArray = request(seq, 7, Wire.fieldBytes(field, body)) + + private fun itemInfo(itemType: Int, version: Long): ByteArray = Wire.fieldVarint(1, itemType.toLong()) + Wire.fieldVarint(2, version) + + fun isGesture(type: Int): Boolean = type == 13 || type in 60..200 + + fun gesture(type: Int, payload: ByteArray): Gesture { + val f = Wire.parse(payload) + return Gesture(type, f.varint(1).toInt(), f.varint(3).toInt(), f.varint(4).toInt(), f.varint(5).toInt()) + } + + /** RpcRequest {1: seq, 5: ConfigReq}: with [leftHanded] it sets the wrist (field 10), empty it reads the config. */ + fun configRequest(seq: Int, leftHanded: Boolean? = null): ByteArray = + request(seq, 5, leftHanded?.let { Wire.fieldVarint(10, if (it) 1 else 0) } ?: ByteArray(0)) + + fun response(payload: ByteArray): Response { + val f = Wire.parse(payload) + val battery = if (f.has(3)) battery(Wire.parse(f.bytes(3)).bytes(1)) else null + val left = if (f.has(6)) Wire.parse(f.bytes(6)).let { c -> if (c.has(10)) c.varint(10) != 0L else null } else null + val protocol = if (f.has(4)) Wire.parse(f.bytes(4)).let { d -> if (d.has(15)) d.varint(15).toInt() else null } else null + // HapticsResp {1: code}, in field 18 of the response. + val haptics = if (f.has(18)) Wire.parse(f.bytes(18)).let { h -> if (h.has(1)) h.varint(1).toInt() else null } else null + return Response(f.varint(1).toInt(), f.varint(2).toInt(), battery, left, protocol, haptics) + } + + /** RpcStreamUpdate field 2, BatteryUpdate {1: BatteryData}. */ + fun streamBattery(payload: ByteArray): Battery? { + val f = Wire.parse(payload) + return if (f.has(2)) battery(Wire.parse(f.bytes(2)).bytes(1)) else null + } + + // BatteryData {1: level, 2: charging, 3: temperature (float), 4: mV, 5: mA, 6: RTC seconds}. + private fun battery(data: ByteArray): Battery? { + val d = Wire.parse(data) + return if (d.has(1)) Battery(d.varint(1).toInt(), d.varint(2) != 0L) else null + } + + private fun request(seq: Int, field: Int, body: ByteArray) = Wire.fieldVarint(1, seq.toLong()) + Wire.fieldBytes(field, body) + + // Every gesture stream the Meta app turns on: 60-70, 100-109, 150-156 and 200. + private val GESTURE_STREAMS = (60..70) + (100..109) + (150..156) + 200 + private const val PRIORITY_GENERIC_C50 = 90L + private const val SOURCE_MWA_ANDROID = 2L + private const val TOPOLOGY_DIRECT = 0L +} + +/** svc150's NotifyLease {1: peerKeyHint, 2: leaseType}: the Display taking its band. Type 1 on the channel it opens. */ +class NotifyLease(val keyHint: ByteArray, val type: Int) { + /** A relay lease leaves the band's own phone link up; the others ask the phone to step back. */ + val relay: Boolean get() = type == RELAY + + companion object { + const val MESSAGE_TYPE = 1 + const val RELAY = 1 + + // The phone's answers: Meta's DeviceLeaseService sends Error 0 ("OK") once the lease is held. + const val OK = 0x0000 + const val UNKNOWN_TYPE = 0xD001 + const val BAD_REQUEST = 0xD010 + + fun decode(payload: ByteArray): NotifyLease { + val f = Wire.parse(payload) + return NotifyLease(f.bytes(1), f.varint(2).toInt()) + } + } +} diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Constants.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Constants.kt index 3839a172..572fdc0a 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Constants.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Constants.kt @@ -140,7 +140,10 @@ enum class ServiceId(val value: Int) { AIRSHIELD_AUTH(54), CONSTELLATION_AUTH(79), - /** `wireless_input_service` in the firmware's registry. */ + /** `conn_device_lease`: the Display asks the phone for its band here. */ + CONN_DEVICE_LEASE(150), + + /** `wireless_input_service` in the firmware's registry; the Neural Band's whole RPC set. */ WIRELESS_INPUT(52822), ; diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Kdf.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Kdf.kt index 7b373e44..80333975 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Kdf.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Kdf.kt @@ -10,6 +10,9 @@ object Kdf { val AIRSHIELD_INFO: ByteArray = "AirShield".toByteArray(Charsets.US_ASCII) val HMAC_DERIVE_LABEL: ByteArray = "hmac_derive".toByteArray(Charsets.US_ASCII) + /** EncryptionParameters bit 4: a separate HMAC key. The Neural Band leaves it out (27, the glasses 31). */ + const val MULTIKEY: Int = 4 + /** Derive ONE direction's key set, in the live-confirmed construction. */ fun deriveDirectionKeys( ecdhSecret: ByteArray, @@ -17,13 +20,16 @@ object Kdf { seed: ByteArray, preambleIv: ByteArray, info: ByteArray = AIRSHIELD_INFO, + // Without Multikey on both ends the AES key is the HMAC key too (CipherBuilder::buildFramingKeys). + multikey: Boolean = true, ): SessionKeys { require(ecdhSecret.size == 32) { "ECDH secret must be 32 bytes, got ${ecdhSecret.size}" } require(preambleIv.size == 16) { "preamble IV must be 16 bytes, got ${preambleIv.size}" } val aesSalt = sha256(nonce, seed) - val hmacSalt = sha256(seed, nonce, HMAC_DERIVE_LABEL) val aesKey = hkdfExpand(hkdfExtract(aesSalt, ecdhSecret), info, 32) + if (!multikey) return SessionKeys(aesKey = aesKey, hmacKey = aesKey, iv = preambleIv) + val hmacSalt = sha256(seed, nonce, HMAC_DERIVE_LABEL) val hmacKey = hkdfExpand(hkdfExtract(hmacSalt, ecdhSecret), info, 32) return SessionKeys(aesKey = aesKey, hmacKey = hmacKey, iv = preambleIv) } diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Messages.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Messages.kt index 14a2497e..fbbb4bde 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Messages.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Messages.kt @@ -151,7 +151,8 @@ class EnableEncryption( seed = f.bytes(2), iv = f.bytes(3), base = f.varint(4), - parameters = f.varint(5).toInt(), + // Left out, the glasses mean 31: Multikey on. + parameters = if (f.has(5)) f.varint(5).toInt() else 31, quirks = f.varint(6).toInt(), phasedLinkSetupSupported = f.varint(7).toInt(), supportedLinkSetupServices = f.varint(8), diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Thrift.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Thrift.kt index 5b318d5f..4aa557a4 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Thrift.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Thrift.kt @@ -228,6 +228,30 @@ object Thrift { fun loads(b: ByteArray): Map = readStruct(b, 0).fields + /** The bytes of top-level struct field [fid] as written, to pass a sub-struct on unread. */ + fun rawStruct(b: ByteArray, fid: Int): ByteArray? { + var i = 0 + var cur = 0 + while (i < b.size) { + val h = b[i].toInt() and 0xFF + if (h == 0) break + i += 1 + val delta = (h shr 4) and 0x0F + val t = h and 0x0F + if (delta == 0) { + val v = uvarint(b, i) + i = v.offset + cur = zz(v.value).toInt() + } else { + cur += delta + } + val end = readValue(b, i, t).offset + if (cur == fid) return if (t == STRUCT) b.copyOfRange(i, end) else null + i = end + } + return null + } + // `thrift.py` also has `render`, a debug pretty-printer whose output is Python repr, and // a __main__ block that dumps a file from the command line. Neither is ported. diff --git a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt index 4278ac09..cdb722f7 100644 --- a/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt +++ b/core/protocol/src/main/kotlin/com/macedonga/visor/core/protocol/Wire.kt @@ -53,6 +53,18 @@ object Wire { fun fieldBytes(fieldNumber: Int, value: ByteArray): ByteArray = tag(fieldNumber, 2) + encodeVarint(value.size.toLong()) + value + /** A float as protobuf writes it: fixed32, little-endian. */ + fun fieldFloat(fieldNumber: Int, value: Float): ByteArray { + val bits = java.lang.Float.floatToRawIntBits(value) + return tag(fieldNumber, 5) + ByteArray(4) { i -> (bits ushr (8 * i)).toByte() } + } + + /** A double as protobuf writes it: fixed64, little-endian. */ + fun fieldDouble(fieldNumber: Int, value: Double): ByteArray { + val bits = java.lang.Double.doubleToRawLongBits(value) + return tag(fieldNumber, 1) + ByteArray(8) { i -> (bits ushr (8 * i)).toByte() } + } + // --- decode ------------------------------------------------------------ /** A varint and the offset just past it. */ @@ -101,6 +113,22 @@ object Wire { return default } + /** The first fixed32 value for [fn] as a float, or [default]. */ + fun float(fn: Int, default: Float = 0f): Float { + byField[fn]?.forEach { (wireType, value) -> + if (wireType == 5) return Float.fromBits(value as Int) + } + return default + } + + /** The first fixed64 value for [fn] as a double, or [default]. */ + fun double(fn: Int, default: Double = 0.0): Double { + byField[fn]?.forEach { (wireType, value) -> + if (wireType == 1) return Double.fromBits(value as Long) + } + return default + } + fun repeatedBytes(fn: Int): List = byField[fn].orEmpty().filter { it.first == 2 }.map { it.second as ByteArray } @@ -110,7 +138,13 @@ object Wire { fun fieldNumbers(): List = byField.keys.sorted() } - /** Parse a flat protobuf message: varint and length-delimited fields. */ + private fun littleEndian(data: ByteArray, at: Int, size: Int): Long { + var v = 0L + for (i in size - 1 downTo 0) v = (v shl 8) or (data[at + i].toLong() and 0xFF) + return v + } + + /** Parse a flat protobuf message: varint, fixed and length-delimited fields. */ fun parse(data: ByteArray): Fields { val fields = Fields() var offset = 0 @@ -144,8 +178,14 @@ object Wire { offset = if (fits) offset + take else n } - 5 -> offset += 4 // fixed32 - 1 -> offset += 8 // fixed64 + 5 -> { // fixed32, little-endian + if (offset + 4 <= n) fields.add(fieldNumber, 5, littleEndian(data, offset, 4).toInt()) + offset += 4 + } + 1 -> { // fixed64, little-endian + if (offset + 8 <= n) fields.add(fieldNumber, 1, littleEndian(data, offset, 8)) + offset += 8 + } else -> throw ProtocolFormatException( "unsupported wire type ${key.value and 0x07L} for field $fieldNumber" ) diff --git a/core/session/src/main/kotlin/com/macedonga/visor/core/session/DataXConnection.kt b/core/session/src/main/kotlin/com/macedonga/visor/core/session/DataXConnection.kt index da729199..83143a87 100644 --- a/core/session/src/main/kotlin/com/macedonga/visor/core/session/DataXConnection.kt +++ b/core/session/src/main/kotlin/com/macedonga/visor/core/session/DataXConnection.kt @@ -171,6 +171,8 @@ class DataXConnection( append("\n msg ch=0x%04x svc=%d mt=0x%04x len=%d hdr=%d payload=%d".format( m.info.channelId, m.info.serviceId, m.info.messageType, m.info.length, m.info.headerLen, m.payload.size, )) + // Small ones whole: what a new service says is read from here. + if (m.payload.size <= TRACE_PAYLOAD) append(" data=").append(hex(m.payload)) } if (carry.isNotEmpty()) append("\n CARRY %d".format(carry.size)) } @@ -216,6 +218,8 @@ class DataXConnection( @Volatile var trace: ((String) -> Unit)? = null + private const val TRACE_PAYLOAD = 512 + /** The ack channel for a peer-originated channel: clear the secure bit, 0x8000. */ fun mirror(channelId: Int): Int = channelId and 0x7FFF } diff --git a/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt b/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt index a5bd0a3e..407a7cad 100644 --- a/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt +++ b/core/session/src/main/kotlin/com/macedonga/visor/core/session/GlassesSession.kt @@ -206,9 +206,7 @@ class GlassesSession( withLinkDeadline(END_LINK_SETUP_TIMEOUT_MILLIS, "the glasses did not send their EndLinkSetup") { while (true) { val message = mux.receive() - if (message.info.channelId == LINK_SETUP_CHANNEL && - message.info.messageType == LinkSetupMsgType.END_LINK_SETUP.value - ) { + if (isTheirEndLinkSetup(message)) { mux.sendStatus(DataXConnection.mirror(message.info.channelId), SUCCESS) return@withLinkDeadline } @@ -217,6 +215,15 @@ class GlassesSession( } } + // The glasses answer on the link setup channel our RequestEncryption opened; the Neural Band on one of its own, + // 0x9001, that names no service. + private fun isTheirEndLinkSetup(message: DataXMessage): Boolean { + val info = message.info + if (info.messageType != LinkSetupMsgType.END_LINK_SETUP.value || info.flags.hasError) return false + return info.channelId == LINK_SETUP_CHANNEL || + ((info.channelId and 0x8000) != 0 && info.serviceId == 0 && registry.serviceForInbound(info.channelId) == 0) + } + /** Drain what the glasses said during link setup before anyone allocates a local channel. */ private suspend fun settleChannels(mux: DataXConnection) { while (!stopped) { @@ -260,6 +267,15 @@ class GlassesSession( mux.sendStatus(DataXConnection.mirror(info.channelId), SERVICE_NOT_FOUND) } + // svc4, connection_monitor: the glasses' delivery check (type 0, a counter) is confirmed with a bare OK status. + if (serviceId == CONNECTION_MONITOR && + (info.channelId and 0x8000) != 0 && + !info.flags.hasError && + !info.flags.isClosed + ) { + mux.sendStatus(DataXConnection.mirror(info.channelId), OK) + } + val responder = responders[serviceId] if (responder != null && !info.flags.hasError) { try { @@ -385,12 +401,17 @@ class GlassesSession( private val BACKOFF_MILLIS = longArrayOf(1_000, 2_000, 4_000, 8_000) - /** 0x1000 is the DataX success status every service answers with. */ + /** 0x1000 is the success status svc36 answers with; DataX's own is [OK]. */ private const val SUCCESS = 0x1000 /** DataX service_not_found: what a peer answers for a service it does not host. */ private const val SERVICE_NOT_FOUND = 0xC001 private val REFUSED_PEER_SERVICES = setOf(30, 79) + + /** DataX OK, the status a delivery check is confirmed with. */ + private const val OK = 0 + + private const val CONNECTION_MONITOR = 4 } } diff --git a/core/session/src/main/kotlin/com/macedonga/visor/core/session/SecureLink.kt b/core/session/src/main/kotlin/com/macedonga/visor/core/session/SecureLink.kt index 1c264483..9da4a279 100644 --- a/core/session/src/main/kotlin/com/macedonga/visor/core/session/SecureLink.kt +++ b/core/session/src/main/kotlin/com/macedonga/visor/core/session/SecureLink.kt @@ -137,10 +137,12 @@ class SecureLink( ecdhSecret = secret, ) + // What both ends support is what the link uses: the band offers 27, no Multikey. + val multikey = (OUR_PARAMETERS and request.supportedParameters and enable.parameters and Kdf.MULTIKEY) != 0 // Keep exactly as measured: TX takes the peer's RequestEncryption challenge as nonce and // our seed. - val rxKeys = Kdf.deriveDirectionKeys(secret, myChallenge, enable.seed, enable.iv) - val txKeys = Kdf.deriveDirectionKeys(secret, request.challenge, mySeed, myIv) + val rxKeys = Kdf.deriveDirectionKeys(secret, myChallenge, enable.seed, enable.iv, multikey = multikey) + val txKeys = Kdf.deriveDirectionKeys(secret, request.challenge, mySeed, myIv, multikey = multikey) rxCrypto = FrameCrypto(rxKeys, initRxSeq = enable.base.toInt()) txCrypto = FrameCrypto(txKeys, initTxSeq = mySeq.toInt()) @@ -206,13 +208,21 @@ class SecureLink( txMutex.withLock { transport.send(sdu) } } - /** Seal and send under one lock, and without a cancellation point between them. */ + /** + * Seal and send under one lock, and without a cancellation point between them. A message longer than one frame + * runs on into the next frames, as the glasses send theirs; the lock keeps another message out of the middle. + */ private suspend fun sealAndSend(plaintext: ByteArray) { txMutex.withLock { val crypto = checkNotNull(txCrypto) { "link not open" } withContext(NonCancellable) { - // Framing.pack adds the 0x40 type byte: [0x40][8B tag][1B blockCount][ct]. - transport.send(Framing.pack(FrameType.DATA, 0, crypto.seal(plaintext))) + var at = 0 + do { + val end = minOf(plaintext.size, at + FRAME_PLAINTEXT) + // Framing.pack adds the 0x40 type byte: [0x40][8B tag][1B blockCount][ct]. + transport.send(Framing.pack(FrameType.DATA, 0, crypto.seal(plaintext.copyOfRange(at, end)))) + at = end + } while (at < plaintext.size) } } } @@ -224,5 +234,11 @@ class SecureLink( /** How many frames [open] will read while waiting for the peer's two messages. */ const val HANDSHAKE_READS = 8 + + /** The parameters this side's RequestEncryption and EnableEncryption announce. */ + const val OUR_PARAMETERS = 31 + + /** The most one frame holds: 256 blocks, less the pad's one byte at least. */ + const val FRAME_PLAINTEXT = 256 * 16 - 1 } } \ No newline at end of file diff --git a/core/settings/src/main/kotlin/com/macedonga/visor/core/settings/GlassesSettings.kt b/core/settings/src/main/kotlin/com/macedonga/visor/core/settings/GlassesSettings.kt index 3265a29f..d1f60dfa 100644 --- a/core/settings/src/main/kotlin/com/macedonga/visor/core/settings/GlassesSettings.kt +++ b/core/settings/src/main/kotlin/com/macedonga/visor/core/settings/GlassesSettings.kt @@ -18,6 +18,7 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asSharedFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch @@ -75,6 +76,8 @@ data class SettingsState( val problem: String = "", // The Meta product id of the glasses last connected (266 for Ray-Ban Meta Gen 2), for what the catalogue offers. val product: Int? = null, + // The serial of the pair these are, so a reader never takes them for another's. + val pair: String? = null, ) /** @@ -90,7 +93,7 @@ class GlassesSettings(private val connection: GlassesConnection, drive: RlDriveR private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - private val _state = MutableStateFlow(SettingsState()) + private val _state = MutableStateFlow(SettingsState(pair = connection.pair.value)) val state: StateFlow = _state.asStateFlow() private val _notes = MutableSharedFlow(extraBufferCapacity = 64) @@ -128,8 +131,22 @@ class GlassesSettings(private val connection: GlassesConnection, drive: RlDriveR init { drive.handler = ::onReply + // Another pair's settings are not these: what was read goes, and the next read is a full one. scope.launch { - connection.state.map { it.info?.build }.distinctUntilChanged().collect { build -> + connection.pair.filterNotNull().distinctUntilChanged().collect { serial -> + reads.withLock { + if (_state.value.pair == serial) return@withLock + versions = emptyMap() + byNamespace = emptyMap() + raw.clear() + retried.clear() + writtenAt.clear() + _state.value = SettingsState(pair = serial) + } + } + } + scope.launch { + connection.state.map { it.target?.serial to it.info?.build }.distinctUntilChanged().collect { (_, build) -> productId(build)?.let { id -> _state.update { it.copy(product = id) } } } } diff --git a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json index 8c043b50..c4dc8b4c 100644 --- a/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json +++ b/core/settings/src/main/resources/com/macedonga/visor/core/settings/settings_ui.json @@ -1064,6 +1064,7 @@ "label_it":"Ascolta mentre registri", "note_it":"La riproduzione dell'audio continuerà mentre registri video", "products":[ +259, 262, 263, 264 @@ -4825,7 +4826,8 @@ }, "label_it":"Orientamento dei contenuti multimediali", "not_products":[ -258 +258, +259 ], "note_it":"Foto e video in verticale (3:4) o in orizzontale (4:3). Gli occhiali scattano sempre l'intera inquadratura verticale; Visor ritaglia quelli orizzontali a 4:3 quando li importa." }, @@ -5835,6 +5837,7 @@ }, "label_it":"Stabilizzazione video", "products":[ +259, 262, 263, 264, @@ -10798,7 +10801,8 @@ "label_it":"Modalità orizzontale", "note_it":"Il vecchio interruttore orizzontale. Visor lo tiene allineato all'orientamento dei contenuti multimediali.", "not_products":[ -258 +258, +259 ] }, { @@ -13014,10 +13018,36 @@ ], "Display":[ { +"n":1, +"label":"Item", +"type":"message", +"ref":"Item" +}, +{ "n":2, "label":"Link", "type":"string" } +], +"Item":[ +{ +"n":1, +"label":"Title", +"type":"string" +}, +{ +"n":2, +"label":"Icon", +"type":"message", +"ref":"Image" +} +], +"Image":[ +{ +"n":1, +"label":"Uri", +"type":"string" +} ] } } @@ -13328,10 +13358,7 @@ ] }, "label_it":"Risoluzione e frequenza dei fotogrammi", -"note_it":"La modifica della risoluzione o della frequenza dei fotogrammi influisce su autonomia della batteria, spazio, durata di registrazione e orientamento.", -"not_products":[ -259 -] +"note_it":"La modifica della risoluzione o della frequenza dei fotogrammi influisce su autonomia della batteria, spazio, durata di registrazione e orientamento." }, { "key":"sample_app_1_enabled", @@ -14996,10 +15023,7 @@ ] }, "label_it":"Durata del video", -"note_it":"Scegli una durata massima del video. Questi limiti aiutano a ottimizzare la condivisione e lo spazio.", -"not_products":[ -259 -] +"note_it":"Scegli una durata massima del video. Questi limiti aiutano a ottimizzare la condivisione e lo spazio." }, { "key":"video_fps", diff --git a/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt b/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt index 76556fa0..ea7fba41 100644 --- a/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt +++ b/core/transport/src/main/kotlin/com/macedonga/visor/core/transport/BluetoothLeTransport.kt @@ -12,6 +12,7 @@ import android.content.Context import android.content.pm.PackageManager import java.io.IOException import java.util.UUID +import java.util.concurrent.atomic.AtomicBoolean import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.Dispatchers @@ -37,6 +38,8 @@ class BluetoothLeTransport( private val wireLog: WireLog? = null, private val keeper: LeKeeper? = null, private val io: CoroutineDispatcher = Dispatchers.IO, + // The Neural Band's is 129. + private val defaultPsm: Int = META_LE_DEFAULT_PSM, ) : Transport { @Volatile private var socket: BluetoothSocket? = null @@ -51,19 +54,52 @@ class BluetoothLeTransport( } val device = context.getSystemService(BluetoothManager::class.java)?.adapter?.getRemoteDevice(address) ?: throw TransportConnectException(ConnectFailure.BLUETOOTH_OFF, "Bluetooth is off") - val psm = keeper?.psm ?: withTimeoutOrNull(GATT_MS) { readPsm(device) } ?: META_LE_DEFAULT_PSM - keeper?.note("opening L2CAP psm $psm, LE link ${if (keeper.connected.value) "up" else "down"}") + // The usual PSM first: reading it over GATT on a fresh link cost up to 6 s; read only when that one fails. + val known = keeper?.psm + val first = known ?: defaultPsm + keeper?.note("opening L2CAP psm $first, LE link ${if (keeper.connected.value) "up" else "down"}") + try { + open(device, first) + } catch (refused: TransportConnectException) { + if (known != null || refused.failure == ConnectFailure.PERMISSION_REFUSED) throw refused + val read = withTimeoutOrNull(GATT_MS) { readPsm(device) } + if (read == null || read == first) throw refused + keeper?.note("opening L2CAP psm $read, as the glasses name it") + open(device, read) + } + } + + @android.annotation.SuppressLint("MissingPermission") + private suspend fun open(device: android.bluetooth.BluetoothDevice, psm: Int) { withContext(io) { val opening = try { device.createInsecureL2capChannel(psm) } catch (error: SecurityException) { throw TransportConnectException(ConnectFailure.PERMISSION_REFUSED, "Visor may not use Bluetooth yet", error) } + // A healthy open takes a fraction of a second; after Visor died with a channel open, Android's own wait is + // 20 to 30 s for nothing. Closing the socket ends the wait. + // Whoever flips [settled] first wins: the timer closes the socket only if it gets there before the connect ends. + val settled = AtomicBoolean(false) + val timedOut = AtomicBoolean(false) + val watchdog = java.util.Timer(true).apply { + schedule(object : java.util.TimerTask() { + override fun run() { + if (!settled.compareAndSet(false, true)) return + timedOut.set(true) + runCatching { opening.close() } + } + }, L2CAP_OPEN_MS) + } try { opening.connect() + if (!settled.compareAndSet(false, true)) throw IOException("the watchdog closed the socket") } catch (error: IOException) { runCatching { opening.close() } - throw TransportConnectException(ConnectFailure.NO_ANSWER, "the glasses did not open L2CAP psm $psm: ${error.message}", error) + val why = if (timedOut.get()) "no answer in ${L2CAP_OPEN_MS / 1000} s" else error.message + throw TransportConnectException(ConnectFailure.NO_ANSWER, "the glasses did not open L2CAP psm $psm: $why", error) + } finally { + watchdog.cancel() } socket = opening stream = SduStream(opening.inputStream, opening.outputStream, wireLog) @@ -117,6 +153,7 @@ class BluetoothLeTransport( private companion object { const val GATT_MS = 6_000L + const val L2CAP_OPEN_MS = 8_000L } }