K2GO-394 feat(maps): resilient base-map download via dash-node

.github/workflows/publish-catalogs.yml

@@ -38,6 +38,7 @@ jobs:
           python3 tools/build_kolibri_catalog.py || echo "::warning::kolibri generator failed; its upload will be skipped"
           python3 tools/build_kolibri_catalog.py --tree topics && gzip -9 -n -f src/main/assets/kolibri_tree.jsonl || echo "::warning::kolibri tree generator failed; its upload will be skipped"
           python3 tools/build_kiwix_catalog.py   || echo "::warning::kiwix generator failed; its upload will be skipped"
+          python3 tools/build_maps_catalog.py    || echo "::warning::maps generator failed; its upload will be skipped"
 
       - name: Build manifests and upload to Cloudflare R2
         env:
@@ -86,3 +87,4 @@ jobs:
           publish kolibri      src/main/assets/kolibri_catalog.jsonl "application/x-ndjson" jsonl
           publish kolibri-tree src/main/assets/kolibri_tree.jsonl.gz "application/gzip"     none
           publish kiwix        src/main/assets/kiwix_catalog.csv     "text/csv"             none
+          publish maps         src/main/assets/maps_catalog.csv      "text/csv"             none

controller/app/build.gradle

@@ -637,91 +637,33 @@ task refreshModuleSizes {
 
 preBuild.dependsOn refreshModuleSizes
 
-// ---- maps layer size auto-capture (offline fallback) --------------------------
-// Fetches the <size> from each map layer's .meta4 on the maps mirror and writes
-// src/main/assets/maps_sizes.csv, so the Choose screen's OFFLINE fallback (MapsCatalog)
-// is captured at package time instead of hand-maintained — same pattern as rootfs above.
-// The file names follow roles/maps/defaults/main.yml; update the three data dates below
-// when the role bumps them. Non-fatal: an incomplete fetch keeps the committed CSV.
-task refreshMapsSizes {
+// ---- maps catalog refresh (release / manual) ----------------------------------
+// Regenerates src/main/assets/maps_catalog.csv (group,level,file,bytes,date) from the
+// maps mirror's .meta4 pointers via tools/build_maps_catalog.py -- the ONE builder for
+// this catalog (it replaced the old inline refreshMapsSizes fetch; no double build path).
+// Like the Kiwix catalog it does NOT run on every build: it runs on a RELEASE and on
+// demand (./gradlew refreshMapsCatalog), and the weekly publish-catalogs workflow runs
+// the same script to publish the CSV + manifest to Cloudflare. The committed CSV is the
+// offline floor -- MapsCatalog reads it for the Choose size AND the download file id.
+// Never fails the build: if python3 is missing or the fetch is blocked, the committed
+// CSV (last known) is kept.
+task refreshMapsCatalog {
     group = 'build setup'
-    description = 'Capture map layer sizes (.meta4) into assets/maps_sizes.csv for the offline fallback'
-    def csv = file('src/main/assets/maps_sizes.csv')
+    description = 'Regenerate assets/maps_catalog.csv from the maps mirror .meta4 (release / manual)'
     doLast {
-        def base = 'https://iiab.switnet.org/maps/2/'
-        // Data dates from roles/maps/defaults/main.yml (maps_vector_data_date / maps_slow_data_date
-        // / maps_search_data_date). Keep in sync with the role.
-        def vectorDate = '2026-07-01'
-        def slowDate = '2025-12-10'
-        def searchDate = '2026-04-22'
-        // "group|level" -> mirror file name (whole-world pmtiles; search is a tarball).
-        def files = [
-                'base|nat-z8'          : "naturalearth-openmaptiles.${slowDate}.z00-z08.pmtiles",
-                'base|11'              : "openstreetmap-openmaptiles.${vectorDate}.z00-z11.pmtiles",
-                'base|14'              : "openstreetmap-openmaptiles.${vectorDate}.z00-z14.pmtiles",
-                'satellite|7'          : "s2maps-sentinel2-2023.${slowDate}.z00-z07.pmtiles",
-                'satellite|9'          : "s2maps-sentinel2-2023.${slowDate}.z00-z09.pmtiles",
-                'satellite|11'         : "s2maps-sentinel2-2023.${slowDate}.z00-z11.pmtiles",
-                'satellite|13'         : "s2maps-sentinel2-2023.${slowDate}.z00-z13.pmtiles",
-                'terrain|7'            : "terrarium.${slowDate}.z00-z07.pmtiles",
-                'terrain|8'            : "terrarium.${slowDate}.z00-z08.pmtiles",
-                'terrain|9'            : "terrarium.${slowDate}.z00-z09.pmtiles",
-                'terrain|10'           : "terrarium.${slowDate}.z00-z10.pmtiles",
-                'search|pop-1k-cities' : "static-search.${searchDate}.pop-1k-cities.tar.gz",
-        ]
-        def sizePat = ~/<size>(\d+)<\/size>/
-        def fetched = [:]
-        files.each { k, fname ->
-            def url = "${base}${fname}.meta4"
-            try {
-                def conn = (HttpURLConnection) new URL(url).openConnection()
-                conn.setRequestProperty('User-Agent', 'iiab-android-build')
-                conn.connectTimeout = 6000
-                conn.readTimeout = 6000
-                if (conn.responseCode != 200) {
-                    logger.warn(">> [maps sizes] HTTP ${conn.responseCode} for ${url}")
-                    conn.disconnect(); return
-                }
-                def body = conn.inputStream.getText('UTF-8')
-                conn.disconnect()
-                def m = sizePat.matcher(body)
-                if (m.find()) { fetched[k] = m.group(1) }
-                else { logger.warn(">> [maps sizes] no <size> element in ${url}") }
-            } catch (Exception e) {
-                logger.warn(">> [maps sizes] fetch failed for ${url}: ${e.message}")
-            }
-        }
-        if (fetched.size() != files.size()) {
-            logger.warn(">> [maps sizes] incomplete fetch (${fetched.size()}/${files.size()}); keeping existing ${csv.name} (last known).")
-            return
-        }
-        def existing = [:]
-        if (csv.exists()) {
-            csv.eachLine { line ->
-                line = line.trim()
-                if (line.isEmpty() || line.startsWith('#')) return
-                def p = line.split(',')
-                if (p.length >= 3) existing["${p[0].trim()}|${p[1].trim()}".toString()] = p[2].trim()
-            }
-        }
-        if (existing == fetched) {
-            println ">> [maps sizes] up to date (${fetched.size()} entries unchanged)."
-            return
-        }
-        def today = new Date().format('yyyy-MM-dd')
-        def rows = files.keySet().collect { k ->
-            def gl = k.split('\\|', 2)
-            "${gl[0]},${gl[1]},${fetched[k]},${today}".toString()
+        try {
+            def proc = ['python3', 'tools/build_maps_catalog.py'].execute(null, projectDir)
+            proc.consumeProcessOutput(System.out, System.err)
+            proc.waitFor()
+            if (proc.exitValue() != 0) logger.warn(">> [maps catalog] generator exit ${proc.exitValue()}; kept existing CSV.")
+        } catch (Exception e) {
+            logger.warn(">> [maps catalog] skipped (${e.message}); kept existing CSV.")
         }
-        if (!csv.parentFile.exists()) csv.parentFile.mkdirs()
-        csv.text = '# group,level,bytes,fetched_date -- AUTO-GENERATED by the refreshMapsSizes Gradle task; do not edit by hand\n' +
-                '# Whole-world pmtiles from https://iiab.switnet.org/maps/2 (roles/maps/defaults dates).\n' +
-                rows.join('\n') + '\n'
-        println ">> [maps sizes] updated ${rows.size()} entries in ${csv.name} (${today})."
     }
 }
-
-preBuild.dependsOn refreshMapsSizes
+tasks.matching { it.name == 'assembleRelease' || it.name == 'bundleRelease' }.configureEach {
+    dependsOn refreshMapsCatalog
+}
 
 // ---- Kiwix ZIM catalog refresh (release-only) ---------------------------------
 // Regenerates src/main/assets/kiwix_catalog.csv from the Kiwix directory listings via

controller/app/src/main/assets/maps_catalog.csv

@@ -0,0 +1,14 @@
+# group,level,file,bytes,fetched_date -- AUTO-GENERATED by tools/build_maps_catalog.py; do not edit by hand
+# Whole-world pmtiles from https://iiab.switnet.org/maps/2 (roles/maps/defaults dates). The app sends 'file' as the download id; the box composes the URL.
+base,nat-z8,naturalearth-openmaptiles.2025-12-10.z00-z08.pmtiles,89435511,2026-09-07
+base,11,openstreetmap-openmaptiles.2026-07-01.z00-z11.pmtiles,9010265257,2026-09-07
+base,14,openstreetmap-openmaptiles.2026-07-01.z00-z14.pmtiles,87411164966,2026-09-07
+satellite,7,s2maps-sentinel2-2023.2025-12-10.z00-z07.pmtiles,88772638,2026-09-07
+satellite,9,s2maps-sentinel2-2023.2025-12-10.z00-z09.pmtiles,1247283362,2026-09-07
+satellite,11,s2maps-sentinel2-2023.2025-12-10.z00-z11.pmtiles,22312696177,2026-09-07
+satellite,13,s2maps-sentinel2-2023.2025-12-10.z00-z13.pmtiles,290318099827,2026-09-07
+terrain,7,terrarium.2025-12-10.z00-z07.pmtiles,978219778,2026-09-07
+terrain,8,terrarium.2025-12-10.z00-z08.pmtiles,6427331113,2026-09-07
+terrain,9,terrarium.2025-12-10.z00-z09.pmtiles,28621213143,2026-09-07
+terrain,10,terrarium.2025-12-10.z00-z10.pmtiles,106207946492,2026-09-07
+search,pop-1k-cities,static-search.2026-04-22.pop-1k-cities.tar.gz,16111581,2026-09-07

controller/app/src/main/java/org/appdevforall/k2go/install/domain/MapsRunroleCommand.java

@@ -14,6 +14,10 @@
  *               RUNTIME from the completion marker in iiab_state.yml -- --reinstall over a completed or
  *               base-seeded install, plain runrole to recover a half-done one (a bare --reinstall errors
  *               when the marker was already deleted by a prior failed --reinstall run).
+ *               K2GO-394: base-map pmtiles are downloaded by dash-node (the durable job engine)
+ *               BEFORE this runs, so the role's is_proot download task only asserts the files are
+ *               present and skips via creates:. This command carries no download handshake -- it is
+ *               the plain post-processing runrole.
  * ============================================================================
  */
 package org.appdevforall.k2go.install.domain;