From 985d7bece39b1d9c228203aa8d2aecdde74e026f Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 14 Aug 2026 15:50:41 -0400 Subject: [PATCH] feat: add Colombia damage assessment dataset and processing scripts --- .../predicted-damage-colombia-2026/README.md | 108 ++++++++++ .../collection/config.json | 65 +++++++ .../collection/description.md | 29 +++ .../collection/template.json | 184 ++++++++++++++++++ .../dataset.yaml | 22 +++ .../predicted_damage_colombia_2026.py | 172 ++++++++++++++++ .../test_predicted_damage_colombia_2026.py | 103 ++++++++++ 7 files changed, 683 insertions(+) create mode 100644 datasets/predicted-damage-colombia-2026/README.md create mode 100644 datasets/predicted-damage-colombia-2026/collection/config.json create mode 100644 datasets/predicted-damage-colombia-2026/collection/description.md create mode 100644 datasets/predicted-damage-colombia-2026/collection/template.json create mode 100644 datasets/predicted-damage-colombia-2026/dataset.yaml create mode 100644 datasets/predicted-damage-colombia-2026/predicted_damage_colombia_2026.py create mode 100644 datasets/predicted-damage-colombia-2026/test_predicted_damage_colombia_2026.py diff --git a/datasets/predicted-damage-colombia-2026/README.md b/datasets/predicted-damage-colombia-2026/README.md new file mode 100644 index 00000000..05930997 --- /dev/null +++ b/datasets/predicted-damage-colombia-2026/README.md @@ -0,0 +1,108 @@ +# Colombia Damage 2026 + +## First-time publishing + +Validate the STAC collection first and fix any validation errors: + +```bash +pctasks dataset validate-collection datasets/predicted-damage-colombia-2026/collection/template.json +``` + +Then submit the collection ingestion: + +```bash +pctasks dataset ingest-collection -d datasets/predicted-damage-colombia-2026/dataset.yaml -s -a registry pccomponents +``` + +Take the workflow ID from the output and watch it. It must succeed. + +```bash +pctasks runs status $WORKFLOW_ID --watch +curl "https://planetarycomputer.microsoft.com/api/stac/v1/collections/predicted-damage-colombia-2026" +``` + +## Updating + +Simply add `-u` to the command. + +```bash +pctasks dataset ingest-collection -d datasets/predicted-damage-colombia-2026/dataset.yaml -u -s -a registry pccomponents +``` + +## Item ingestion + +Assets live under `blob://ai4edataeuwest/ai4good/colombia2026/` and are +organized as `//`. Chunking lists the `*model-predictions.tif` +files, so each folder produces exactly one STAC item. The sibling GeoPackages +and valid-area mask in that folder become its assets. + +Print the workflow without submitting anything: + +```bash +pctasks dataset process-items -d datasets/predicted-damage-colombia-2026/dataset.yaml \ + initial-ingest -a registry pccomponents.azurecr.io +``` + +Try a single item first: + +```bash +pctasks dataset process-items -d datasets/predicted-damage-colombia-2026/dataset.yaml \ + test-ingest -a registry pccomponents.azurecr.io --limit 1 --submit +``` + +Then ingest everything: + +```bash +pctasks dataset process-items -d datasets/predicted-damage-colombia-2026/dataset.yaml \ + initial-ingest -a registry pccomponents.azurecr.io --upsert --submit +``` + +`initial-ingest` and `test-ingest` are chunkset IDs. Use a fresh one whenever +you want to re-list the assets, or pass `-e` to reuse an existing chunkset. + +Watch the run and read logs with: + +```bash +pctasks runs status $RUN_ID --watch +pctasks runs get run-log $RUN_ID +pctasks runs get task-log $RUN_ID create-splits create-splits -p 0 +``` + +Verify the items landed: + +```bash +curl "https://planetarycomputer.microsoft.com/api/stac/v1/collections/predicted-damage-colombia-2026/items?limit=10" +``` + +## Adding a new area + +Upload four files to `colombia2026///`: + +| File | Becomes | +| --- | --- | +| `*model-predictions.tif` | `visual` | +| `*valid_area_mask.geojson` | `valid-area-mask` | +| `*overture*.gpkg` | `overture-buildings` | +| `*google*.gpkg` or `*hdx*.gpkg` | `google-buildings` | + +The folder name sets the item ID, so `pereira/2026-08-12` produces +`pereira-2026-08-12`. Use lowercase, hyphenated area names. Nothing is read +from the file names except the four patterns above. + +Then widen `extent.spatial.bbox` in `collection/template.json` if the new area +falls outside it, re-run the collection ingestion with `-u`, and run +`process-items` with a new chunkset ID. + +## FAQs + +> The workflow failed with "Expected exactly one file matching ...". Why? + +Every item needs all four files in its folder, and each pattern must match +exactly one file. Check for a missing upload or two files matching the same +pattern. + +> Can I reuse a chunkset ID? + +Only with `-e`, which skips re-listing the assets. Newly uploaded areas will +not be picked up that way. + diff --git a/datasets/predicted-damage-colombia-2026/collection/config.json b/datasets/predicted-damage-colombia-2026/collection/config.json new file mode 100644 index 00000000..db59d558 --- /dev/null +++ b/datasets/predicted-damage-colombia-2026/collection/config.json @@ -0,0 +1,65 @@ +{ + "render_config": { + "render_params": { + "tile_format": "png", + "colormap": "{\"2\":[0,255,0,255],\"3\":[255,0,0,255],\"4\":[128,128,128,255]}", + "nodata": 0 + }, + "minzoom": 11, + "assets": [ + "visual" + ], + "maxzoom": 18, + "create_links": true, + "has_mosaic": false, + "mosaic_preview_zoom": 13, + "mosaic_preview_coords": [ + 3.4803, + -76.5367 + ], + "requires_token": true, + "max_items_per_tile": null, + "vector_tilesets": null, + "hidden": false + }, + "mosaic_info": { + "mosaics": [ + { + "name": "Most recent available", + "description": null, + "cql": [] + } + ], + "render_options": [ + { + "name": "Predicted building damage", + "description": "Per-building damage predictions from the Microsoft AI for Good Lab damage assessment model.", + "type": "raster-tile", + "options": "assets=visual&colormap=%7B%222%22%3A%5B0%2C255%2C0%2C255%5D%2C%223%22%3A%5B255%2C0%2C0%2C255%5D%2C%224%22%3A%5B128%2C128%2C128%2C255%5D%7D&nodata=0", + "vector_options": null, + "min_zoom": 11, + "legend": { + "type": "classmap", + "labels": [ + "Building (no damage)", + "Damaged", + "Unknown" + ], + "trim_start": null, + "trim_end": null, + "scale_factor": null + }, + "conditions": null + } + ], + "default_location": { + "zoom": 13, + "coordinates": [ + 3.4803, + -76.5367 + ] + }, + "default_custom_query": null, + "animation_hint": null + } +} \ No newline at end of file diff --git a/datasets/predicted-damage-colombia-2026/collection/description.md b/datasets/predicted-damage-colombia-2026/collection/description.md new file mode 100644 index 00000000..f068ac7f --- /dev/null +++ b/datasets/predicted-damage-colombia-2026/collection/description.md @@ -0,0 +1,29 @@ +Creamos evaluaciones de daños a nivel de edificio tras el terremoto en Colombia +mediante el entrenamiento y la posterior ejecución de un modelo de inteligencia +artificial sobre imágenes satelitales adquiridas después del desastre. El modelo +de IA clasifica cada edificio identificado en las imágenes como "sin daños", +"afectado" o "desconocido". La categoría "desconocido" se utiliza cuando el +edificio no puede ser evaluado adecuadamente, por ejemplo, debido a la presencia +de nubes. Utilizamos los polígonos de edificios de Overture Maps, que representan +el estado de las edificaciones sobre el terreno antes del evento. Los resultados +se distribuyen como un archivo vectorial en formato GeoPackage, con los +siguientes atributos para cada edificio: + +- `id` – identificador único de Overture Maps y Google para cada edificio. +- `damaged` – valor 1 si el edificio está dañado; de lo contrario, 0. +- `unknown` – valor 1 si el edificio está cubierto por nubes, neblina, humo o + si, por alguna otra razón, no fue posible clasificarlo; de lo contrario, 0. +- `area` – área del edificio en metros cuadrados. + +We create building level damage assessments by training and then running an AI +model on the post-disaster imagery. The AI model predicts whether each footprint +in the imagery is "building", "damaged", or "unknown" (i.e. cloudy). We use +Overture Maps and Google building footprints which represent the state on the +ground pre-event and distribute the resulting data as a vector file GeoPackage +with the following per-footprint attributes: + +- `id` – the Overture Maps unique ID for each footprint. +- `damaged` – 1 if the building is damaged, else 0. +- `unknown` – 1 if the building was covered by clouds/haze/smoke or otherwise + unable to be classified, else 0. +- `area` – area of the building in sq meters. diff --git a/datasets/predicted-damage-colombia-2026/collection/template.json b/datasets/predicted-damage-colombia-2026/collection/template.json new file mode 100644 index 00000000..a0a5555e --- /dev/null +++ b/datasets/predicted-damage-colombia-2026/collection/template.json @@ -0,0 +1,184 @@ +{ + "type": "Collection", + "id": "predicted-damage-colombia-2026", + "stac_version": "1.0.0", + "description": "{{ collection.description }}", + "links": [ + { + "rel": "cite-as", + "href": "https://www.microsoft.com/en-us/research/group/ai-for-good-research-lab/", + "type": "text/html", + "title": "AI For Good Lab Harnessing AI to help solve some of the world’s greatest challenges" + }, + { + "rel": "license", + "href": "https://creativecommons.org/licenses/by/4.0/", + "type": "text/html", + "title": "Creative Commons Attribution 4.0 International" + }, + { + "rel": "describedby", + "href": "https://planetarycomputer.microsoft.com/dataset/predicted-damage-colombia-2026", + "type": "text/html", + "title": "Human readable dataset overview and reference" + } + ], + "stac_extensions": [ + "https://stac-extensions.github.io/table/v1.2.0/schema.json", + "https://stac-extensions.github.io/storage/v2.0.0/schema.json" + ], + "item_assets": { + "visual": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "visual" + ], + "title": "Model prediction imagery", + "description": "Cloud-optimized GeoTIFF of the post-event imagery that the damage model was run on." + }, + "overture-buildings": { + "type": "application/geopackage+sqlite3", + "roles": [ + "data" + ], + "title": "Overture Maps building damage footprints", + "description": "GeoPackage of Overture Maps building footprints with predicted earthquake damage.", + "table:storage_options": { + "account_name": "ai4edataeuwest" + } + }, + "google-buildings": { + "type": "application/geopackage+sqlite3", + "roles": [ + "data" + ], + "title": "Google building damage footprints", + "description": "GeoPackage of Google building footprints with predicted earthquake damage.", + "table:storage_options": { + "account_name": "ai4edataeuwest" + } + }, + "valid-area-mask": { + "type": "application/geo+json", + "roles": [ + "metadata" + ], + "title": "Valid area mask", + "description": "GeoJSON mask delineating the area covered by the damage assessment." + } + }, + "msft:region": "westeurope", + "table:columns": [ + { + "name": "geometry", + "type": "byte_array", + "description": "Building footprint polygons" + }, + { + "name": "id", + "type": "string", + "description": "Overture Maps unique ID for each footprint" + }, + { + "name": "damaged", + "type": "int64", + "description": "1 if the building is damaged, else 0" + }, + { + "name": "unknown", + "type": "int64", + "description": "1 if the building was covered by clouds/haze/smoke or otherwise unable to be classified, else 0" + }, + { + "name": "area", + "type": "double", + "description": "Area of the building in square meters", + "unit": "m2" + } + ], + "msft:container": "ai4good", + "msft:storage_account": "ai4edataeuwest", + "msft:short_description": "AI-predicted building damage footprints for areas of Colombia affected by the August 2026 earthquake.", + "title": "Predicted Building Damage: Colombia Earthquake 2026", + "extent": { + "spatial": { + "bbox": [ + [ + -76.6146, + 3.4127, + -75.6922, + 4.8433 + ], + [ + -76.6146, + 3.4127, + -76.4588, + 3.5479 + ], + [ + -75.7856, + 4.7758, + -75.6922, + 4.8433 + ] + ] + }, + "temporal": { + "interval": [ + [ + "2026-08-08T00:00:00Z", + null + ] + ] + } + }, + "license": "CC-BY-4.0", + "keywords": [ + "Colombia", + "Cali", + "Pereira", + "Earthquake", + "Building damage", + "Damage assessment", + "Buildings", + "Microsoft AI for Good Lab" + ], + "providers": [ + { + "name": "Microsoft AI for Good Lab", + "roles": [ + "producer", + "licensor" + ], + "url": "https://www.microsoft.com/en-us/research/group/ai-for-good-research-lab/" + }, + { + "name": "Microsoft", + "roles": [ + "host", + "processor" + ], + "url": "https://planetarycomputer.microsoft.com" + } + ], + "assets": { + "thumbnail": { + "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/colombia2026.png", + "type": "image/png", + "title": "Predicted Building Damage Thumbnail", + "roles": [ + "thumbnail" + ] + } + }, + "storage:schemes": { + "azure": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "ai4edataeuwest", + "container": "ai4good", + "region": "westeurope" + } + } +} \ No newline at end of file diff --git a/datasets/predicted-damage-colombia-2026/dataset.yaml b/datasets/predicted-damage-colombia-2026/dataset.yaml new file mode 100644 index 00000000..45710636 --- /dev/null +++ b/datasets/predicted-damage-colombia-2026/dataset.yaml @@ -0,0 +1,22 @@ +# Keep this id short: it is truncated into Azure Batch job ids, and above ~23 +# characters the job names collide (create-splits vs create-chunks). +id: colombia-damage-2026 +image: ${{ args.registry }}/pctasks-task-base:2026.01.12 + +args: +- registry + +code: + src: ${{ local.path(./predicted_damage_colombia_2026.py) }} + +collections: + - id: predicted-damage-colombia-2026 + template: ${{ local.path(./collection) }} + class: predicted_damage_colombia_2026:PredictedDamageColombia2026 + asset_storage: + - uri: blob://ai4edataeuwest/ai4good/colombia2026 + chunks: + options: + ends_with: "model-predictions.tif" + chunk_storage: + uri: "blob://ai4edataeuwest/ai4good-etl-data/chunks" diff --git a/datasets/predicted-damage-colombia-2026/predicted_damage_colombia_2026.py b/datasets/predicted-damage-colombia-2026/predicted_damage_colombia_2026.py new file mode 100644 index 00000000..7a158727 --- /dev/null +++ b/datasets/predicted-damage-colombia-2026/predicted_damage_colombia_2026.py @@ -0,0 +1,172 @@ +import posixpath +import re +from datetime import datetime, timezone +from typing import Any, Dict, Iterator, List, Tuple, Union + +import pystac + +from pctasks.core.models.task import WaitTaskResult +from pctasks.core.storage import Storage, StorageFactory +from pctasks.dataset.collection import Collection + +# Assets are organized as colombia2026///, one item per +# area/date folder. +GPKG_MEDIA_TYPE = "application/geopackage+sqlite3" +TABLE_EXTENSION = "https://stac-extensions.github.io/table/v1.2.0/schema.json" +STORAGE_OPTIONS = {"account_name": "ai4edataeuwest"} + +MASK_MATCH = "valid_area_mask" + +GPKG_ASSETS: Dict[str, Dict[str, Any]] = { + "overture-buildings": { + "match": ["overture"], + "title": "Overture Maps building damage footprints", + "description": ( + "GeoPackage of Overture Maps building footprints with predicted " + "earthquake damage." + ), + }, + "google-buildings": { + # Cali labels these "hdx"; later areas label them "google". + "match": ["google", "hdx"], + "title": "Google building damage footprints", + "description": ( + "GeoPackage of Google building footprints with predicted earthquake damage." + ), + }, +} + + +def _slug(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") + + +def _bbox_geometry(bbox: List[float]) -> Dict[str, Any]: + xmin, ymin, xmax, ymax = bbox + return { + "type": "Polygon", + "coordinates": [ + [ + [xmin, ymin], + [xmax, ymin], + [xmax, ymax], + [xmin, ymax], + [xmin, ymin], + ] + ], + } + + +def _positions(coordinates: Any) -> Iterator[Tuple[float, float]]: + if coordinates and isinstance(coordinates[0], (int, float)): + yield (coordinates[0], coordinates[1]) + else: + for part in coordinates: + yield from _positions(part) + + +def _mask_extent( + storage: Storage, mask_path: str +) -> Tuple[Dict[str, Any], List[float]]: + mask = storage.read_json(mask_path) + geometries = [f.get("geometry", f) for f in mask.get("features", [mask])] + + positions = [p for g in geometries for p in _positions(g["coordinates"])] + xs = [p[0] for p in positions] + ys = [p[1] for p in positions] + bbox = [min(xs), min(ys), max(xs), max(ys)] + + if len(geometries) == 1: + return geometries[0], bbox + return _bbox_geometry(bbox), bbox + + +def _find_one(paths: List[str], matches: List[str], folder: str) -> str: + found = [ + p for p in paths if any(m in posixpath.basename(p).lower() for m in matches) + ] + if len(found) != 1: + raise ValueError( + f"Expected exactly one file matching {matches} in {folder}, found {found}" + ) + return found[0] + + +class PredictedDamageColombia2026(Collection): + @classmethod + def create_item( + cls, asset_uri: str, storage_factory: StorageFactory + ) -> Union[List[pystac.Item], WaitTaskResult]: + storage, tif_path = storage_factory.get_storage_for_file(asset_uri) + + folder = posixpath.dirname(tif_path) + date_str = posixpath.basename(folder) + area = _slug(posixpath.basename(posixpath.dirname(folder))) + try: + item_datetime = datetime.strptime(date_str, "%Y-%m-%d").replace( + tzinfo=timezone.utc + ) + except ValueError: + raise ValueError( + f"Expected an / path for asset {asset_uri}" + ) + if not area: + raise ValueError(f"Expected an / path for {asset_uri}") + + sibling_paths = list(storage.list_files(name_starts_with=f"{folder}/")) + gpkg_paths = [p for p in sibling_paths if p.lower().endswith(".gpkg")] + + mask_path = _find_one(sibling_paths, [MASK_MATCH], folder) + geometry, bbox = _mask_extent(storage, mask_path) + + item = pystac.Item( + id=f"{area}-{date_str}", + geometry=geometry, + bbox=bbox, + datetime=item_datetime, + properties={}, + stac_extensions=[TABLE_EXTENSION], + ) + + item.add_asset( + "visual", + pystac.Asset( + href=storage.get_url(tif_path), + media_type=pystac.MediaType.COG, + title="Model prediction imagery", + description=( + "Cloud-optimized GeoTIFF of the post-event imagery that the " + "damage model was run on." + ), + roles=["data", "visual"], + ), + ) + item.add_asset( + "valid-area-mask", + pystac.Asset( + href=storage.get_url(mask_path), + media_type=pystac.MediaType.GEOJSON, + title="Valid area mask", + description=( + "GeoJSON mask delineating the area covered by the damage " + "assessment." + ), + roles=["metadata"], + ), + ) + + for key, info in GPKG_ASSETS.items(): + gpkg_path = _find_one(gpkg_paths, info["match"], folder) + item.add_asset( + key, + pystac.Asset( + href=storage.get_url(gpkg_path), + media_type=GPKG_MEDIA_TYPE, + title=info["title"], + description=info["description"], + roles=["data"], + extra_fields={"table:storage_options": STORAGE_OPTIONS}, + ), + ) + + return [item] diff --git a/datasets/predicted-damage-colombia-2026/test_predicted_damage_colombia_2026.py b/datasets/predicted-damage-colombia-2026/test_predicted_damage_colombia_2026.py new file mode 100644 index 00000000..f9ce367f --- /dev/null +++ b/datasets/predicted-damage-colombia-2026/test_predicted_damage_colombia_2026.py @@ -0,0 +1,103 @@ +from datetime import datetime, timezone + +import pytest +from predicted_damage_colombia_2026 import PredictedDamageColombia2026 + +from pctasks.core.storage import StorageFactory + +CALI = ( + "blob://ai4edataeuwest/ai4good/colombia2026/cali/2026-08-10/" + "airbus_cali_warped_cog_model-predictions.tif" +) +PEREIRA = ( + "blob://ai4edataeuwest/ai4good/colombia2026/pereira/2026-08-12/" + "vantor_8-12_pereira_model-predictions.tif" +) + +ASSET_KEYS = {"visual", "valid-area-mask", "overture-buildings", "google-buildings"} + + +@pytest.mark.parametrize( + "href,expected_id,expected_datetime,expected_bbox", + [ + ( + CALI, + "cali-2026-08-10", + datetime(2026, 8, 10, tzinfo=timezone.utc), + [-76.6146, 3.4127, -76.4588, 3.5479], + ), + ( + PEREIRA, + "pereira-2026-08-12", + datetime(2026, 8, 12, tzinfo=timezone.utc), + [-75.7856, 4.7758, -75.6922, 4.8433], + ), + ], +) +def test_create_item( + href: str, + expected_id: str, + expected_datetime: datetime, + expected_bbox: list, +) -> None: + (item,) = PredictedDamageColombia2026.create_item(href, StorageFactory()) + item.validate() + + assert item.id == expected_id + assert item.datetime == expected_datetime + assert set(item.assets) == ASSET_KEYS + assert item.bbox == pytest.approx(expected_bbox, abs=1e-4) + assert item.geometry["type"] == "Polygon" + + +@pytest.mark.parametrize("href", [CALI, PEREIRA]) +def test_asset_media_types(href: str) -> None: + (item,) = PredictedDamageColombia2026.create_item(href, StorageFactory()) + + assert item.assets["visual"].media_type == ( + "image/tiff; application=geotiff; profile=cloud-optimized" + ) + assert item.assets["valid-area-mask"].media_type == "application/geo+json" + for key in ["overture-buildings", "google-buildings"]: + asset = item.assets[key] + assert asset.media_type == "application/geopackage+sqlite3" + assert asset.extra_fields["table:storage_options"] == { + "account_name": "ai4edataeuwest" + } + + +@pytest.mark.parametrize( + "href,expected_google_file", + [ + ( + CALI, + "airbus_8-10_cali_hdx_building_footprints_with_predictions_validated.gpkg", + ), + (PEREIRA, "vantor_8-12_pereira_google_buildings_with_predictions.gpkg"), + ], +) +def test_google_buildings_naming_varies_by_area( + href: str, expected_google_file: str +) -> None: + (item,) = PredictedDamageColombia2026.create_item(href, StorageFactory()) + + assert item.assets["google-buildings"].href.endswith(expected_google_file) + assert "overture" in item.assets["overture-buildings"].href + + +def test_missing_assets_raises() -> None: + href = ( + "blob://ai4edataeuwest/ai4good/colombia2026/nowhere/2026-08-12/" + "x_model-predictions.tif" + ) + with pytest.raises(ValueError, match="Expected exactly one file matching"): + PredictedDamageColombia2026.create_item(href, StorageFactory()) + + +def test_non_date_folder_raises() -> None: + href = ( + "blob://ai4edataeuwest/ai4good/colombia2026/cali/not-a-date/" + "x_model-predictions.tif" + ) + with pytest.raises(ValueError, match="Expected an / path"): + PredictedDamageColombia2026.create_item(href, StorageFactory())