From 7e3f4da229ba92b0a51bee8552435e62326bdffb Mon Sep 17 00:00:00 2001 From: hrodmn Date: Fri, 26 Jun 2026 06:17:44 -0500 Subject: [PATCH 1/4] feat: update to handle non-tif files - add CI and tests resolves #1 --- .github/workflows/ci.yml | 64 +++++++++++ README.md | 28 +++-- algorithm-config.yml | 14 ++- main.py | 86 +-------------- pyproject.toml | 22 +++- run-named.sh | 34 +++++- run.sh | 13 ++- src/dps_stac_item_generator/__init__.py | 5 + src/dps_stac_item_generator/cli.py | 64 +++++++++++ src/dps_stac_item_generator/generator.py | 135 +++++++++++++++++++++++ tests/test_generator.py | 95 ++++++++++++++++ tests/test_stac_item_outputs.py | 69 ++++++++++++ uv.lock | 42 ++++++- 13 files changed, 562 insertions(+), 109 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 src/dps_stac_item_generator/__init__.py create mode 100644 src/dps_stac_item_generator/cli.py create mode 100644 src/dps_stac_item_generator/generator.py create mode 100644 tests/test_generator.py create mode 100644 tests/test_stac_item_outputs.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f654ec0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,64 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: {} + +jobs: + lint-python: + name: Lint and typecheck + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.11.*" + enable-cache: false + + - name: Set up Python + run: uv python install 3.14 + + - name: Run ruff + run: uv run ruff check . + + - name: Run mypy + run: uv run mypy src tests + + test: + name: Test Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + matrix: + python-version: ["3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.11.*" + enable-cache: false + + - name: Set up Python + run: uv python install ${{ matrix.python-version }} + + - name: Run pre-commit + if: matrix.python-version == '3.14' + run: uv run pre-commit run --all-files + + - name: Run tests + run: uv run pytest diff --git a/README.md b/README.md index f7d41ae..e6cd172 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@ -# DPS Import COGs +# DPS STAC Item Generator -Create STAC items for raster files in object storage +Create STAC items for selected files in object storage. ## About -This DPS algorithm creates STAC metadata for existing raster files (GeoTIFFs) stored in object storage (S3, Azure, GCS, etc.). The workflow: +This DPS algorithm creates STAC metadata for existing files stored in object storage (S3, Azure, GCS, etc.). The workflow: 1. Lists all files at the specified source location -2. Filters for `.tif` files -3. Creates a STAC item for each raster using `rio-stac` +2. Filters files by configurable include/exclude extension lists +3. Creates a STAC item for each matching asset using `rio-stac` 4. Exports a self-contained STAC catalog with all items -This tool is useful for importing existing COG (Cloud Optimized GeoTIFF) datasets into STAC format for better discoverability and interoperability. +By default, the algorithm includes GeoTIFF (`.tif`, `.tiff`) and NetCDF (`.nc`) files. This tool is useful for importing existing geospatial datasets into STAC format for better discoverability and interoperability. The STAC items will be uploaded to the DPS User STAC in a collection associated with your username, the algorithm name/version, and the job tag. ## Usage @@ -27,7 +27,7 @@ from maap.maap import MAAP maap = MAAP(maap_host="api.maap-project.org") job = maap.submitJob( - algo_id="GenerateCogStacItems", + algo_id="GenerateStacItems", version="v0.1", identifier="test-run", queue="maap-dps-worker-8gb", @@ -36,9 +36,9 @@ job = maap.submitJob( ``` -Each job will produce a STAC item for each .tif file that exists under the provided `source`. The STAC items will be uploaded to the DPS User STAC catalog automatically after job completion. All jobs associated with the same algorithm, version, username, and job tag/identifier will be added to the same collection with the following format: +Each job will produce a STAC item for each file under the provided `source` whose extension matches the configured filters. The STAC items will be uploaded to the DPS User STAC catalog automatically after job completion. All jobs associated with the same algorithm, version, username, and job tag/identifier will be added to the same collection with the following format: -`{username}__GenerateCogStacItems__v0.1__{identifier}` +`{username}__GenerateStacItems__v0.1__{identifier}` You can access the items following this pattern: @@ -62,7 +62,9 @@ To customize the visualization, you can add all of the familiar visualization pa uv run main.py \ --source "s3://bucket/path/to/files/" \ --output_dir "/tmp/output" \ - --region "us-west-2" + --region "us-west-2" \ + --include-extensions ".tif,.tiff,.nc" \ + --exclude-extensions "" ``` ### Parameters @@ -70,11 +72,13 @@ uv run main.py \ - `--source`: Source location of the files for which you want to generate STAC items (e.g., `s3://bucket/path/to/files/`) - `--output_dir`: Directory where the STAC catalog will be saved - `--region`: AWS region where the storage container exists (default: `us-west-2`) +- `--include-extensions`: Comma-separated extensions to include (default: `.tif,.tiff,.nc`). Use an empty string to include all files. +- `--exclude-extensions`: Comma-separated extensions to exclude. Exclusions override inclusions. ## Output The tool generates a self-contained STAC catalog in the output directory containing: - A `catalog.json` file with metadata about the collection -- Individual STAC item JSON files for each raster -- Each STAC item includes projection (`proj`) and raster band information +- Individual STAC item JSON files for each matching asset +- Each STAC item includes projection (`proj`) and raster band information when `rio-stac` can derive it from the source asset diff --git a/algorithm-config.yml b/algorithm-config.yml index 587c5a1..85bef3c 100644 --- a/algorithm-config.yml +++ b/algorithm-config.yml @@ -1,5 +1,5 @@ -algorithm_description: Generate STAC item metadata for COGs in cloud storage -algorithm_name: GenerateCogStacItems +algorithm_description: Generate STAC item metadata for selected files in cloud storage +algorithm_name: GenerateStacItems algorithm_version: v0.1 build_command: dps-import-cogs/build-env.sh disk_space: 8GB @@ -9,13 +9,21 @@ inputs: file: [] positional: - default: '' - description: ' source location for the COGs in object storage, e.g. s3://bucket/path/to/folder/' + description: ' source location for the files in object storage, e.g. s3://bucket/path/to/folder/' name: source required: true - default: '' description: ' region in which the storage container exists' name: region required: false + - default: '.tif,.tiff,.nc' + description: ' comma-separated file extensions to include, e.g. .tif,.tiff,.nc. Leave empty to include all files.' + name: include_extensions + required: false + - default: '' + description: ' comma-separated file extensions to exclude. Exclusions override inclusions.' + name: exclude_extensions + required: false queue: maap-dps-worker-8gb repository_url: https://github.com/MAAP-Project/dps-import-cogs run_command: dps-import-cogs/run.sh diff --git a/main.py b/main.py index 16d8442..79c31d9 100644 --- a/main.py +++ b/main.py @@ -1,87 +1,7 @@ -"""Create STAC items for all raster files in object storage that match a prefix""" +"""Compatibility entry point for the DPS STAC item generator.""" -import argparse -import logging -import os -from pathlib import Path - -import obstore -from obstore.store import from_url -from pystac import Catalog, CatalogType, MediaType -from rio_cogeo.cogeo import cog_validate -from rio_stac import create_stac_item - -logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s" -) -logging.getLogger("botocore").setLevel(logging.WARNING) -logger = logging.getLogger(__name__) - -DEFAULT_REGION = "us-west-2" - - -def run( - source: str, - output_dir: Path, - region: str | None = None, -): - source_store = from_url(source, region=region or DEFAULT_REGION) - - catalog = Catalog( - id="DPS", - description="DPS", - catalog_type=CatalogType.SELF_CONTAINED, - ) - - stream = obstore.list(source_store) - for batch in stream: - for obj in batch: - obj_key = os.path.join(source, obj["path"]) - if not obj_key.endswith(".tif"): - logging.info(f"skipping {obj_key} because it is not a .tif") - continue - - logging.info(f"processing {obj_key}") - is_cog = cog_validate(obj_key) - - item = create_stac_item( - source=obj_key, - id=obj["path"].replace(".tif", ""), - with_proj=True, - with_raster=True, - asset_media_type=MediaType.COG if is_cog else MediaType.GEOTIFF, - ) - - catalog.add_item(item) - - catalog.normalize_and_save( - root_href=str(output_dir), - catalog_type=CatalogType.SELF_CONTAINED, - ) +from dps_stac_item_generator.cli import main if __name__ == "__main__": - parse = argparse.ArgumentParser( - description="Queries the HLS STAC geoparquet archive and writes the result to a file" - ) - parse.add_argument( - "--source", - help="Source location of the files for which you want to generate STAC items." - " e.g. 's3://bucket/path/to/files/'", - required=True, - ) - parse.add_argument( - "--output_dir", help="Directory in which to save output", required=True - ) - parse.add_argument( - "--region", - help="region in which the storage container exists. e.g. 'us-west-2'", - default="us-west-2", - ) - args = parse.parse_args() - - output_dir = Path(args.output_dir) - run( - source=args.source, - output_dir=output_dir, - ) + main() diff --git a/pyproject.toml b/pyproject.toml index 552a0d4..6307806 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,10 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + [project] -name = "dps-import-cogs" -description = "DPS algorithm for generating STAC item metadata for COGs from external sources" +name = "dps-stac-item-generator" +description = "DPS algorithm for generating STAC item metadata for selected files from external sources" readme = "README.md" requires-python = ">=3.12" dynamic = ["version"] @@ -12,18 +16,32 @@ dependencies = [ "rio-stac>=0.10.1", ] +[project.scripts] +dps-stac-item-generator = "dps_stac_item_generator.cli:main" + [dependency-groups] dev = [ "ipython>=9.8.0", "maap-py>=4.2.0", "mypy>=1.18.2", "pre-commit>=4.3.0", + "pytest>=9.1.1", "ruff>=0.14.3", ] [tool.pdm.version] source = "scm" +[tool.setuptools.packages.find] +where = ["src"] + +[tool.mypy] +ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-vv" + [tool.ruff] [tool.ruff.format] diff --git a/run-named.sh b/run-named.sh index a0b281e..0f6df73 100755 --- a/run-named.sh +++ b/run-named.sh @@ -8,6 +8,10 @@ basedir=$(dirname "$(readlink -f "$0")") # Initialize variables source="" region="" +include_extensions="" +exclude_extensions="" +include_extensions_set=false +exclude_extensions_set=false # Parse named arguments while [[ $# -gt 0 ]]; do @@ -20,9 +24,19 @@ while [[ $# -gt 0 ]]; do region="$2" shift 2 ;; + --include-extensions) + include_extensions="$2" + include_extensions_set=true + shift 2 + ;; + --exclude-extensions) + exclude_extensions="$2" + exclude_extensions_set=true + shift 2 + ;; *) echo "Error: Unknown argument '$1'" - echo "Usage: $0 --source [--region ]" + echo "Usage: $0 --source [--region ] [--include-extensions ] [--exclude-extensions ]" exit 1 ;; esac @@ -36,8 +50,18 @@ fi # Call run.sh with positional arguments -if [[ -n "$region" ]]; then - "${basedir}/run.sh" "$source" "$region" -else - "${basedir}/run.sh" "$source" +args=("$source") + +if [[ -n "$region" || "$include_extensions_set" == true || "$exclude_extensions_set" == true ]]; then + args+=("$region") fi + +if [[ "$include_extensions_set" == true || "$exclude_extensions_set" == true ]]; then + args+=("$include_extensions") +fi + +if [[ "$exclude_extensions_set" == true ]]; then + args+=("$exclude_extensions") +fi + +"${basedir}/run.sh" "${args[@]}" diff --git a/run.sh b/run.sh index 7d63076..5ece1a3 100755 --- a/run.sh +++ b/run.sh @@ -25,6 +25,8 @@ OUTPUT_DIR=output source="$1" region="${2:-}" +include_extensions="${3:-}" +exclude_extensions="${4:-}" # Call the script using the absolute paths # Use the updated environment when calling 'uv run' @@ -41,9 +43,18 @@ args=( --output_dir="${OUTPUT_DIR}" ) -# Only add --region if it was provided +# Only add optional parameters if provided. Extension arguments may be +# intentionally empty, so check the number of positional arguments for those. if [ -n "$region" ]; then args+=(--region="${region}") fi +if [ "$#" -ge 3 ]; then + args+=(--include-extensions="${include_extensions}") +fi + +if [ "$#" -ge 4 ]; then + args+=(--exclude-extensions="${exclude_extensions}") +fi + UV_PROJECT=${basedir} uv run --no-dev ${basedir}/main.py "${args[@]}" diff --git a/src/dps_stac_item_generator/__init__.py b/src/dps_stac_item_generator/__init__.py new file mode 100644 index 0000000..01d90d6 --- /dev/null +++ b/src/dps_stac_item_generator/__init__.py @@ -0,0 +1,5 @@ +"""Generate STAC items for DPS-accessible object storage assets.""" + +from dps_stac_item_generator.generator import run + +__all__ = ["run"] diff --git a/src/dps_stac_item_generator/cli.py b/src/dps_stac_item_generator/cli.py new file mode 100644 index 0000000..036f633 --- /dev/null +++ b/src/dps_stac_item_generator/cli.py @@ -0,0 +1,64 @@ +"""Command-line interface for the DPS STAC item generator.""" + +import argparse +import logging +from pathlib import Path + +from dps_stac_item_generator.generator import DEFAULT_INCLUDE_EXTENSIONS, run + + +def configure_logging() -> None: + """Configure default process logging.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + ) + logging.getLogger("botocore").setLevel(logging.WARNING) + + +def main() -> None: + """Parse CLI arguments and generate STAC items.""" + configure_logging() + + parser = argparse.ArgumentParser( + description="Generate STAC item metadata for matching files in object storage." + ) + parser.add_argument( + "--source", + help="Source location of the files for which you want to generate STAC items." + " e.g. 's3://bucket/path/to/files/'", + required=True, + ) + parser.add_argument( + "--output_dir", + help="Directory in which to save output", + required=True, + ) + parser.add_argument( + "--region", + help="Region in which the storage container exists. e.g. 'us-west-2'", + default="us-west-2", + ) + parser.add_argument( + "--include-extensions", + default=",".join(DEFAULT_INCLUDE_EXTENSIONS), + help="Comma-separated list of file extensions to include. Use an empty string to include all files.", + ) + parser.add_argument( + "--exclude-extensions", + default="", + help="Comma-separated list of file extensions to exclude. Exclusions override inclusions.", + ) + args = parser.parse_args() + + run( + source=args.source, + output_dir=Path(args.output_dir), + region=args.region, + include_extensions=args.include_extensions, + exclude_extensions=args.exclude_extensions, + ) + + +if __name__ == "__main__": + main() diff --git a/src/dps_stac_item_generator/generator.py b/src/dps_stac_item_generator/generator.py new file mode 100644 index 0000000..05914b8 --- /dev/null +++ b/src/dps_stac_item_generator/generator.py @@ -0,0 +1,135 @@ +"""Generate STAC items for files in object storage.""" + +import logging +import os +from pathlib import Path +from typing import Iterable + +import obstore +from obstore.store import from_url +from pystac import Catalog, CatalogType, MediaType +from rio_cogeo.cogeo import cog_validate +from rio_stac import create_stac_item + +logger = logging.getLogger(__name__) + +DEFAULT_REGION = "us-west-2" +DEFAULT_INCLUDE_EXTENSIONS = (".tif", ".tiff", ".nc") +_EXTENSION_MEDIA_TYPES = { + ".tif": MediaType.GEOTIFF, + ".tiff": MediaType.GEOTIFF, + ".nc": MediaType.NETCDF, +} + + +def normalize_extensions(extensions: str | Iterable[str] | None) -> tuple[str, ...]: + """Normalize file extensions to lowercase dotted suffixes.""" + if extensions is None: + return () + + raw_extensions: Iterable[str] + if isinstance(extensions, str): + raw_extensions = extensions.split(",") + else: + raw_extensions = extensions + + normalized = [] + for extension in raw_extensions: + extension = extension.strip().lower() + if not extension: + continue + if not extension.startswith("."): + extension = f".{extension}" + normalized.append(extension) + + return tuple(dict.fromkeys(normalized)) + + +def should_include_file( + path: str, + include_extensions: Iterable[str], + exclude_extensions: Iterable[str] = (), +) -> bool: + """Return whether a file path passes extension include/exclude filters.""" + suffix = Path(path).suffix.lower() + exclude_extensions = tuple(exclude_extensions) + if suffix in exclude_extensions: + return False + + include_extensions = tuple(include_extensions) + return not include_extensions or suffix in include_extensions + + +def media_type_for_extension( + extension: str, *, is_cog: bool = False +) -> MediaType | str: + """Return the STAC asset media type for a normalized file extension.""" + if extension in {".tif", ".tiff"} and is_cog: + return MediaType.COG + + return _EXTENSION_MEDIA_TYPES.get(extension, "application/octet-stream") + + +def is_cloud_optimized_geotiff(source: str) -> bool: + """Return whether a GeoTIFF source is a valid Cloud Optimized GeoTIFF.""" + result = cog_validate(source) + if isinstance(result, tuple): + return bool(result[0]) + return bool(result) + + +def create_item_id(path: str) -> str: + """Create a STAC item ID from an object path by removing one file suffix.""" + return str(Path(path).with_suffix("")) + + +def run( + source: str, + output_dir: Path, + region: str | None = None, + include_extensions: str | Iterable[str] | None = DEFAULT_INCLUDE_EXTENSIONS, + exclude_extensions: str | Iterable[str] | None = None, +) -> None: + """Generate a self-contained STAC catalog for matching files under a source URL.""" + normalized_include_extensions = normalize_extensions(include_extensions) + normalized_exclude_extensions = normalize_extensions(exclude_extensions) + source_store = from_url(source, region=region or DEFAULT_REGION) + + catalog = Catalog( + id="DPS", + description="DPS", + catalog_type=CatalogType.SELF_CONTAINED, + ) + + stream = obstore.list(source_store) + for batch in stream: + for obj in batch: + obj_key = os.path.join(source, obj["path"]) + if not should_include_file( + obj_key, + normalized_include_extensions, + normalized_exclude_extensions, + ): + logger.info( + "skipping %s because it did not match file filters", obj_key + ) + continue + + logger.info("processing %s", obj_key) + suffix = Path(obj_key).suffix.lower() + is_cog = suffix in {".tif", ".tiff"} and is_cloud_optimized_geotiff(obj_key) + + item = create_stac_item( + source=obj_key, + id=create_item_id(obj["path"]), + with_proj=True, + with_raster=True, + asset_media_type=media_type_for_extension(suffix, is_cog=is_cog), + ) + + catalog.add_item(item) + + catalog.normalize_and_save( + root_href=str(output_dir), + catalog_type=CatalogType.SELF_CONTAINED, + ) diff --git a/tests/test_generator.py b/tests/test_generator.py new file mode 100644 index 0000000..08048f0 --- /dev/null +++ b/tests/test_generator.py @@ -0,0 +1,95 @@ +"""Tests for DPS STAC item generation.""" + +from datetime import UTC, datetime +from pathlib import Path + +from pystac import Item, MediaType + +from dps_stac_item_generator import generator + + +def test_normalize_extensions_accepts_strings_and_iterables() -> None: + """Extension filters are normalized for consistent matching.""" + assert generator.normalize_extensions("tif, .NC, ,Tiff") == (".tif", ".nc", ".tiff") + assert generator.normalize_extensions(["nc", ".tif", "NC"]) == (".nc", ".tif") + + +def test_should_include_file_applies_exclusions_after_inclusions() -> None: + """Exclude filters win over include filters.""" + assert generator.should_include_file("s3://bucket/a.nc", (".tif", ".nc"), ()) + assert not generator.should_include_file( + "s3://bucket/a.nc", (".tif", ".nc"), (".nc",) + ) + assert generator.should_include_file("s3://bucket/a.h5", (), ()) + + +def test_media_type_for_extension_handles_known_types_and_cogs() -> None: + """Known extensions use specific media types.""" + assert generator.media_type_for_extension(".nc") == MediaType.NETCDF + assert generator.media_type_for_extension(".tif") == MediaType.GEOTIFF + assert generator.media_type_for_extension(".tif", is_cog=True) == MediaType.COG + assert generator.media_type_for_extension(".unknown") == "application/octet-stream" + + +def test_is_cloud_optimized_geotiff_reads_first_cog_validate_tuple_value( + monkeypatch, +) -> None: + """rio-cogeo returns a tuple whose first item is the validation result.""" + + def validate(_source: str) -> tuple[bool, list[str], list[str]]: + return False, ["not a COG"], [] + + monkeypatch.setattr(generator, "cog_validate", validate) + + assert not generator.is_cloud_optimized_geotiff("s3://bucket/not-cog.tif") + + +def test_run_filters_extensions_and_assigns_media_types( + monkeypatch, tmp_path: Path +) -> None: + """The catalog includes only files accepted by extension filters.""" + created_items = [] + + def from_url(source: str, *, region: str) -> str: + assert source == "s3://bucket/prefix" + assert region == "us-west-2" + return "store" + + def list_objects(_store: str) -> list[list[dict[str, str]]]: + return [ + [ + {"path": "image.tif"}, + {"path": "cube.nc"}, + {"path": "notes.txt"}, + ] + ] + + def create_item(**kwargs) -> Item: + created_items.append(kwargs) + return Item( + id=kwargs["id"], + geometry=None, + bbox=None, + datetime=datetime(2026, 1, 1, tzinfo=UTC), + properties={}, + ) + + def validate(_source: str) -> tuple[bool, list[str], list[str]]: + return True, [], [] + + monkeypatch.setattr(generator, "from_url", from_url) + monkeypatch.setattr(generator.obstore, "list", list_objects) + monkeypatch.setattr(generator, "create_stac_item", create_item) + monkeypatch.setattr(generator, "cog_validate", validate) + + generator.run( + source="s3://bucket/prefix", + output_dir=tmp_path, + include_extensions=".tif,.nc,.txt", + exclude_extensions=".txt", + ) + + assert [item["id"] for item in created_items] == ["image", "cube"] + assert created_items[0]["asset_media_type"] == MediaType.COG + assert created_items[1]["asset_media_type"] == MediaType.NETCDF + assert (tmp_path / "catalog.json").exists() diff --git a/tests/test_stac_item_outputs.py b/tests/test_stac_item_outputs.py new file mode 100644 index 0000000..84f6f90 --- /dev/null +++ b/tests/test_stac_item_outputs.py @@ -0,0 +1,69 @@ +"""Characterization tests for real STAC item creation outputs.""" + +from pathlib import Path + +import numpy as np +import pytest +import rasterio +from rasterio.errors import RasterioIOError +from rasterio.transform import from_origin +from rio_stac import create_stac_item + +from dps_stac_item_generator.generator import media_type_for_extension + + +def write_geotiff(path: Path) -> None: + """Write a tiny georeferenced GeoTIFF for STAC item characterization.""" + with rasterio.open( + path, + "w", + driver="GTiff", + height=1, + width=1, + count=1, + dtype="uint16", + crs="EPSG:4326", + transform=from_origin(-180, 90, 1, 1), + ) as dataset: + dataset.write(np.array([[[42]]], dtype="uint16")) + + +def test_real_geotiff_stac_item_shape(tmp_path: Path) -> None: + """A real GeoTIFF produces a STAC item with expected core fields.""" + source = tmp_path / "asset.tif" + write_geotiff(source) + + item = create_stac_item( + source=str(source), + id="asset", + with_proj=True, + with_raster=True, + asset_media_type=media_type_for_extension(".tif"), + ) + + item_dict = item.to_dict() + + assert item_dict["type"] == "Feature" + assert item_dict["stac_version"] + assert item_dict["id"] == "asset" + assert item_dict["geometry"]["type"] == "Polygon" + assert item_dict["bbox"] == [-180.0, 89.0, -179.0, 90.0] + assert item_dict["properties"]["proj:epsg"] == 4326 + assert item_dict["assets"]["asset"]["href"] == str(source) + assert item_dict["assets"]["asset"]["type"] == "image/tiff; application=geotiff" + assert item_dict["assets"]["asset"]["raster:bands"][0]["data_type"] == "uint16" + + +def test_text_file_stac_item_creation_fails(tmp_path: Path) -> None: + """rio-stac cannot create raster metadata for a plain text file.""" + source = tmp_path / "notes.txt" + source.write_text("hello\n") + + with pytest.raises( + RasterioIOError, match="not recognized as being in a supported file format" + ): + create_stac_item( + source=str(source), + id="notes", + asset_media_type="text/plain", + ) diff --git a/uv.lock b/uv.lock index 2785704..5e9b79b 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14'", @@ -317,8 +317,8 @@ wheels = [ ] [[package]] -name = "dps-import-cogs" -source = { virtual = "." } +name = "dps-stac-item-generator" +source = { editable = "." } dependencies = [ { name = "obstore" }, { name = "pystac" }, @@ -332,6 +332,7 @@ dev = [ { name = "maap-py" }, { name = "mypy" }, { name = "pre-commit" }, + { name = "pytest" }, { name = "ruff" }, ] @@ -349,6 +350,7 @@ dev = [ { name = "maap-py", specifier = ">=4.2.0" }, { name = "mypy", specifier = ">=1.18.2" }, { name = "pre-commit", specifier = ">=4.3.0" }, + { name = "pytest", specifier = ">=9.1.1" }, { name = "ruff", specifier = ">=0.14.3" }, ] @@ -447,6 +449,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/ed/1f1afb2e9e7f38a545d628f864d562a5ae64fe6f7a10e28ffb9b185b4e89/importlib_resources-6.5.2-py3-none-any.whl", hash = "sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec", size = 37461, upload-time = "2025-01-03T18:51:54.306Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "ipython" version = "9.8.0" @@ -1042,6 +1053,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl", hash = "sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3", size = 18651, upload-time = "2025-10-08T17:44:47.223Z" }, ] +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "pre-commit" version = "4.3.0" @@ -1260,6 +1280,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/01/eb465e19137b36ba683417e982907aa9c7df1fb0b968e1424e5d678ba0dc/pystac-1.14.1-py3-none-any.whl", hash = "sha256:19d73306d8fb94fbd66b7945ee5510e3574c8d48462f86e1e91e3f257b79722b", size = 207710, upload-time = "2025-09-18T15:13:47.189Z" }, ] +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" From 228ff6e8c978dd4a34aaef23b921564c990a6e26 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Fri, 26 Jun 2026 07:33:21 -0500 Subject: [PATCH 2/4] chore: simplify ci --- .github/workflows/ci.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f654ec0..90720ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,8 +9,8 @@ on: permissions: {} jobs: - lint-python: - name: Lint and typecheck + quality: + name: Quality checks runs-on: ubuntu-latest permissions: contents: read @@ -28,11 +28,8 @@ jobs: - name: Set up Python run: uv python install 3.14 - - name: Run ruff - run: uv run ruff check . - - - name: Run mypy - run: uv run mypy src tests + - name: Run pre-commit + run: uv run pre-commit run --all-files test: name: Test Python ${{ matrix.python-version }} @@ -56,9 +53,5 @@ jobs: - name: Set up Python run: uv python install ${{ matrix.python-version }} - - name: Run pre-commit - if: matrix.python-version == '3.14' - run: uv run pre-commit run --all-files - - name: Run tests run: uv run pytest From addaa5afe767a80be7f017cc1e0f32fc5b7bde9e Mon Sep 17 00:00:00 2001 From: hrodmn Date: Fri, 26 Jun 2026 11:16:26 -0500 Subject: [PATCH 3/4] chore: add smoketest script for local testing --- README.md | 16 +++ scripts/local_format_smoke_test.py | 215 +++++++++++++++++++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 scripts/local_format_smoke_test.py diff --git a/README.md b/README.md index e6cd172..f6a5467 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,22 @@ uv run main.py \ --exclude-extensions "" ``` +### Local format smoke test + +To generate local fixtures in `/tmp` and run the generator against them: + +```bash +uv run python scripts/local_format_smoke_test.py +``` + +The script creates tiny GeoTIFF, COG, NetCDF, HDF5, JPEG 2000, PNG, and JPEG fixtures when the required GDAL/HDF5 command-line tools are available. It logs the `/tmp/dps-stac-local-*` input and output directories when it finishes. + +To exercise the DPS wrapper instead of calling `main.py` directly: + +```bash +uv run python scripts/local_format_smoke_test.py --runner run-sh +``` + ### Parameters - `--source`: Source location of the files for which you want to generate STAC items (e.g., `s3://bucket/path/to/files/`) diff --git a/scripts/local_format_smoke_test.py b/scripts/local_format_smoke_test.py new file mode 100644 index 0000000..511b1b6 --- /dev/null +++ b/scripts/local_format_smoke_test.py @@ -0,0 +1,215 @@ +"""Generate local raster fixtures and run the STAC item generator against them.""" + +from __future__ import annotations + +import argparse +import logging +import shutil +import subprocess +import tempfile +from pathlib import Path + +import numpy as np +import rasterio +from rasterio.transform import from_origin + +LOGGER = logging.getLogger(__name__) +DEFAULT_EXTENSIONS = ".tif,.tiff,.nc,.h5,.hdf5,.jp2,.png,.jpg,.jpeg" + + +def configure_logging() -> None: + """Configure default logging for the smoke-test script.""" + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + +def write_raster(path: Path, *, driver: str, dtype: str = "uint16") -> None: + """Write a tiny georeferenced raster fixture.""" + profile = { + "driver": driver, + "height": 8, + "width": 8, + "count": 1, + "dtype": dtype, + "crs": "EPSG:4326", + "transform": from_origin(-180, 90, 1, 1), + } + data = np.arange(64, dtype=dtype).reshape(1, 8, 8) + + with rasterio.open(path, "w", **profile) as dataset: + dataset.write(data) + + +def run_command(command: list[str], *, cwd: Path | None = None) -> None: + """Run a command and raise if it fails.""" + LOGGER.info("running: %s", " ".join(command)) + subprocess.run(command, cwd=cwd, check=True) + + +def maybe_run_command(command: list[str]) -> bool: + """Run a fixture-generation command when its executable is available.""" + if shutil.which(command[0]) is None: + LOGGER.warning( + "skipping %s fixture because %s is not installed", command[-1], command[0] + ) + return False + + run_command(command) + return True + + +def write_hdf5_fixture(path: Path) -> bool: + """Write a small HDF5 fixture with h5import when that tool is available.""" + if shutil.which("h5import") is None: + LOGGER.warning("skipping %s fixture because h5import is not installed", path) + return False + + data_path = path.with_suffix(".h5-data.txt") + config_path = path.with_suffix(".h5import.conf") + data_path.write_text("1 2 3 4\n5 6 7 8\n", encoding="utf-8") + config_path.write_text( + "\n".join( + [ + "PATH /data", + "INPUT-CLASS TEXTUIN", + "INPUT-SIZE 8", + "INPUT-BYTE-ORDER LE", + "RANK 2", + "DIMENSION-SIZES 2 4", + "OUTPUT-CLASS UIN", + "OUTPUT-SIZE 8", + "", + ] + ), + encoding="utf-8", + ) + run_command(["h5import", str(data_path), "-c", str(config_path), "-o", str(path)]) + return True + + +def generate_fixtures(input_dir: Path) -> list[Path]: + """Generate local files in formats commonly readable by GDAL/rasterio.""" + input_dir.mkdir(parents=True, exist_ok=True) + created: list[Path] = [] + + geotiff = input_dir / "sample-geotiff.tif" + write_raster(geotiff, driver="GTiff") + created.append(geotiff) + + cog = input_dir / "sample-cog.tif" + write_raster(cog, driver="COG") + created.append(cog) + + conversions = [ + ("netCDF", input_dir / "sample-netcdf.nc"), + ("JP2OpenJPEG", input_dir / "sample-jpeg2000.jp2"), + ("PNG", input_dir / "sample.png"), + ("JPEG", input_dir / "sample.jpg"), + ] + for driver, output_path in conversions: + if maybe_run_command( + ["gdal_translate", "-q", "-of", driver, str(geotiff), str(output_path)] + ): + created.append(output_path) + + hdf5 = input_dir / "sample-hdf5.h5" + if write_hdf5_fixture(hdf5): + created.append(hdf5) + + (input_dir / "skip-me.txt").write_text("not a raster\n", encoding="utf-8") + return created + + +def log_rasterio_open_results(paths: list[Path]) -> None: + """Log whether rasterio can open each generated fixture.""" + for path in paths: + with rasterio.open(path) as dataset: + LOGGER.info( + "rasterio opened %-24s driver=%s size=%sx%s bands=%s crs=%s", + path.name, + dataset.driver, + dataset.width, + dataset.height, + dataset.count, + dataset.crs, + ) + + +def run_generator( + args: argparse.Namespace, repo_root: Path, source: str, output_dir: Path +) -> None: + """Run the project entry point selected by the caller.""" + if args.runner == "run-sh": + run_command( + [ + str(repo_root / "run.sh"), + source, + args.region, + args.include_extensions, + args.exclude_extensions, + ], + cwd=output_dir.parent, + ) + return + + run_command( + [ + "uv", + "run", + str(repo_root / "main.py"), + f"--source={source}", + f"--output_dir={output_dir}", + f"--region={args.region}", + f"--include-extensions={args.include_extensions}", + f"--exclude-extensions={args.exclude_extensions}", + ], + cwd=repo_root, + ) + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Generate local GDAL/rasterio fixtures in /tmp and run this project against them." + ) + parser.add_argument( + "--runner", + choices=("main-py", "run-sh"), + default="main-py", + help="Project entry point to exercise.", + ) + parser.add_argument( + "--work-dir", + type=Path, + help="Directory to create/use. Defaults to a new /tmp/dps-stac-local-* directory.", + ) + parser.add_argument("--region", default="us-west-2") + parser.add_argument("--include-extensions", default=DEFAULT_EXTENSIONS) + parser.add_argument("--exclude-extensions", default=".txt,.xml") + return parser.parse_args() + + +def main() -> None: + """Generate fixtures, run the STAC item generator, and print output paths.""" + configure_logging() + args = parse_args() + repo_root = Path(__file__).resolve().parents[1] + work_dir = args.work_dir or Path( + tempfile.mkdtemp(prefix="dps-stac-local-", dir="/tmp") + ) + input_dir = work_dir / "input" + output_dir = work_dir / "output" + output_dir.mkdir(parents=True, exist_ok=True) + + fixtures = generate_fixtures(input_dir) + log_rasterio_open_results(fixtures) + + source = input_dir.resolve().as_uri() + run_generator(args, repo_root, source, output_dir) + + LOGGER.info("input fixtures: %s", input_dir) + LOGGER.info("STAC output: %s", output_dir) + LOGGER.info("catalog: %s", output_dir / "catalog.json") + + +if __name__ == "__main__": + main() From 37629a2f37ed82714eb349b85a6b6aa8c702d927 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Fri, 26 Jun 2026 11:22:50 -0500 Subject: [PATCH 4/4] ci: add release-please workflow --- .../workflows/conventional-commits-prs.yml | 23 +++++++++++++++ .github/workflows/release-please.yml | 24 ++++++++++++++++ .release-please-manifest.json | 3 ++ CHANGELOG.md | 28 +++++++++++++++++++ release-please-config.json | 9 ++++++ 5 files changed, 87 insertions(+) create mode 100644 .github/workflows/conventional-commits-prs.yml create mode 100644 .github/workflows/release-please.yml create mode 100644 .release-please-manifest.json create mode 100644 CHANGELOG.md create mode 100644 release-please-config.json diff --git a/.github/workflows/conventional-commits-prs.yml b/.github/workflows/conventional-commits-prs.yml new file mode 100644 index 0000000..bdbd6fb --- /dev/null +++ b/.github/workflows/conventional-commits-prs.yml @@ -0,0 +1,23 @@ +name: PR Conventional Commit Validation + +# NOTE: pull_request_target runs in the base repo context with access to secrets. +# Do NOT checkout untrusted code in this workflow. +on: + # zizmor: ignore[dangerous-triggers] + # Safe: this workflow does not check out any code; it only validates + # the PR title via the GitHub API. + pull_request_target: + types: [opened, synchronize, reopened, edited] + +permissions: {} + +jobs: + validate-pr-title: + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: PR Conventional Commit Validation + uses: ytanikin/pr-conventional-commits@639145d78959c53c43112365837e3abd21ed67c1 # 1.5.2 + with: + task_types: '["feat","fix","docs","test","ci","refactor","perf","chore","revert"]' diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 0000000..e58572f --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,24 @@ +name: Release Please + +on: + push: + branches: [main] + +permissions: + contents: write + pull-requests: write + issues: write + +concurrency: + group: release-please-${{ github.ref }} + cancel-in-progress: false + +jobs: + release-please: + name: Create release PR or GitHub release + runs-on: ubuntu-latest + steps: + - name: Run release-please + uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..466df71 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.1.0" +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f0e1343 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,28 @@ +# Changelog + +## [0.1.0] - 2025-12-12 + +Initial release of the MAAP DPS algorithm for generating STAC item metadata for existing Cloud Optimized GeoTIFF assets in object storage. + +### Added + +- Added the `GenerateCogStacItems` DPS algorithm at version `v0.1`. +- Added direct Python invocation through `main.py` for local STAC item generation. +- Added object storage listing for source locations such as S3 paths. +- Added filtering for GeoTIFF assets before STAC item creation. +- Added STAC item generation with `rio-stac` and catalog output with `pystac`. +- Added projection and raster metadata handling for generated STAC items. +- Added MAAP DPS wrapper scripts, including positional and named-argument execution paths. +- Added support for direct bucket access and MAAP-compatible cloud raster defaults. + +### Fixed + +- Improved generated STAC item IDs. +- Fixed Fmask nodata handling. +- Fixed bbox serialization. +- Set missing `proj:transform` and `proj:shape` values when available. + +### Documentation + +- Documented local usage with `uv run main.py`. +- Documented required inputs and generated STAC catalog outputs. diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..0433e47 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,9 @@ +{ + "include-v-in-tag": true, + "packages": { + ".": { + "release-type": "simple", + "changelog-path": "CHANGELOG.md" + } + } +}