diff --git a/README.md b/README.md index 651ba1b..e450921 100644 --- a/README.md +++ b/README.md @@ -81,12 +81,22 @@ SLC_masking_algo: str = 'mask_scl_dilation' Masking method for Sentinel-2 optical data ('satio', 'mask_scl_dilation', None) Note: if set to None, no masking is applied and the S2 L2A data is used as is. -apply_cloud_mask : bool = True +apply_cloud_mask: bool = True if True, the Sentinel-2 or PlanetScope data is masked for clouds (based on Sentinel-2 QA band). - If False, no masking is applied but the mask band is still created and added to the cube. - Note: no effect when 'mask_scl_dilation' parameter is set to None. - -append : bool = True + If False, no masking is applied but the mask band is still created and added to the cube. (Note: the cloud + mask band is removed from the feature cube as soon the time domain is aggregated.) + Note: no effect when 'mask_scl_dilation' parameter is set to None. + +get_NVBT: bool = False + Number of Valid Binned Timesteps: Specifies the count of valid timesteps after temporal binning. + This can be used as an input data quality indicator. + if True, NVBT is calculated for optical data after cloud masking and temporal binning and radar data after + sar_backscatter correction and temporal binning. + (if both were activated) but without linear interpolation (if activated). + Note: 'band' S1_NVBT and/or S2_NVBT is only added to cubes with removed time domain (feature cubes). + Note: algorithm is applied even when cloud masking was deactivated or temporal aggregation was skipped. + +append: bool = True if the VI's are appended to the reflectance/radar time series cube OR replace them S2_scaling: list = [0, 10000, 0, 1.0] diff --git a/notebooks/test_nobs.ipynb b/notebooks/test_nobs.ipynb new file mode 100644 index 0000000..eae896e --- /dev/null +++ b/notebooks/test_nobs.ipynb @@ -0,0 +1,460 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d46e36dcd1f7a0ef", + "metadata": {}, + "source": [ + "# example how to generate the WEED standard feature cube based on EO data (Sentinel-1 and Sentinel-2)\n", + "this tests include the generation of the feature cube with and without NVBT band. The NVBT band is a measure of the number of valid input data timesteps after cloud masking and the internal temporal binning.\n", + "The NVBT band can be useful for identifying areas with high or low levels of observation, which can be important for a variety of applications, such as monitoring changes in land use or assessing the quality of data in a given area." + ] + }, + { + "cell_type": "code", + "id": "initial_id", + "metadata": { + "ExecuteTime": { + "end_time": "2026-07-01T10:10:45.301457300Z", + "start_time": "2026-07-01T10:10:43.592557200Z" + } + }, + "source": [ + "from eo_processing.utils.helper import init_connection\n", + "from eo_processing.openeo.processing import generate_master_feature_cube\n", + "from eo_processing.config.settings import get_advanced_options, get_job_options, get_collection_options" + ], + "outputs": [], + "execution_count": 1 + }, + { + "cell_type": "markdown", + "id": "c9abe4dec39f3145", + "metadata": {}, + "source": [ + "#### declare space and time" + ] + }, + { + "cell_type": "code", + "id": "1c319454e1c292f1", + "metadata": { + "ExecuteTime": { + "end_time": "2026-07-01T10:10:48.565131400Z", + "start_time": "2026-07-01T10:10:48.548481800Z" + } + }, + "source": [ + "# the time context is given by start and end date\n", + "year = 2024\n", + "start = f'{year}-01-01'\n", + "end = f'{year+1}-01-01' # the end is always exclusive\n", + "\n", + "# the space context is defined as a bounding box dictionary with south,west,north,east and crs\n", + "# we take as example an 10x10km tile in EU LAEA grid around Vienna\n", + "AOI = {'west': 4780000, 'east': 4790000, 'south': 2830000, 'north': 2840000, 'crs': 3035}" + ], + "outputs": [], + "execution_count": 2 + }, + { + "cell_type": "markdown", + "id": "39fc1ee260fa1492", + "metadata": {}, + "source": [ + "### get processing_options for the eo_processing functions, collection_options and job_options" + ] + }, + { + "cell_type": "code", + "id": "fa16159203be2475", + "metadata": { + "ExecuteTime": { + "end_time": "2026-07-01T10:11:06.858284600Z", + "start_time": "2026-07-01T10:11:06.842773300Z" + } + }, + "source": [ + "processing_options = get_advanced_options(provider='cdse', skip_check_S1=False, skip_check_S2=True)\n", + "job_options = get_job_options(provider='cdse', task='feature_generation')\n", + "collection_options = get_collection_options(provider='cdse')\n", + "processing_options.update({'openeo_chunk_size': 64})" + ], + "outputs": [], + "execution_count": 3 + }, + { + "cell_type": "code", + "id": "72616c9c-4583-4132-a38c-23988aeb8434", + "metadata": { + "ExecuteTime": { + "end_time": "2026-07-01T10:11:08.354813700Z", + "start_time": "2026-07-01T10:11:08.305942900Z" + } + }, + "source": [ + "processing_options" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "{'provider': 'cdse',\n", + " 's1_orbitdirection': 'DESCENDING',\n", + " 'target_crs': 3035,\n", + " 'resolution': 10.0,\n", + " 'time_interpolation': False,\n", + " 'ts_interval': 'dekad',\n", + " 'S2_temporal_reducer': 'median',\n", + " 'S1_temporal_reducer': 'mean',\n", + " 'SLC_masking_algo': 'mask_scl_dilation',\n", + " 'S2_max_cloud_cover': 95,\n", + " 'S2_bands': ['B02',\n", + " 'B03',\n", + " 'B04',\n", + " 'B05',\n", + " 'B06',\n", + " 'B07',\n", + " 'B08',\n", + " 'B8A',\n", + " 'B11',\n", + " 'B12'],\n", + " 's2_tileid_list': None,\n", + " 'skip_check_S1': False,\n", + " 'skip_check_S2': True,\n", + " 'apply_cloud_mask': True,\n", + " 'get_NVBT': False,\n", + " 'optical_vi_list': ['ABDI1',\n", + " 'ABDI2',\n", + " 'AWEInsh',\n", + " 'AVI',\n", + " 'BLFEI',\n", + " 'CIRE',\n", + " 'EVI',\n", + " 'IRECI',\n", + " 'MBWI',\n", + " 'MNDWI',\n", + " 'MNDVI',\n", + " 'NDMI',\n", + " 'NDVI',\n", + " 'NDVIMNDWI',\n", + " 'NDWI',\n", + " 'NMDI',\n", + " 'NIRv',\n", + " 'S2WI',\n", + " 'S2REP',\n", + " 'WRI'],\n", + " 'radar_vi_list': ['VHVVD', 'VHVVR', 'DpRVIVV'],\n", + " 'S2_scaling': [0, 10000, 0, 1.0],\n", + " 'S1_db_rescale': True,\n", + " 'append': True,\n", + " 'openeo_chunk_size': 64}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 4 + }, + { + "cell_type": "markdown", + "id": "a05d287923cf93eb", + "metadata": {}, + "source": [ + "### establish connection to openEO" + ] + }, + { + "cell_type": "code", + "id": "d06f89bea20dce40", + "metadata": { + "ExecuteTime": { + "end_time": "2026-07-01T10:11:15.268379200Z", + "start_time": "2026-07-01T10:11:14.743118500Z" + } + }, + "source": "con = init_connection(provider='cdse')", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Authenticated using refresh token.\n" + ] + } + ], + "execution_count": 5 + }, + { + "cell_type": "markdown", + "id": "adcab1031cc5e6c2", + "metadata": {}, + "source": [ + "### run the feature cube generation WITHOUT NVBT band" + ] + }, + { + "cell_type": "code", + "id": "f32c4461ec4331a5", + "metadata": { + "ExecuteTime": { + "end_time": "2026-07-01T10:11:21.277193500Z", + "start_time": "2026-07-01T10:11:21.260122500Z" + } + }, + "source": [ + "# update job_options due toBerts setting from last inference runs\n", + "# ToDO: optimize the job settings for feature_cube_generation_with_nobs & feature_cube_generation and put in settings + add correct task to 'get_job_options'\n", + "job_options.update({\n", + " \"driver-memory\": \"4G\",\n", + " \"driver-memoryOverhead\": \"4G\",\n", + " \"executor-memory\": \"5G\",\n", + " \"executor-memoryOverhead\": \"3g\",\n", + " \"max-executors\": 10,\n", + " \"python-memory\": \"disable\",\n", + " \"allow_empty_cubes\": True,\n", + " \"soft-errors\": 0.05})" + ], + "outputs": [], + "execution_count": 6 + }, + { + "cell_type": "code", + "id": "fc107ebf279499cf", + "metadata": {}, + "source": [ + "# get master cube without nobs\n", + "data = generate_master_feature_cube(con, AOI, start, end, **collection_options, **processing_options)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "id": "f0b7c0577d597daf", + "metadata": {}, + "source": "data.execute_batch(r'C:\\Users\\buchhorm\\Downloads\\test_cube\\features_cube_v5.tif', title='feature without nobs (10x10km)', job_options=job_options)", + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "3f775e9b75ec1382", + "metadata": {}, + "source": [ + "## run with activated NVBT generation" + ] + }, + { + "cell_type": "code", + "id": "95a0e6270cf65cc3", + "metadata": {}, + "source": [ + "# now we run same with nobs_perc band\n", + "processing_options.update({'get_NVBT': True})\n", + "data2 = generate_master_feature_cube(con, AOI, start, end, **collection_options, **processing_options)" + ], + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "data2.execute_batch(r'C:\\Users\\buchhorm\\Downloads\\test_cube\\features_cube_with_nobs_v5.tif', title='feature with nobs (10x10km)', job_options=job_options)", + "id": "b522dd0e9b4adaa5" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## now we run this 10x10km test in a bigger context - we generate the full EO feature cube with NVBT band\n", + "Note: since the new ECDC version and alphaEarth STACs are not complete THIS loading has to be tested later" + ], + "id": "3684f8b64901068a" + }, + { + "cell_type": "code", + "id": "bcb5d1b835c659eb", + "metadata": { + "ExecuteTime": { + "end_time": "2026-07-01T10:12:04.872738900Z", + "start_time": "2026-07-01T10:11:27.488000800Z" + } + }, + "source": [ + "from habitat_mapping.openeo.feature_cubes import create_EOfeature_cube_WEED_V1\n", + "processing_options.update(target_crs = 3035)\n", + "processing_options.update({'get_NVBT': True})\n", + "processing_options.update({'openeo_chunk_size': 16})\n", + "job_options.update({\"allow_empty_cubes\": True})\n", + "job_options.update({\n", + " \"driver-memory\": \"4G\",\n", + " \"driver-memoryOverhead\": \"4G\",\n", + " \"executor-memory\": \"5G\",\n", + " \"executor-memoryOverhead\": \"3G\",\n", + " \"max-executors\": 10,\n", + " \"python-memory\": \"disable\",\n", + " \"soft-errors\": 0.05,\n", + "})\n", + "data3 = create_EOfeature_cube_WEED_V1(con, AOI, start, end, collection_options, processing_options)" + ], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Deriving band listing from unordered `item_assets`\n", + "The specified bands ['precipitation-flux', 'temperature-mean'] in `load_stac` are not a subset of the bands [] found in the STAC metadata (unknown bands: ['precipitation-flux', 'temperature-mean']). Working with specified bands as is.\n" + ] + } + ], + "execution_count": 7 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2026-07-01T10:42:53.609735700Z", + "start_time": "2026-07-01T10:12:16.483423100Z" + } + }, + "cell_type": "code", + "source": "data3.execute_batch(r'C:\\Users\\buchhorm\\Downloads\\test_cube\\eo_feature_cube_beta2_10x10.tif', title='eo_feature cube (10x10km)', job_options=job_options)", + "id": "641442d062a77b1e", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0:00:00 Job 'j-2607011012174934aeec7870e2685f99': send 'start'\n", + "0:00:07 Job 'j-2607011012174934aeec7870e2685f99': queued (progress 0%)\n", + "0:00:13 Job 'j-2607011012174934aeec7870e2685f99': queued (progress 0%)\n", + "0:00:19 Job 'j-2607011012174934aeec7870e2685f99': queued (progress 0%)\n", + "0:00:27 Job 'j-2607011012174934aeec7870e2685f99': queued (progress 0%)\n", + "0:00:37 Job 'j-2607011012174934aeec7870e2685f99': queued (progress 0%)\n", + "0:00:50 Job 'j-2607011012174934aeec7870e2685f99': queued (progress 0%)\n", + "0:01:05 Job 'j-2607011012174934aeec7870e2685f99': running (progress 8.9%)\n", + "0:01:24 Job 'j-2607011012174934aeec7870e2685f99': running (progress 11.5%)\n", + "0:01:48 Job 'j-2607011012174934aeec7870e2685f99': running (progress 14.5%)\n", + "0:02:19 Job 'j-2607011012174934aeec7870e2685f99': running (progress 18.0%)\n", + "0:02:56 Job 'j-2607011012174934aeec7870e2685f99': running (progress 22.0%)\n", + "0:03:43 Job 'j-2607011012174934aeec7870e2685f99': running (progress 26.5%)\n", + "0:04:41 Job 'j-2607011012174934aeec7870e2685f99': running (progress 31.4%)\n", + "0:05:41 Job 'j-2607011012174934aeec7870e2685f99': running (progress 35.8%)\n", + "0:06:41 Job 'j-2607011012174934aeec7870e2685f99': running (progress 39.7%)\n", + "0:07:42 Job 'j-2607011012174934aeec7870e2685f99': running (progress 43.1%)\n", + "0:08:42 Job 'j-2607011012174934aeec7870e2685f99': running (progress 46.2%)\n", + "0:09:42 Job 'j-2607011012174934aeec7870e2685f99': running (progress 49.0%)\n", + "0:10:43 Job 'j-2607011012174934aeec7870e2685f99': running (progress 51.5%)\n", + "0:11:43 Job 'j-2607011012174934aeec7870e2685f99': running (progress 53.7%)\n", + "0:12:43 Job 'j-2607011012174934aeec7870e2685f99': running (progress 55.8%)\n", + "0:13:44 Job 'j-2607011012174934aeec7870e2685f99': running (progress 57.7%)\n", + "0:14:44 Job 'j-2607011012174934aeec7870e2685f99': running (progress 59.4%)\n", + "0:15:44 Job 'j-2607011012174934aeec7870e2685f99': running (progress 61.0%)\n", + "0:16:44 Job 'j-2607011012174934aeec7870e2685f99': running (progress 62.4%)\n", + "0:17:45 Job 'j-2607011012174934aeec7870e2685f99': running (progress 63.8%)\n", + "0:18:45 Job 'j-2607011012174934aeec7870e2685f99': running (progress 65.1%)\n", + "0:19:45 Job 'j-2607011012174934aeec7870e2685f99': running (progress 66.3%)\n", + "0:20:45 Job 'j-2607011012174934aeec7870e2685f99': running (progress 67.4%)\n", + "0:21:45 Job 'j-2607011012174934aeec7870e2685f99': running (progress 68.4%)\n", + "0:22:45 Job 'j-2607011012174934aeec7870e2685f99': running (progress 69.4%)\n", + "0:23:46 Job 'j-2607011012174934aeec7870e2685f99': running (progress 70.3%)\n", + "0:24:46 Job 'j-2607011012174934aeec7870e2685f99': running (progress 71.1%)\n", + "0:25:46 Job 'j-2607011012174934aeec7870e2685f99': running (progress 72.0%)\n", + "0:26:47 Job 'j-2607011012174934aeec7870e2685f99': running (progress 72.7%)\n", + "0:27:47 Job 'j-2607011012174934aeec7870e2685f99': running (progress 73.5%)\n", + "0:28:49 Job 'j-2607011012174934aeec7870e2685f99': running (progress 74.2%)\n", + "0:29:50 Job 'j-2607011012174934aeec7870e2685f99': finished (progress 100%)\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 8 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## now upscale the the final 20x20km tiles", + "id": "3d9134ce13226561" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "# we take as example an 20x20km tile in EU LAEA grid around Vienna\n", + "AOI = {'west': 4780000, 'east': 4800000, 'south': 2820000, 'north': 2840000, 'crs': 3035}\n", + "# increasing driver memory\n", + "job_options.update({\n", + " \"driver-memory\": \"10G\",\n", + " \"driver-memoryOverhead\": \"4G\",\n", + "})\n", + "data4 = create_EOfeature_cube_WEED_V1(con, AOI, start, end, collection_options, processing_options)" + ], + "id": "b7fe80d4f70e9b87" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "data4.execute_batch(r'C:\\Users\\buchhorm\\Downloads\\test_cube\\eo_feature_cube_beta2_20x20.tif', title='eo_feature cube (20x20km)', job_options=job_options)", + "id": "a2be83f50c1aa7d0" + } + ], + "metadata": { + "kernelspec": { + "display_name": "weed", + "language": "python", + "name": "weed" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/requirements.txt b/requirements.txt index d88a11c..1f5f001 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ fastparquet>=2024.5.0 pydrive2[fsspec]>=1.21.1 hvac>=2.3.0 boto3>=1.36.16 +requests>=2.32.0 skl2onnx==1.19.1 psycopg2-binary>=2.9.10 folium>=0.17.0 diff --git a/scripts/global_120x120km_UTM_tiling-grid.py b/scripts/global_120x120km_UTM_tiling-grid.py new file mode 100644 index 0000000..b692e37 --- /dev/null +++ b/scripts/global_120x120km_UTM_tiling-grid.py @@ -0,0 +1,120 @@ +""" +this script generates a global 120x120km tiling grid in UTM. + +""" + + +''' +STEPS +1. load the UTM zones gpkg +2. load the global land masses gpkg +3. loop over the 120 UTM zones + a. load the full 120x120 km grid depending on the hemisphere and change the epsg to the UTM zone + b. load a copy of the UTM zone polygon and warp to correct epsg + c. select the grid cells intersecting the UTM zone polygon and delete non-needed + d. load a copy of the land masses polygon and warp to correct epsg + e. select the grid cells intersecting the land masses and dlete non-needed ones + f. run over the remaining 120x120km tiles and add the naming as well as + add the openEO bbox dict using the bounds of the polygon (plus add the bounds in WGS84) + g. convert the geodataframe back to EPSG:4326 and save it to a new gpkg +4. combine all UTM zones into one gpkg and clean up the metadata +5. save the final global processing grid in the package ressources + +''' + +import geopandas as gpd +import pandas as pd +import math +from eo_processing.utils.mgrs import UTM_2_grid20id +import os + +# standard paths +path_grid_n = os.path.normpath(r'C:\Users\buchhorm\Downloads\120x120km_grid\basic_120x120km_grid_nothern_no-crs_v2.gpkg') +path_grid_s = os.path.normpath(r'C:\Users\buchhorm\Downloads\120x120km_grid\basic_120x120km_grid_southern_no-crs_v2.gpkg') +path_land = os.path.normpath(r'C:\Users\buchhorm\Downloads\120x120km_grid\land_sea_mask_20kmbuffered_EPSG4326_v2.gpkg') +path_utm = os.path.normpath(r'C:\Users\buchhorm\Downloads\120x120km_grid\UTM_zones_high-res_EPSG4326.gpkg') + +# load UTM zones +gdf_utm = gpd.read_file(path_utm) + +# loop over all UTM zones +lZones = gdf_utm.name.unique().tolist() +lFiles = [] + +for UTMzone in lZones: + print(f'processing zone: {UTMzone}') + # get the UTM zone polygon and the EPSG + if UTMzone[-1] == 'N': + epsg = 32600 + int(UTMzone[:2]) + else: + epsg = 32700 + int(UTMzone[:2]) + + gdf_zone = gdf_utm[gdf_utm.name == UTMzone].copy() + + path_out = os.path.normpath( + r'C:\Users\buchhorm\Downloads\120x120km_grid\results\UTM_zone_{0}.gpkg'.format( + epsg)) + os.makedirs(os.path.dirname(path_out), exist_ok=True) + + if not os.path.exists(path_out): + # get the land masses + gdf_land = gpd.read_file(path_land) + gdf_land = gdf_land.clip(gdf_zone) + gdf_land = gdf_land.dissolve() + + # bring all to correct epsg + gdf_zone = gdf_zone.to_crs(epsg=epsg) + gdf_land = gdf_land.to_crs(epsg=epsg) + + # short cut if the gdf_land is empty + if gdf_land.empty: + continue + + # now we load the basic 120x120km grid and set the crs to the correct epsg + if UTMzone[-1] == 'N': + gdf_grid = gpd.read_file(path_grid_n).set_crs(epsg=epsg, allow_override=True) + else: + gdf_grid = gpd.read_file(path_grid_s).set_crs(epsg=epsg, allow_override=True) + + # select grid cells which are intersecting with utm zone + gdf_grid = gdf_grid[gdf_grid.intersects(gdf_zone.union_all(method='coverage'))] + + # intersecting + gdf_grid = gdf_grid[gdf_grid.intersects(gdf_land.union_all(method='coverage'))] + + # add the bbox dict for openEO + gdf_grid['bbox_dict'] = gdf_grid.apply( + lambda row: { + 'west': math.floor(row.geometry.bounds[0] / 100) * 100., + 'south': math.floor(row.geometry.bounds[1] / 100) * 100., + 'east': math.ceil(row.geometry.bounds[2] / 100) * 100., + 'north': math.ceil(row.geometry.bounds[3] / 100) * 100., + 'crs': epsg + }, axis=1 + ) + + # save to disk + + + gdf_grid[['left', 'top', 'right', 'bottom', 'row_index','col_index', 'bbox_dict', 'geometry']].to_file( + path_out) + else: + # load + gdf_grid = gpd.read_file(path_out) + gdf_zone = gdf_zone.to_crs(epsg=epsg) + gdf_grid = gdf_grid.clip(gdf_zone) + gdf_grid.to_file(path_out) + + lFiles.append(path_out) + + +# load all geopackages which were produced and convert to EPSG:4326 and merge them into one GeoDataFrame +result = gpd.GeoDataFrame(columns=['left', 'top', 'right', 'bottom', 'row_index','col_index', 'bbox_dict', 'geometry'], geometry='geometry', crs='EPSG:4326') + +for file in lFiles: + gdf_tmp = gpd.read_file(file) + gdf_tmp = gdf_tmp.to_crs(epsg=4326) + result = pd.concat([result, gdf_tmp], ignore_index=True) + +# save final result to disk +result.to_file(os.path.normpath(r'C:\Users\buchhorm\Downloads\120x120km_grid\ECDC_global_120x120km_grid.gpkg')) diff --git a/scripts/laea_tiling_grids_2_high_res_Sentinel2_tiles.py b/scripts/laea_tiling_grids_2_high_res_Sentinel2_tiles.py index eb6500a..4444e26 100644 --- a/scripts/laea_tiling_grids_2_high_res_Sentinel2_tiles.py +++ b/scripts/laea_tiling_grids_2_high_res_Sentinel2_tiles.py @@ -28,17 +28,17 @@ def get_epsg(tile_id): return target_EPSG # filter to tiles in UTM zones needed for pan EU -df2 = pd.read_csv(r'C:\Users\buchhorm\Downloads\new_grids\utm_codes_europe.csv') -df2['utm'] = df2['ZONE'].astype(str) + df2['ROW_'] -lNeeded = df2['utm'].tolist() +#df2 = pd.read_csv(r'C:\Users\buchhorm\Downloads\new_grids\utm_codes_europe.csv') +#df2['utm'] = df2['ZONE'].astype(str) + df2['ROW_'] +#lNeeded = df2['utm'].tolist() ## Start read out everything we need with beautiful soup from Sentinel-2 KML -xml_file = r'C:\Users\buchhorm\Downloads\new_grids\Sentinel2_tiles.kml' -soup = bs.BeautifulSoup(open(xml_file), 'xml') +xml_file = r'C:\Users\buchhorm\Downloads\120x120km_grid\S2_tiles.kml' +soup = bs.BeautifulSoup(open(xml_file), 'html.parser') #find all VRTRasterBands in the soup -tiles = soup.findAll('description') +tiles = soup.find_all('description') data = [] @@ -57,8 +57,8 @@ def get_epsg(tile_id): cols = [ele.text.strip() for ele in cols] if cols[0] == 'TILE_ID': data1 = cols[1] if cols[0] == 'UTM_WKT': data2 = cols[1] - if data1[:3] in lNeeded: - data.append((data1, data2, get_epsg(data1))) + + data.append((data1, data2, get_epsg(data1))) df = pd.DataFrame(data, columns=['tile_id', 'geometry', 'epsg']) data = None @@ -80,10 +80,10 @@ def get_epsg(tile_id): # add extra points in 250m intervall gdf['geometry'] = gdf['geometry'].apply(lambda x: x.segmentize(250)) - gdf2 = gdf.to_crs(epsg=3035) + gdf2 = gdf.to_crs(epsg=4326) list_results.append(gdf2) result = gpd.GeoDataFrame(pd.concat(list_results, ignore_index=True), crs=list_results[0].crs) # write out -result.to_file(r'C:\Users\buchhorm\Downloads\new_grids\Sentinel2_tiling_grid_EU_high_res_EPSG3035.gpkg', driver='GPKG') +result.to_file(r'C:\Users\buchhorm\Downloads\120x120km_grid\Sentinel2_tiling_grid_high_res_EPSG4326.gpkg', driver='GPKG') diff --git a/src/eo_processing/_version.py b/src/eo_processing/_version.py index 5680ebc..35b620f 100644 --- a/src/eo_processing/_version.py +++ b/src/eo_processing/_version.py @@ -1,3 +1,3 @@ #!/usr/bin/env python3 -__version__ = '0.6.0' +__version__ = '0.7.0' diff --git a/src/eo_processing/config/settings.py b/src/eo_processing/config/settings.py index e9a0312..12f11a1 100644 --- a/src/eo_processing/config/settings.py +++ b/src/eo_processing/config/settings.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import List, Optional, Dict, Union, TYPE_CHECKING +from typing import List, Optional, Dict, Union, TYPE_CHECKING, Any if TYPE_CHECKING: from eo_processing.config.data_formats import storage_option_format @@ -14,12 +14,13 @@ # --------------------------------------------------- # standard processing options S1/S2 TARGET_CRS: int = 3035 # can be all known EPSG codes -TARGET_RESOLUTION: float = 10. +TARGET_RESOLUTION: float|int = 10. S1_ORBITDIRECTION: str = 'DESCENDING' S2_BANDS: List[str] = ["B02", "B03", "B04", "B05", "B06", "B07", "B08", "B8A", "B11", "B12"] S2_MAX_CLOUD_COVER: int = 95 MASKING_ALGO: str = 'mask_scl_dilation' APPLY_CLOUD_MASK: bool = True +GET_NVBT: bool = False S2_TILEID_LIST: Optional[List[str]] = None SKIP_CHECK_S1: bool = False SKIP_CHECK_S2: bool = False @@ -52,23 +53,23 @@ RADAR_LIST: List[str] = ['VHVVD', 'VHVVR', 'DpRVIVV'] -S2_SCALING: List = [0, 10000, 0, 1.0] +S2_SCALING: List[Union[int, float]] = [0, 10000, 0, 1.0] # --------------------------------------------------- # Planet Processing options PLANET_MASKING_ALGO: str = 'mask_udm_dilation' PLANET_BANDS: List[str] = ["B01", "B02", "B03", "B04", "B05", "B06", "B07", "B08"] # Same for UDM2 and Spectral bands -PLANET_RESOLUTION: float = 3.0 +PLANET_RESOLUTION: float|int = 3.0 PLANET_VI_LIST: List[str] = ['NDVI', 'AVI', 'CIRE', 'NIRv', 'NDWI' ] -PLANET_SCALING: List = [0, 10000, 0, 1.0] - +PLANET_SCALING: List[Union[int, float]] = [0, 10000, 0, 1.0] +HRVPP_COLLECTION: str = 'copernicus_r_utm-wgs84_10_m_hrvpp-vpp_p_2017-now_v01_direct' #special case with 2 seasons iso 1 year # --------------------------------------------------- # Job options for OpenEO -OPENEO_EXTRACT_JOB_OPTIONS: Dict[str, str] = { +OPENEO_EXTRACT_JOB_OPTIONS: Dict[str, Union[str, int, float, List]] = { "driver-memory": "4G", "driver-memoryOverhead": "8G", "driver-cores": 2, @@ -80,7 +81,7 @@ "stac-version":"1.1" } -OPENEO_EXTRACT_CREO_JOB_OPTIONS: Dict[str, str] = { +OPENEO_EXTRACT_CREO_JOB_OPTIONS: Dict[str, Union[str, int, float, List]] = { "driver-memory": "4G", "driver-memoryOverhead": "2G", "driver-cores": 1, @@ -91,7 +92,7 @@ "max-executors": 200 } -OPENEO_EXTRACT_CDSE_JOB_OPTIONS: Dict[str, str] = { +OPENEO_EXTRACT_CDSE_JOB_OPTIONS: Dict[str, Union[str, int, float, List]] = { "driver-memory": "8G", "driver-memoryOverhead": "5G", "driver-cores": 1, @@ -103,7 +104,7 @@ "logging-threshold": "info" } -OPENEO_INFERENCE_CDSE_JOB_OPTIONS: Dict[str, str] = { +OPENEO_INFERENCE_CDSE_JOB_OPTIONS: Dict[str, Union[str, int, float, List]] = { "driver-memory": "1000m", "driver-memoryOverhead": "1000m", "driver-cores": 1, @@ -118,7 +119,7 @@ ] } -OPENEO_POINTEXTRACTION_CDSE_JOB_OPTIONS: Dict[str, str] = { +OPENEO_POINTEXTRACTION_CDSE_JOB_OPTIONS: Dict[str, Union[str, int, float, List]] = { "driver-memory": "4G", "driver-memoryOverhead": "2G", "driver-cores": 1, @@ -130,7 +131,7 @@ "logging-threshold": "info" } -OPENEO_CUBEEXTRACTION_CDSE_JOB_OPTIONS: Dict[str, str] = { +OPENEO_CUBEEXTRACTION_CDSE_JOB_OPTIONS: Dict[str, Union[str, int, float, List]] = { "driver-memory": "4G", "driver-memoryOverhead": "4G", "driver-cores": 1, @@ -168,7 +169,7 @@ 'S1_collection': "SENTINEL1_GRD" } -def _get_default_job_options() -> Dict[str, str]: +def _get_default_job_options() -> Dict[str, Union[str, int, float, List]]: """ Retrieves the default job options for OpenEO extract operations. @@ -181,7 +182,7 @@ def _get_default_job_options() -> Dict[str, str]: """ return OPENEO_EXTRACT_JOB_OPTIONS.copy() -def get_job_options(provider: str = None, task: str = 'raw_extraction') -> Dict[str, str]: +def get_job_options(provider: str = None, task: str = 'raw_extraction') -> Dict[str, Union[str, int, float, List]]: """ Retrieve job options based on the specified provider and task. @@ -249,7 +250,8 @@ def get_collection_options(provider: str) -> Dict[str, str]: else: raise ValueError(f'Provider `{provider}` not known.') -def get_standard_processing_options(provider: str, task: str = 'raw_extraction') -> dict: +def get_standard_processing_options(provider: str, task: str = 'raw_extraction') \ + -> Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]]: """ Generate standard processing options based on the provider and specified task. @@ -288,6 +290,7 @@ def get_standard_processing_options(provider: str, task: str = 'raw_extraction') "skip_check_S1": SKIP_CHECK_S1, "skip_check_S2": SKIP_CHECK_S2, "apply_cloud_mask": APPLY_CLOUD_MASK, + "get_NVBT": False, # NVBT is only added for feature cubes without time domain "openeo_chunk_size": CHUNK_SIZE, } elif (task == 'feature_generation') or (task == 'vi_generation'): @@ -307,6 +310,7 @@ def get_standard_processing_options(provider: str, task: str = 'raw_extraction') "skip_check_S1": SKIP_CHECK_S1, "skip_check_S2": SKIP_CHECK_S2, "apply_cloud_mask": APPLY_CLOUD_MASK, + "get_NVBT": GET_NVBT, "optical_vi_list" : VI_LIST, "radar_vi_list" : RADAR_LIST, "S2_scaling" : S2_SCALING, @@ -314,6 +318,8 @@ def get_standard_processing_options(provider: str, task: str = 'raw_extraction') "append" : True, "openeo_chunk_size": CHUNK_SIZE, } + if task == 'vi_generation': + proc_opt['get_NVBT'] = False # NVBT is only added for feature cubes without time domain else: raise ValueError(f'Task `{task}` not known.') return proc_opt @@ -325,62 +331,50 @@ def get_advanced_options(provider: str, s1_orbitdirection: Optional[str] = S1_OR S1_temporal_reducer: str = S1_TEMPORAL_REDUCER, slc_masking: Optional[str] = MASKING_ALGO, S2_bands: List[str] = S2_BANDS, s2_tileid_list: Optional[List[str]] = S2_TILEID_LIST, skip_check_S1: bool = SKIP_CHECK_S1, skip_check_S2: bool = SKIP_CHECK_S2, - apply_cloud_mask: bool = APPLY_CLOUD_MASK, S2_max_cloud_cover: int = S2_MAX_CLOUD_COVER, + apply_cloud_mask: bool = APPLY_CLOUD_MASK, get_NVBT: bool = GET_NVBT, + S2_max_cloud_cover: int = S2_MAX_CLOUD_COVER, optical_vi_list: List[str] = VI_LIST, radar_vi_list: List[str] = RADAR_LIST, S2_scaling: List[int | float] = S2_SCALING, S1_db_rescale: bool = True, - append: bool = True) -> Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]: + append: bool = True) \ + -> Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]]: """ - Generate a dictionary of advanced options for processing satellite imagery. - - This function validates input parameters and prepares a dictionary of options - required for processing satellite imagery data. Validation is applied for - various parameters to ensure that the provided values conform to the expected - formats or constraints. The function checks for valid values for key options - like satellite orbit direction, target coordinate reference system (CRS), - resolution, and others before constructing the dictionary. - - :param provider: The provider of satellite data. - :param s1_orbitdirection: Direction of Sentinel-1 pass, can be 'ASCENDING', - 'DESCENDING' or None. - :param target_crs: An integer representing the target coordinate reference - system (CRS). - :param resolution: The spatial resolution of the output in integer or float - format. - :param ts_interpolation: Boolean indicating whether to apply linear time-series - interpolation. - :param ts_interval: Interval for time series aggregation (temporal binning). Accepted values - are 'day', 'week', 'dekad', 'month', 'season', 'year', or - None. - :param S2_temporal_reducer: Method for reducing temporal data for Sentinel-2. - Valid options include 'median', 'mean', 'max', - 'min', 'first', 'last', 'product', 'sd', 'sum', - or 'variance'. - :param S1_temporal_reducer: Method for reducing temporal data for Sentinel-1. - Valid options include 'median', 'mean', 'max', - 'min', 'first', 'last', 'product', 'sd', 'sum', - or 'variance'. - :param slc_masking: Masking approach to be applied. Valid options are - 'mask_scl_dilation', 'satio', or None. - :param S2_bands: List of selected Sentinel-2 reflectance bands. - :param s2_tileid_list: Optional list of Sentinel-2 tiles for processing. Can - be None if tiles are not specified. - :param skip_check_S1: Boolean indicating whether to skip validation checks - for Sentinel-1 data. - :param skip_check_S2: Boolean indicating whether to skip validation checks - for Sentinel-2 data. - :param apply_cloud_mask: Boolean indicating whether to apply cloud masking or to add it as own band. - :param S2_max_cloud_cover: Maximum allowable cloud cover percentage for - Sentinel-2 data. Acceptable values are integers - between 0 and 100. - :param optical_vi_list: List of selected vegetation indices for optical data. - :param radar_vi_list: List of selected vegetation indices for radar data. - :param S2_scaling: List of scaling factors to be applied to Sentinel-2 data. - :param S1_db_rescale: Boolean controlling whether Sentinel-1 data must be - rescaled in decibels. - :param append: Boolean indicating whether to append the VI's to the reflectance cube or to replace them. - :param openeo_chunk_size: Chunk size for OpenEO processing. - - :return: A dictionary containing all validated options as key-value pairs. + Generates a comprehensive dictionary of advanced processing options for satellite data workflows. + This function validates the provided parameters and ensures that the inputs comply with the + accepted formats and values. It returns a dictionary containing key-value pairs suitable for + configuring advanced processing tasks. + + :param provider: (str) The provider for the data processing task. + :param s1_orbitdirection: (Optional[str]) The direction of Sentinel-1 orbit. Acceptable values + are 'ASCENDING', 'DESCENDING', or None. + :param target_crs: (Optional[int]) The target coordinate reference system (CRS) as an integer value. + :param resolution: (int | float) The spatial resolution of the data. + :param ts_interpolation: (bool) Whether to enable time-series interpolation. + :param ts_interval: (Optional[str]) The time-series interval. Acceptable values are 'day', + 'week', 'dekad', 'month', 'season', 'year', or None. + :param S2_temporal_reducer: (str) Temporal reducer function for Sentinel-2 data. Supported + values include 'median', 'mean', 'max', 'min', 'first', 'last', 'product', 'sd', 'sum', + and 'variance'. + :param openeo_chunk_size: (int) The OpenEO chunk size for processing data. + :param S1_temporal_reducer: (str) Temporal reducer function for Sentinel-1 data. Supported + values include 'median', 'mean', 'max', 'min', 'first', 'last', 'product', 'sd', 'sum', + and 'variance'. + :param slc_masking: (Optional[str]) Masking algorithm for Sentinel-1 SLC data. Acceptable + values are 'mask_scl_dilation', 'satio', or None. + :param S2_bands: (List[str]) List of Sentinel-2 reflectance bands to include. + :param s2_tileid_list: (Optional[List[str]]) List of specific Sentinel-2 tile IDs to process, or None. + :param skip_check_S1: (bool) Toggles whether to skip quality checks for Sentinel-1 data. + :param skip_check_S2: (bool) Toggles whether to skip quality checks for Sentinel-2 data. + :param apply_cloud_mask: (bool) Whether to apply cloud masking to the data. + :param get_NVBT: (bool) Whether to generate the input data quality indicator (NVBT) for the data. + :param S2_max_cloud_cover: (int) Maximum allowed cloud cover percentage for Sentinel-2 + data. Must be between 0 and 100. + :param optical_vi_list: (List[str]) List of optical vegetation indices to include in the workflow. + :param radar_vi_list: (List[str]) List of radar vegetation indices to include in the workflow. + :param S2_scaling: (List[int | float]) Scaling factors for Sentinel-2 data processing. + :param S1_db_rescale: (bool) Whether to apply dB rescaling to Sentinel-1 data. + :param append: (bool) Whether to append additional configuration options. + :return: Dictionary containing the validated processing options. Each key corresponds to a + configurable parameter, and the values reflect the input parameter values or defaults. """ if s1_orbitdirection not in ['ASCENDING', 'DESCENDING', None]: @@ -448,6 +442,9 @@ def get_advanced_options(provider: str, s1_orbitdirection: Optional[str] = S1_OR if type(apply_cloud_mask) != bool: raise ValueError(f'parameter for apply_cloud_mask must be an boolean.') + if type(get_NVBT) != bool: + raise ValueError(f'parameter for get_NVBT must be an boolean.') + proc_opt = { "provider": provider, "s1_orbitdirection": s1_orbitdirection, @@ -464,6 +461,7 @@ def get_advanced_options(provider: str, s1_orbitdirection: Optional[str] = S1_OR "skip_check_S1": skip_check_S1, "skip_check_S2": skip_check_S2, "apply_cloud_mask": apply_cloud_mask, + "get_NVBT": get_NVBT, "optical_vi_list": optical_vi_list, "radar_vi_list": radar_vi_list, "S2_scaling": S2_scaling, @@ -504,9 +502,9 @@ def generate_storage_options(workspace_export: bool = False, #just some checks to avoid missing parameters if workspace_export and not S3_prefix: - raise print("You want to export the openEO results to S3, please specify the S3_prefix parameter.") + raise ValueError("You want to export the openEO results to S3, please specify the S3_prefix parameter.") if workspace_export and not storage: - raise print("A storage object has to be defined to specify the export_workspace name " + raise ValueError("A storage object has to be defined to specify the export_workspace name " "and/or allow local_file_copy.") return storage_options diff --git a/src/eo_processing/openeo/preprocessing.py b/src/eo_processing/openeo/preprocessing.py index 8195331..f7160d7 100644 --- a/src/eo_processing/openeo/preprocessing.py +++ b/src/eo_processing/openeo/preprocessing.py @@ -8,7 +8,8 @@ catalogue_check_CDSE_S1, catalogue_check_CDSE_S2) from eo_processing.config.settings import S2_BANDS, PLANET_BANDS, CHUNK_SIZE -from typing import Optional, Dict, Union, List, TYPE_CHECKING +from typing import Optional, Dict, Union, List, TYPE_CHECKING, Any, Tuple + if TYPE_CHECKING: from eo_processing.config.data_formats import openEO_bbox_format @@ -16,7 +17,7 @@ def ts_datacube_extraction( connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, S2_collection: str ='SENTINEL2_L2A', S1_collection: Optional[str] ='SENTINEL1_GRD', - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]]) -> DataCube: """ Warper to extract a full data cube of preprocessed data :param connection: active openEO connection object @@ -29,11 +30,15 @@ def ts_datacube_extraction( resolution, ts_interval, time_interpolation, SLC_masking_algo, s1_orbitdirection, S2_bands) :return: DataCube """ + # Note: for cubes with a time dimension, no S1_NVBT or S2_NVBT band can be attached + get_NVBT:bool = processing_options.get("get_NVBT", False) + if get_NVBT: + processing_options["get_NVBT"] = False + # get the Sentinel-2 datacube as starting point bands = extract_S2_datacube(connection, bbox, start, end, - S2_collection=S2_collection, - **processing_options) - + S2_collection=S2_collection, + **processing_options) # add the Sentinel-1 data if S1_collection is not None: bands = bands.merge_cubes(extract_S1_datacube(connection, bbox, start, end, @@ -49,7 +54,8 @@ def ts_datacube_extraction( def extract_S1_datacube( connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, S1_collection: str = 'SENTINEL1_GRD', - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]]) \ + -> Union[DataCube, Tuple[DataCube, Optional[DataCube]]]: """ extract the Sentinel-1 data for requested time period and preprocess the data :param connection: active openEO connection object @@ -71,7 +77,6 @@ def extract_S1_datacube( if processing_options.get('skip_check_S1', False): catalogue_check = False - isCreo = "creo" in processing_options.get("provider", "").lower() orbit_direction: Optional[str] = processing_options.get('s1_orbitdirection', None) target_crs: Optional[str] = processing_options.get("target_crs", None) target_res: float = processing_options.get("resolution", 10.) @@ -79,7 +84,9 @@ def extract_S1_datacube( ts_reducer: str = processing_options.get("S1_temporal_reducer", "mean") ts_interpolation: bool = processing_options.get("time_interpolation", False) chunk_size: int = processing_options.get("openeo_chunk_size", CHUNK_SIZE) + get_NVBT: bool = processing_options.get("get_NVBT", False) + # set the correct DEM for SAR backscatter correction based on platform if ("creo" in processing_options.get("provider", "").lower()) or \ (processing_options.get("provider", "").lower() == "terrascope") or \ (processing_options.get("provider", "").lower() == "development") or \ @@ -102,7 +109,7 @@ def extract_S1_datacube( properties = {} # fix no-VH-data issue - properties.update({"polarisation": lambda pol: pol == "VV&VH"}) + properties.update({"sar:polarizations": lambda pol: pol == ["VV", "VH"]}) # Load collection bands = connection.load_collection(S1_collection, @@ -118,10 +125,10 @@ def extract_S1_datacube( bands = bands.sar_backscatter( coefficient='sigma0-ellipsoid', local_incidence_angle=False, - # DO NOT USE MAPZEN + # DO NOT USE MAPZEN, removed "otb_memory": 1024 from options on advice of JD, elevation_model='COPERNICUS_30' if flag_DEM else None, options={"implementation_version": "2", - "tile_size": chunk_size, "otb_memory": 1024, "debug": False, + "tile_size": chunk_size, "debug": False, "elev_geoid": "/opt/openeo-vito-aux-data/egm96.tif"} ) check_flag = True @@ -138,11 +145,19 @@ def extract_S1_datacube( if ts_interval is not None: bands = bands.aggregate_temporal_period(period=ts_interval, reducer=ts_reducer) + # here we have to add the calculation of the NVBT (Number of Valid Binned Timesteps) + # Note: we want to create the NVBT here before any possible linear interpolation is done + # therefore, the integration has to be this ugly and not only in the final warper functions + if get_NVBT: + nvbt_band = bands.filter_bands(bands.metadata.band_names[0]).count_time().rename_labels("bands", + ["S1-NVBT".lower()]) + nvbt_band = nvbt_band.apply(lambda x: if_(x.is_nodata(), 0, x)) + # Linearly interpolate missing values if wished if ts_interpolation: bands = bands.apply_dimension(dimension="t", process="array_interpolate_linear") - # Scale to Uint16 range + # Scale to Uint16 range from dB to natural values if check_flag: # for CREO, rescaling also replaces nodata introduced by orfeo # with a low value @@ -165,12 +180,17 @@ def extract_S1_datacube( # Force a linear scale removing values not expected bands = bands.linear_scale_range(1, 65534, 1, 65534) - return bands + # the return is tricky since we have to be backwards compatible and still get NVBT calculation working + if get_NVBT: + return bands, nvbt_band + else: + return bands def extract_S2_datacube( connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, S2_collection: str='SENTINEL2_L2A', - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]])\ + -> Union[DataCube, Tuple[DataCube, Optional[DataCube]]]: """ extract the Sentinel-2 data for requested time period and preprocess the data :param connection: active openEO connection object @@ -179,7 +199,7 @@ def extract_S2_datacube( :param end: str, End date for requested input data (yyyy-mm-dd) :param S2_collection: (str, optional): Collection name for S2 data :param processing_options: (dict, optional), processing options for preprocessing routine (provider, target_crs, - resolution, ts_interval, time_interpolation, SLC_masking_algo, S2_bands) + resolution, ts_interval, time_interpolation, SLC_masking_algo, S2_bands, apply_cloud_mask, getNOBSperc) :return: DataCube """ # evaluate additional processing_options @@ -203,6 +223,7 @@ def extract_S2_datacube( max_cloud_max: int = processing_options.get("S2_max_cloud_cover", 95) chunk_size: int = processing_options.get("openeo_chunk_size", CHUNK_SIZE) s2_tileid_list: Optional[List[str]] = processing_options.get("s2_tileid_list", None) + get_NVBT: bool = processing_options.get("get_NVBT", False) # check if the masking parameter is valid if masking not in ['satio', 'mask_scl_dilation', None]: @@ -271,9 +292,10 @@ def extract_S2_datacube( ).rename_labels("bands", ["S2-CLOUD-MASK"]) if apply_mask: + # apply the cloud mask to the reference data bands = bands.mask(scl_dilated_mask) # here I do trust the automatic resampling of the mask else: - bands = bands.merge_cubes(scl_dilated_mask) + bands = bands.merge_cubes(scl_dilated_mask) #cubes are automatically resampled if needed elif masking == 'satio': # Apply satio-based mask mask = scl_mask_erode_dilate( @@ -287,20 +309,31 @@ def extract_S2_datacube( if apply_mask: bands = bands.mask(mask) # masks are automatically resampled/warped else: - bands = bands.merge_cubes(mask) + bands = bands.merge_cubes(mask) #cubes are automatically resampled if needed # time aggregation if wished if ts_interval is not None: bands = bands.aggregate_temporal_period(period=ts_interval, reducer=ts_reducer) + # here we have to add the calculation of the NVBT (Number of Valid Binned Timesteps) + # Note: we want to create the NVBT here before any possible linear interpolation is done + # therefore, the integration has to be this ugly and not only in the final warper functions + if get_NVBT: + nvbt_band = bands.filter_bands(S2_bands[0]).count_time().rename_labels("bands", ["S2-NVBT".lower()]) + nvbt_band = nvbt_band.apply(lambda x: if_(x.is_nodata(), 0, x)) + # Linearly interpolate missing values if wished if ts_interpolation: bands = bands.apply_dimension(dimension="t", process="array_interpolate_linear") # forcing 16bit --> UInt16 bands = bands.linear_scale_range(0, 65534, 0, 65534) - - return bands + + # the return is tricky since we have to be backwards compatible and still get NVBT calculation working + if get_NVBT: + return bands, nvbt_band + else: + return bands def extract_planet_datacube( connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, diff --git a/src/eo_processing/openeo/processing.py b/src/eo_processing/openeo/processing.py index 48da2f6..05a970e 100644 --- a/src/eo_processing/openeo/processing.py +++ b/src/eo_processing/openeo/processing.py @@ -1,6 +1,7 @@ from __future__ import annotations import itertools import openeo +import numpy as np from openeo.rest.datacube import DataCube from openeo.extra.spectral_indices import append_indices, compute_indices from openeo.processes import array_create, ProcessBuilder, array_concat, subtract @@ -11,14 +12,15 @@ from eo_processing.config.settings import VI_LIST, RADAR_LIST, S2_SCALING, \ PLANET_VI_LIST, PLANET_SCALING, CHUNK_SIZE -from typing import Optional, Dict, Union, List, Literal, TYPE_CHECKING +from typing import Optional, Dict, Union, List, TYPE_CHECKING if TYPE_CHECKING: from eo_processing.config.data_formats import openEO_bbox_format def optical_indices( input_cube: DataCube, collection: str ='SENTINEL2_L2A', - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]])\ + -> DataCube: """creates vegetation indices times series cube from given datacube of optical EO data. Currently Sentinel-2 and PlanetScope are supported @@ -29,12 +31,12 @@ def optical_indices( """ # evaluate additional processing_options if collection == 'SENTINEL2_L2A': - vi_list = processing_options.get("optical_vi_list", VI_LIST) + vi_list: List = processing_options.get("optical_vi_list", VI_LIST) input_scaling = processing_options.get("S2_scaling", S2_SCALING) append = processing_options.get("append", True) platform = 'Sentinel-2A' elif collection == 'PlanetScope': - vi_list = processing_options.get("optical_vi_list", PLANET_VI_LIST) + vi_list: List = processing_options.get("optical_vi_list", PLANET_VI_LIST) input_scaling = processing_options.get("planet_scaling", PLANET_SCALING) append = processing_options.get("append", True) platform = 'PlanetScope' @@ -52,12 +54,12 @@ def optical_indices( # TODO: convert the datacube back to int16 - using the output scaling functionality tested in one # of the example notebooks - return vi_cube def radar_indices( input_cube: DataCube, - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]])\ + -> DataCube: """creates radar indices times series cube from given datacube of radar EO data :param input_cube: openEO DataCube @@ -65,10 +67,9 @@ def radar_indices( :return: VI datacube merged of input_cube and vi results """ # evaluate additional processing_options - vi_list = processing_options.get("radar_vi_list", RADAR_LIST) + vi_list: List = processing_options.get("radar_vi_list", RADAR_LIST) db_rescaling = processing_options.get("S1_db_rescale", True) append = processing_options.get("append", True) - chunk_size = processing_options.get("openeo_chunk_size", CHUNK_SIZE) platform = "sentinel1" # convert input DataCube into float (db) @@ -80,20 +81,21 @@ def radar_indices( (20. * x[1].log(base=10)) - 83.]) ) - # calculate VI's if append: vi_cube = append_indices(datacube=input_cube, indices=vi_list, platform=platform) else: vi_cube = compute_indices(datacube=input_cube, indices=vi_list, platform=platform, append=False) - + # TODO: convert the datacube back to int16 - using the output scaling functionality tested in one + # of the example notebooks return vi_cube def generate_S1_indices( connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, S1_collection: str ='SENTINEL1_GRD', - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]])\ + -> DataCube: """ Warper to extract a full data cube of preprocessed data :param connection: active openEO connection object @@ -105,6 +107,11 @@ def generate_S1_indices( resolution, ts_interval, time_interpolation, s1_orbitdirection, radar_vi_list, S1_db_rescale, append) :return: DataCube """ + # Note: for cubes with a time dimension, no s1-nvbt band can be attached + get_NVBT: bool = processing_options.get("get_NVBT", False) + if get_NVBT: + processing_options["get_NVBT"] = False + # get the S1 input data pre-processed input_cube = extract_S1_datacube(connection, bbox, start, end, S1_collection=S1_collection, **processing_options) @@ -116,7 +123,8 @@ def generate_S1_indices( def generate_S2_indices( connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, S2_collection: str ='SENTINEL2_L2A', - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]])\ + -> DataCube: """ Warper to extract a full data cube of preprocessed data :param connection: active openEO connection object @@ -128,9 +136,14 @@ def generate_S2_indices( resolution, ts_interval, time_interpolation, SLC_masking_algo, optical_vi_list, S2_scaling, append, S2_bands) :return: DataCube """ - # get the S2 input data pre-processed - input_cube = extract_S2_datacube(connection, bbox, start, end, S2_collection=S2_collection, - **processing_options) + # Note: for cubes with a time dimension, no s2-nvbt band can be attached + get_NVBT: bool = processing_options.get("get_NVBT", False) + if get_NVBT: + processing_options["get_NVBT"] = False + + # get the Sentinel-2 datacube as starting point + input_cube = extract_S2_datacube(connection, bbox, start, end, S2_collection=S2_collection, **processing_options) + # call the VI generator result_cube = optical_indices(input_cube, collection=S2_collection, **processing_options) @@ -138,8 +151,9 @@ def generate_S2_indices( def generate_indices_master_cube( connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, - S2_collection: str ='SENTINEL2_L2A', S1_collection: str ='SENTINEL1_GRD', - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + S2_collection: str ='SENTINEL2_L2A', S1_collection: Optional[str] ='SENTINEL1_GRD', + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]])\ + -> DataCube: """ Warper to extract a full data cube of preprocessed data :param connection: active openEO connection object @@ -153,10 +167,11 @@ def generate_indices_master_cube( S2_scaling, append, S2_bands, radar_vi_list, S1_db_rescale) :return: DataCube """ - # get the S2 indices + # get S2 indices cube - no NVBT band needed indices_cube = generate_S2_indices(connection, bbox, start, end, S2_collection=S2_collection, **processing_options) - # merge the S1 indices + + # merge the S1 indices (the called warper makes sure no NVBT band is generated) if S1_collection is not None: indices_cube = indices_cube.merge_cubes(generate_S1_indices(connection, bbox, start, end, S1_collection=S1_collection, @@ -166,7 +181,8 @@ def generate_indices_master_cube( def generate_indices_planet_cube( connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, planet_collection: str = 'PlanetScope', - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]])\ + -> DataCube: """ Warper to extract a full data cube of preprocessed PlanetScope data :param connection: active openEO connection object @@ -225,26 +241,28 @@ def calculate_features_cube(input_data: DataCube, chunk_size: int = CHUNK_SIZE) features_cube = input_data.apply_dimension(dimension='t', process=_compute_features, target_dimension='bands', - context={"parallel": True, - "TileSize": chunk_size}) + context={"TileSize": chunk_size}) # adapt the band names new_band_names = [ - band + "_" + stat + band.lower() + "_" + stat.lower() for band in input_data.metadata.band_names for stat in ["p2", "p5", "p25", "median", "p75", "p95", "p98", "mean", "sd", "sum", "iqr", "iqr0595"] ] - #rename 'S2-CLOUD-MASK_sum' - new_band_names = ['valid_obs' if x == 'S2-CLOUD-MASK_sum' else x for x in new_band_names] features_cube = features_cube.rename_labels('bands', new_band_names) # remove some bands which make no sense :) # mainly from S2REP --> sd, sum, iqr + #bands_keep = [band for band in features_cube.metadata.band_names if + # band not in ['S2REP_sd', 'S2REP_sum', 'S2REP_iqr', 'S2REP_iqr0595' , 'VV_sum', 'VH_sum', 'VHVVD_sum', + # 'S2-CLOUD-MASK_p2', 'S2-CLOUD-MASK_p5', 'S2-CLOUD-MASK_p25', 'S2-CLOUD-MASK_median', + # 'S2-CLOUD-MASK_p75','S2-CLOUD-MASK_p95', 'S2-CLOUD-MASK_p98', 'S2-CLOUD-MASK_mean', + # 'S2-CLOUD-MASK_sd','S2-CLOUD-MASK_iqr','S2-CLOUD-MASK_iqr0595', 'S2-CLOUD-MASK_sum']] bands_keep = [band for band in features_cube.metadata.band_names if - band not in ['S2REP_sd', 'S2REP_sum', 'S2REP_iqr', 'S2REP_iqr0595' , 'VV_sum', 'VH_sum', 'VHVVD_sum', - 'S2-CLOUD-MASK_p2', 'S2-CLOUD-MASK_p5', 'S2-CLOUD-MASK_p25', 'S2-CLOUD-MASK_median', - 'S2-CLOUD-MASK_p75','S2-CLOUD-MASK_p95', 'S2-CLOUD-MASK_p98', 'S2-CLOUD-MASK_mean', - 'S2-CLOUD-MASK_sd','S2-CLOUD-MASK_iqr','S2-CLOUD-MASK_iqr0595']] + band not in ['s2rep_sd', 's2rep_sum', 's2rep_iqr', 's2rep_iqr0595' , 'vv_sum', 'vh_sum', 'vhvvd_sum', + 's2-could-mask_p2', 's2-could-mask_p5', 's2-could-mask_p25', 's2-could-mask_median', + 's2-could-mask_p75','s2-could-mask_p95', 's2-could-mask_p98', 's2-could-mask_mean', + 's2-could-mask_sd','s2-could-mask_iqr','s2-could-mask_iqr0595', 's2-could-mask_sum']] features_cube = features_cube.filter_bands(bands=bands_keep) @@ -253,7 +271,8 @@ def calculate_features_cube(input_data: DataCube, chunk_size: int = CHUNK_SIZE) def generate_S1_feature_cube( connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, S1_collection: str ='SENTINEL1_GRD', - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]])\ + -> DataCube: """ Warper to extract a full data cube of preprocessed data :param connection: active openEO connection object @@ -267,18 +286,31 @@ def generate_S1_feature_cube( """ chunk_size: int = processing_options.get("openeo_chunk_size", CHUNK_SIZE) - # get the reflectance and VI time series cube - input_data = generate_S1_indices(connection, bbox, start, end, S1_collection=S1_collection, - **processing_options) + get_NVBT: bool = processing_options.get("get_NVBT", False) + # feature cubes have no time dimension, so we can add the s1-nvbt band to the cube + if get_NVBT: + # get the sigma-naught and VI time series cube PLUS the S1-NVBT band + input_cube, nvbt_band = extract_S1_datacube(connection, bbox, start, end, S1_collection=S1_collection, + **processing_options) + # call the VI generator + indices_cube = radar_indices(input_cube, **processing_options) + else: + indices_cube = generate_S1_indices(connection, bbox, start, end, S1_collection=S1_collection, + **processing_options) # get features - features_cube = calculate_features_cube(input_data, chunk_size=chunk_size) + features_cube = calculate_features_cube(indices_cube, chunk_size=chunk_size) + + # add the S1-NVBT band to cube if needed + if get_NVBT: + features_cube = features_cube.merge_cubes(nvbt_band) return features_cube def generate_S2_feature_cube( connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, S2_collection: str ='SENTINEL2_L2A', - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]])\ + -> DataCube: """ Warper to extract a full data cube of preprocessed data :param connection: active openEO connection object @@ -291,19 +323,34 @@ def generate_S2_feature_cube( :return: DataCube with only features """ chunk_size: int = processing_options.get("openeo_chunk_size", CHUNK_SIZE) - # get the reflectance and VI time series cube - input_data = generate_S2_indices(connection, bbox, start, end, S2_collection=S2_collection, - **processing_options) + + get_NVBT: bool = processing_options.get("get_NVBT", False) + # feature cubes have no time dimension, so we can add the s2-nvbt band to the cube + if get_NVBT: + # get the reflectance and VI time series cube PLUS S2-NVBT band + input_cube, nvbt_band = extract_S2_datacube(connection, bbox, start, end, S2_collection=S2_collection, + **processing_options) + # call the VI generator + indices_cube = optical_indices(input_cube, collection=S2_collection, **processing_options) + else: + # get the reflectance and VI time series cube + indices_cube = generate_S2_indices(connection, bbox, start, end, S2_collection=S2_collection, + **processing_options) + # get features - features_cube = calculate_features_cube(input_data, chunk_size=chunk_size) + features_cube = calculate_features_cube(indices_cube, chunk_size=chunk_size) - return features_cube + # add the S2-NVBT band to the cube if needed + if get_NVBT: + features_cube = features_cube.merge_cubes(nvbt_band) + return features_cube def generate_planet_feature_cube( connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, planet_collection: str ='PlanetScope', - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]])\ + -> DataCube: """ Warper to extract a full data cube of preprocessed Planet data :param connection: active openEO connection object @@ -329,7 +376,8 @@ def generate_planet_feature_cube( def generate_master_feature_cube( connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, S2_collection: str ='SENTINEL2_L2A', S1_collection: str ='SENTINEL1_GRD', - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: Dict[str, Union[str, int, float, bool, List[str], List[Union[int, float]], None]])\ + -> DataCube: """ Warper to extract a full data cube of preprocessed data :param connection: active openEO connection object @@ -343,14 +391,14 @@ def generate_master_feature_cube( S2_scaling, append, S2_bands, radar_vi_list, S1_db_rescale) :return: DataCube with only features """ - chunk_size: int = processing_options.get("openeo_chunk_size", CHUNK_SIZE) - # get the reflectance and VI time series cube - input_data = generate_indices_master_cube(connection, bbox, start, end, S2_collection=S2_collection, - S1_collection=S1_collection, **processing_options) - # get features - features_cube = calculate_features_cube(input_data, chunk_size=chunk_size) - - return features_cube + # optimized pipeline were we run the feature generation per sensor and only merge in the end + feature_cube = generate_S2_feature_cube(connection, bbox, start, end, S2_collection=S2_collection, + **processing_options) + if S1_collection is not None: + feature_cube = feature_cube.merge_cubes(generate_S1_feature_cube(connection, bbox, start, end, + S1_collection=S1_collection, + **processing_options)) + return feature_cube def create_collections_list_from_bands(input_bands : List[str]): """ @@ -394,23 +442,23 @@ def create_collections_list_from_bands(input_bands : List[str]): return collections_list def generate_nonEO_feature_cube( - connection: openeo.Connection, bbox: Optional[openEO_bbox_format], start: str, end: str, - collections_list: List[str], + connection: openeo.Connection, bbox: openEO_bbox_format | None, start: str, end: str, + collections_list: list[str], base_cube : DataCube, - **processing_options: Dict[str, Union[str, bool, int | float, List[str], List[int | float]]]) -> DataCube: + **processing_options: dict[str, str | bool | (int | float) | list[str] | list[int | float]]) -> DataCube: """ Warper to generate the data cube of all nonEO data based on a collections list of the form [(collection, [band1, band2, ...])]""" - temporal_extent = [start, end] - temporal_extent = None chunk_size: int = processing_options.get("openeo_chunk_size", CHUNK_SIZE) - for collection, bands, reproj, year in collections_list: + for collection, bands, reproj, year, stac_url in collections_list: #first need to distinguish between STAC and collection #we assume that they will allways be an url type of link in contrary with a collections which should just be a name - if temporal_extent: - temporal_extent = [f"{year}-01-01T00:00:00Z", f"{year}-12-31T23:59:59Z"] - STAC_url = get_stac_collection_url(collection) + if not np.isnan(year): + temporal_extent = [f"{int(year)}-01-01T00:00:00Z", f"{int(year)}-12-31T23:59:59Z"] + else: + temporal_extent = None + STAC_url = get_stac_collection_url(collection, stac_url) isSTAC = STAC_url is not None #secondly we know there are some specific case of reprojection EG DEM should be bilinear iso near @@ -431,6 +479,9 @@ def generate_nonEO_feature_cube( temporal_extent=temporal_extent ) nonEO_feature_cube.result_node().update_arguments(featureflags={'tilesize': chunk_size}) + # TODO add all the collections that are allowed to be empty + if collection in ["DEM_aspec_30m"]: + nonEO_feature_cube.result_node().update_arguments(featureflags={'allow_empty_cube': True}) else: #if openeo the -v1 should be split off of the collection @@ -448,7 +499,7 @@ def generate_nonEO_feature_cube( # reduce the temporal domain since copernicus_30 collection is "special" and feature only are one time stamp nonEO_feature_cube = nonEO_feature_cube.reduce_dimension(dimension='t', reducer=lambda x: x.last(ignore_nodata=True)) - new_bands = [f"{collection}-{band}" for band in bands] + new_bands = [f"{collection.lower()}-{band.lower()}" for band in bands] # resample the cube to 10m and EPSG of corresponding 20x20km grid tile @@ -471,4 +522,3 @@ def generate_nonEO_feature_cube( base_cube = base_cube.merge_cubes(nonEO_feature_cube) return base_cube - diff --git a/src/eo_processing/openeo/udf_capture.py b/src/eo_processing/openeo/udf_capture.py new file mode 100644 index 0000000..b61cb1b --- /dev/null +++ b/src/eo_processing/openeo/udf_capture.py @@ -0,0 +1,134 @@ +"""Build context for UDF input captures stored in CDSE object storage.""" + +import hashlib +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import unquote, urlparse + +import boto3 +import openeo +import requests + +from eo_processing.utils.helper import getUDFpath + +_CAPTURE_URL_PATTERN = re.compile(r"presigned download:\s*(https?://\S+)") + + +def _build_s3_context( + connection: openeo.Connection, + region: str = "waw3-1", + role_session_name: str = "openeo-udf", + duration_hours: int = 12, +) -> dict[str, str]: + """Build temporary object-storage credentials from an openEO OIDC token.""" + if duration_hours <= 0: + raise ValueError("duration_hours must be positive") + + auth = connection.auth + try: + web_identity_token = auth.bearer.split("/", maxsplit=2)[2] if auth else None + except (AttributeError, IndexError) as error: + raise ValueError("Connection must use an OIDC bearer token") from error + if not web_identity_token: + raise ValueError("Connection must use an OIDC bearer token") + + sts_url = f"https://sts.{region}.openeo.v1.dataspace.copernicus.eu" + s3_url = sts_url.replace("sts", "s3", 1) + role_arn = f"arn:openeo:iam:::role/openeo-artifacts-{region}" + + response: dict[str, Any] = boto3.Session().client( + "sts", endpoint_url=sts_url + ).assume_role_with_web_identity( + RoleArn=role_arn, + RoleSessionName=role_session_name, + WebIdentityToken=web_identity_token, + DurationSeconds=duration_hours * 3600, + ) + credentials = response["Credentials"] + subject = response["SubjectFromWebIdentityToken"] + subject_hash = hashlib.sha1(subject.encode()).hexdigest() + prefix = f"{subject_hash}/{datetime.now(timezone.utc):%Y/%m/%d}/" + + return { + "bucket": f"openeo-artifacts-{region}", + "prefix": prefix, + "endpoint": s3_url, + "token": credentials["SessionToken"], + "access_key_id": credentials["AccessKeyId"], + "secret_access_key": credentials["SecretAccessKey"], + } + + +def get_capture_udf( + connection: openeo.Connection, + *, + output_format: str = "netcdf", + region: str = "waw3-1", + role_session_name: str = "openeo-udf", + duration_hours: int = 12, +) -> openeo.UDF: + """Create an input-capture UDF with temporary object-storage credentials as context.""" + output_format = output_format.lower() + if output_format not in {"netcdf", "gtiff"}: + raise ValueError("output_format must be 'netcdf' or 'gtiff'") + + return openeo.UDF.from_file( + getUDFpath("udf_save_input.py"), + context={ + **_build_s3_context( + connection, + region=region, + role_session_name=role_session_name, + duration_hours=duration_hours, + ), + "output_format": output_format, + }, + runtime="Python", + version="3.11", + ) + + +def download_captured_udf_inputs( + connection: openeo.Connection, + job: openeo.BatchJob | str, + folder: str | Path, + *, + limit: int | None = None, +) -> list[Path]: + """Download UDF inputs whose presigned URLs were emitted in batch-job logs. + + ``job`` can be an openEO batch job or its ID. URLs are read from the + ``presigned download:`` messages emitted by ``udf_save_input.py``. + """ + if limit is not None and limit < 0: + raise ValueError("limit must be non-negative") + + job = connection.job(job) if isinstance(job, str) else job + urls: list[str] = [] + for entry in job.logs(level="INFO"): + message = entry.get("message", "") + match = _CAPTURE_URL_PATTERN.search(message) + if match and match.group(1) not in urls: + urls.append(match.group(1)) + + if limit is not None: + urls = urls[:limit] + + output_folder = Path(folder) + output_folder.mkdir(parents=True, exist_ok=True) + downloads: list[Path] = [] + for url in urls: + filename = Path(unquote(urlparse(url).path)).name + if not filename: + raise ValueError(f"Capture URL has no filename: {url}") + destination = output_folder / filename + response = requests.get(url, stream=True, timeout=60) + response.raise_for_status() + with destination.open("wb") as file: + for chunk in response.iter_content(chunk_size=1024 * 1024): + file.write(chunk) + downloads.append(destination) + + return downloads \ No newline at end of file diff --git a/src/eo_processing/resources/ECDC_global_120x120km_grid_v1.gpkg b/src/eo_processing/resources/ECDC_global_120x120km_grid_v1.gpkg new file mode 100644 index 0000000..45f7762 Binary files /dev/null and b/src/eo_processing/resources/ECDC_global_120x120km_grid_v1.gpkg differ diff --git a/src/eo_processing/resources/udf_save_input.py b/src/eo_processing/resources/udf_save_input.py new file mode 100644 index 0000000..f6952ac --- /dev/null +++ b/src/eo_processing/resources/udf_save_input.py @@ -0,0 +1,109 @@ +"""Save an openEO UDF input chunk to temporary object storage.""" + +import logging +from datetime import datetime +from uuid import uuid4 + +import boto3 +import numpy as np +import xarray as xr +from openeo.udf import inspect +from rasterio.io import MemoryFile + +logger = logging.getLogger(__name__) + + +def _s3_client(context: dict): + return boto3.client( + "s3", + aws_access_key_id=context.get("access_key_id"), + aws_secret_access_key=context.get("secret_access_key"), + aws_session_token=context.get("token"), + endpoint_url=context.get("endpoint"), + ) + + +def _build_object_key(cube: xr.DataArray, context: dict, extension: str) -> str: + title = context.get("title") or context.get("name") or "input" + stamp = datetime.now().strftime("%Y%m%dT%H%M%S") + short_id = uuid4().hex[:6] + + coord_tag: str | None = None + if "x" in cube.coords and "y" in cube.coords: + try: + coord_tag = f"x{int(cube.x.values[0])}_y{int(cube.y.values[0])}" + except (TypeError, ValueError, IndexError): + pass + + parts = [title] + if coord_tag: + parts.append(coord_tag) + parts.extend([stamp, short_id]) + + prefix = context.get("prefix", "").strip("/") + filename = "_".join(parts) + extension + return f"{prefix}/{filename}" if prefix else filename + + +def _serialize_capture(cube: xr.DataArray, output_format: str) -> tuple[bytes, str, str]: + if output_format == "netcdf": + dataset = ( + cube.to_dataset(dim="bands") + if "bands" in cube.dims + else cube.to_dataset(name=cube.name or "data") + ) + return bytes(dataset.to_netcdf()), ".nc", "application/netcdf" + + if output_format == "gtiff": + if "t" in cube.dims: + return _serialize_capture(cube, "netcdf") + if "y" not in cube.dims or "x" not in cube.dims: + raise ValueError("GeoTIFF capture requires y and x dimensions") + + data = np.asarray(cube.values) + if data.ndim == 2: + data = data[np.newaxis, :, :] + elif data.ndim != 3: + raise ValueError("GeoTIFF capture requires a 2D or 3D spatial input cube") + + with MemoryFile() as memory_file: + with memory_file.open( + driver="GTiff", + height=data.shape[-2], + width=data.shape[-1], + count=data.shape[0], + dtype=data.dtype, + ) as dataset: + dataset.write(data) + return memory_file.read(), ".tif", "image/tiff" + + raise ValueError("output_format must be 'netcdf' or 'gtiff'") + + +def apply_datacube(cube: xr.DataArray, context: dict) -> xr.DataArray: + """Upload UDF input, then return it unchanged.""" + bucket = context.get("bucket") + if not bucket: + raise ValueError("Context must contain bucket") + + desired_dims = [dimension for dimension in ["t", "bands", "y", "x"] if dimension in cube.dims] + capture_cube = cube.transpose(*desired_dims).load() + output_format = context.get("output_format", "netcdf").lower() + payload, extension, content_type = _serialize_capture(capture_cube, output_format) + key = _build_object_key(capture_cube, context, extension) + client = _s3_client(context) + client.put_object( + Bucket=bucket, + Key=key, + Body=payload, + ContentType=content_type, + ) + + presigned_url = client.generate_presigned_url( + "get_object", + Params={"Bucket": bucket, "Key": key}, + ExpiresIn=int(context.get("presign_expires", 12 * 3600)), + ) + inspect(message=f"uploaded to s3://{bucket}/{key}; presigned download: {presigned_url}") + logger.info("uploaded to s3://%s/%s; presigned download: %s", bucket, key, presigned_url) + return cube \ No newline at end of file diff --git a/src/eo_processing/utils/alphaearth_utils.py b/src/eo_processing/utils/alphaearth_utils.py new file mode 100644 index 0000000..cb10ca8 --- /dev/null +++ b/src/eo_processing/utils/alphaearth_utils.py @@ -0,0 +1,490 @@ +"""Module to download alphaEarth embeddings for a given AOI and upload them to S3. + +Some snippets extracted from Marcel's code. + +This module provides functions to: +1. Identify intersecting grids from an AOI and a global grid. +2. Download the corresponding alphaEarth embeddings from S3 or HTTP. +3. Convert the downloaded VRT files to Cloud Optimized GeoTIFF (COG) +4. Upload the processed files to S3 using a specified storage utility. + +Example usage: + + # get the intersecting grids and filenames + df = get_embedding_filenames( + "/vitodata/nca/weed/change/AOI/AOI_cze-change_UTM20k_grid.shp", + "/data/mepvm/alphaearth/aef_source-coop_2017-2025_index.gpkg", + '/vitodata/nca/weed/grid/global_terrestrial_UTM20k_grid_v2.gpkg', # not needed if input is already in this grid + years=[2022, 2023], + ) + # download files to local directory + downloaded_files = download_embeddings( + df["path"].tolist(), "/vitodata/nca/test/Manu/alphaearth_embeddings/" + ) + # edit the vrt and then convert to cog + vrt_paths = [Path(f).with_suffix(".vrt") for f in downloaded_files] + cog_paths = [] + for path_vrt in vrt_paths: + path_vrt: Path = Path(path_vrt) + patch_vrt_relative_path(path_vrt) + path_tif = translate_to_cog(path_vrt) + cog_paths.append(path_tif) + + # Upload to S3 + files_df = build_files_dataframe(cog_paths) + storage = WEED_storage(...) # Initialize with appropriate parameters) + upload_files(files_df, storage, S3_ROOT) + print('All done.') + +""" + +import logging +import re +import subprocess +from pathlib import Path +from typing import List, Optional, Tuple + +import geopandas as gpd +import pandas as pd +from eo_processing.utils.storage import WEED_storage + + +# Configure logging +logging.basicConfig( + level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s" +) +logger = logging.getLogger(__name__) + +try: + import boto3 + from botocore import UNSIGNED + from botocore.client import Config + from botocore.exceptions import ClientError + + boto3_available = True +except ImportError: + logger.warning( + "boto3 not available, falling back to HTTP protocol for downloading." + ) + boto3_available = False + + +# download settings +S3_BASE_URL = "s3://us-west-2.opendata.source.coop/" +ENDPOINT_URL = "https://data.source.coop" +# COG settings +GDAL_COG_OPTIONS = [ + "-of COG", + "-co INTERLEAVE=BAND", + "-co COMPRESS=ZSTD", + "-co LEVEL=15", + "-co NUM_THREADS=ALL_CPUS", + "-co PREDICTOR=YES", + "-co OVERVIEW_RESAMPLING=NEAREST", + "-co BIGTIFF=YES", + "-co OVERVIEW_COUNT=6", +] + + +class S3Downloader: + def __init__(self): + self.s3_client = boto3.client( + "s3", + endpoint_url=ENDPOINT_URL, + config=Config(signature_version=UNSIGNED), + ) + + def s3_object_exists(self, bucket, key): + """ + Check if an S3 object exists using unsigned requests. + + Args: + bucket (str): S3 bucket name. + key (str): S3 object key. + endpoint_url (str): Custom S3 endpoint URL. + + Returns: + bool: True if object exists, False otherwise. + """ + try: + logger.debug(f"Checking existence of s3://{bucket}/{key}") + self.s3_client.head_object(Bucket=bucket, Key=key) + logger.info(f"Object exists: s3://{bucket}/{key}") + return True + except ClientError as e: + if e.response["Error"]["Code"] == "404": + logger.warning(f"Object does not exist: s3://{bucket}/{key}") + return False + else: + logger.error(f"Error checking object existence: {e}") + raise + + def get_bucket_and_key(self, s3_url: str) -> Tuple[str, str]: + """ + Parse an S3 URL into its bucket name and object key. + + Args: + s3_url (str): The full S3 URL to parse. + """ + split_str = s3_url.replace(S3_BASE_URL, "").split("/", 1) + return split_str[0], split_str[1] + + def download_s3_file(self, s3_url, output_dir): + """ + Download a single file from S3 using unsigned requests. + + Args: + bucket (str): S3 bucket name. + key (str): S3 object key. + endpoint_url (str): Custom S3 endpoint URL. + output_dir (str or Path): Local directory to save the object. + """ + bucket, key = self.get_bucket_and_key(s3_url) + key1 = key.replace(".tiff", ".vrt") + if not self.s3_object_exists(bucket, key): + logger.error(f"Object does not exist: s3://{bucket}/{key}") + return False, None + if not self.s3_object_exists(bucket, key1): + logger.warning(f"Object does not exist: s3://{bucket}/{key1}") + + try: + output_path = Path(output_dir) / Path(key) + output_path.parent.mkdir(parents=True, exist_ok=True) + self.s3_client.download_file(bucket, key, str(output_path)) + logger.info(f"Download complete: {output_path}") + output_path1 = Path(output_dir) / Path(key1) + self.s3_client.download_file(bucket, key1, str(output_path1)) + logger.info(f"Download complete: {output_path1}") + return True, str(output_path) + except Exception as e: + logger.error(f"Failed to download s3://{bucket}/{key}: {e}") + return False, None + + +def http_download( + filenames: List[str], output_dir: Path, overwrite: bool = False +) -> List[str]: + """ + Download files using HTTP protocol. + + Args: + filenames (List[str]): List of file URLs to download. + output_dir (Path): Directory to save downloaded files. + overwrite (bool): If True, overwrite existing files. + + Returns: + List[str]: List of paths to downloaded files. + """ + import requests + + new_paths = [] + for fl in filenames: + fl_path = Path(fl.replace(S3_BASE_URL, "")) + final_dir = output_dir / fl_path.parent + final_dir.mkdir(parents=True, exist_ok=True) + url = fl.replace(S3_BASE_URL, "https://data.source.coop/") + out_path = final_dir / fl_path.name + if out_path.exists() and not overwrite: + logger.info(f"File {out_path} exists, skipping download.") + new_paths.append(str(out_path)) + continue + logger.info(f"Downloading {url} to {final_dir}") + with requests.get(url, stream=True) as r: + r.raise_for_status() + with open(out_path, "wb") as f: + for chunk in r.iter_content(chunk_size=1024 * 1024): + if chunk: + f.write(chunk) + new_paths.append(str(out_path)) + return new_paths + + +def _ensure_crs(df: gpd.GeoDataFrame, crs: str = "EPSG:4326") -> gpd.GeoDataFrame: + """ + Ensure the GeoDataFrame has the specified CRS. + + Args: + df (gpd.GeoDataFrame): Input GeoDataFrame. + crs (str): Target CRS. + + Returns: + gpd.GeoDataFrame: GeoDataFrame in the target CRS. + """ + if df.crs is None or df.crs != crs: + return df.to_crs(crs) + return df + + +def get_intersecting_grids( + input_gdf: gpd.GeoDataFrame, grid_file: str +) -> gpd.GeoDataFrame: + """ + Intersect the input_gdf with the grid_file and return the intersecting grids. + + Args: + input_gdf (gpd.GeoDataFrame): Input geometries. + grid_file (str): Path to grid file. + + Returns: + gpd.GeoDataFrame: Intersecting grid geometries. + """ + grid_gdf = gpd.read_file(grid_file) + input_union = input_gdf.union_all() + possible_matches_index = list(grid_gdf.sindex.intersection(input_union.bounds)) + possible_matches = grid_gdf.iloc[possible_matches_index] + intersecting = possible_matches[possible_matches.intersects(input_union)] + return intersecting + + +def get_embedding_filenames( + input_shape_filename: str, + alphaearth_metadata_file: str, + format_grid_file: Optional[str] = None, + years: Optional[List[int]] = None, +) -> gpd.GeoDataFrame: + """ + Get the intersecting grids from the input shape file and return the corresponding alphaearth embeddings. + + Args: + input_shape_filename (str): Path to the input shape file. Can be .shp, .gpkg, .geojson, or .parquet. + alphaearth_metadata_file (str): Path to the alphaearth metadata file. + format_grid_file (Optional[str]): Path to the global grid file. If provided, filter grids. + years (Optional[List[int]]): List of years to filter the embeddings. + + Returns: + gpd.GeoDataFrame: Intersecting grids and their corresponding alphaearth embeddings paths. + """ + input_file = Path(input_shape_filename) + if not input_file.exists(): + logger.error(f"Input shape file {input_shape_filename} does not exist.") + raise FileNotFoundError( + f"Input shape file {input_shape_filename} does not exist." + ) + + if input_file.suffix in [".shp", ".gpkg", ".geojson"]: + input_gdf = gpd.read_file(input_file) + elif input_file.suffix == ".parquet": + input_gdf = gpd.read_parquet(input_file) + else: + logger.error( + f"Input shape file {input_shape_filename} has unsupported file type {input_file.suffix}. " + "Supported types are .shp, .gpkg, .geojson, .parquet" + ) + raise ValueError( + f"Input shape file {input_shape_filename} has unsupported file type {input_file.suffix}. " + "Supported types are .shp, .gpkg, .geojson, .parquet" + ) + input_gdf = _ensure_crs(input_gdf) + + if format_grid_file is not None: + if not Path(format_grid_file).exists(): + logger.error(f"Global grid file {format_grid_file} does not exist.") + raise FileNotFoundError( + f"Global grid file {format_grid_file} does not exist." + ) + grids_20km = get_intersecting_grids(input_gdf, format_grid_file) + else: + grids_20km = input_gdf + + if not Path(alphaearth_metadata_file).exists(): + logger.error( + f"AlphaEarth metadata file {alphaearth_metadata_file} does not exist." + ) + raise FileNotFoundError( + f"AlphaEarth metadata file {alphaearth_metadata_file} does not exist." + ) + alphaearth_grids = get_intersecting_grids(grids_20km, alphaearth_metadata_file) + if years: + alphaearth_grids = alphaearth_grids[alphaearth_grids["year"].isin(years)] + else: + logger.info("No years specified, returning all intersecting grids.") + return alphaearth_grids + + +def download_embeddings( + filenames: List[str], output_path: str, overwrite: bool = False +) -> List[str]: + """ + Download the embeddings for the intersecting grids from sourcecoop or HTTP and save them to the output directory. + + Args: + filenames (List[str]): List of S3 paths to the embeddings. + output_path (str): Path to the output directory where the embeddings will be downloaded. + overwrite (bool): If True, download and overwrite files even if they exist. + + Returns: + List[str]: List of paths to downloaded files. + """ + output_dir = Path(output_path) + output_dir.mkdir(parents=True, exist_ok=True) + logger.info(f"Downloading {len(filenames)} files to {output_dir}") + new_paths = [] + successful_downloads = 0 + if boto3_available: + logger.info("Using S3 for downloading files.") + client = S3Downloader() + for fl in filenames: + success, new_path = client.download_s3_file(fl, output_dir) + if success: + new_paths.append(new_path) + successful_downloads += 1 + logger.info( + f"Downloaded {len(new_paths)} files to {output_dir} out of {len(filenames)}" + ) + else: + logger.info("Using HTTP protocol for downloading files.") + new_paths = http_download(filenames, output_dir, overwrite=overwrite) + logger.info(f"Downloaded {len(new_paths)} files to {output_dir}") + return new_paths + + +def patch_vrt_relative_path(path_vrt: Path) -> None: + """ + Modifies the VRT file to set the 'relativeToVRT' attribute for the source dataset to 1. + + This function reads the content of the specified VRT file, locates the SourceDataset + element whose 'relativeToVRT' attribute is set to 0 and whose filename matches the + generated filename, updates its 'relativeToVRT' attribute to 1, and writes the modified + content back to the same file. + + :param path_vrt: A Path object representing the path to the VRT file to be modified. + """ + filename = path_vrt.stem + ".tiff" + vrt_content = path_vrt.read_text(encoding="utf-8") + vrt_content = re.sub( + r'[^<]*' + + re.escape(filename) + + r"", + f'{filename}', + vrt_content, + ) + path_vrt.write_text(vrt_content, encoding="utf-8") + + +def translate_to_cog(path_vrt: Path, version): + """Translates a VRT (Virtual Dataset) file to a Cloud Optimized GeoTIFF (COG) format using GDAL. + + This function converts a given VRT file into a COG-compatible GeoTIFF file. If the output + file already exists, the function will skip the translation process and return the existing + file path. + + Args: + path_vrt (Path): The path to the input VRT file. + version (str): Version string to append to the output filename. + + + urn: The path to the generated Cloud Optimized GeoTIFF file. :return: The path to the generated Cloud Optimized GeoTIFF file. + """ + path_out = path_vrt.parent / f"{path_vrt.stem}_{version}.tif" + + if path_out.exists(): + print(f"File {path_out} already exists, skipping.") + return path_out + + gdal_cmd = " ".join( + ["gdal_translate"] + GDAL_COG_OPTIONS + [str(path_vrt), str(path_out)] + ) + try: + subprocess.check_call(gdal_cmd, shell=True) + except subprocess.CalledProcessError as e: + raise OSError(f"Could not translate file: {e}") from e + return path_out + + +def build_files_dataframe(files: list[Path]) -> pd.DataFrame: + """ + Builds a DataFrame from a list of file paths, extracting relevant metadata from their parent directories. + + Args: + files (list[Path]): List of file paths to include in the DataFrame. + + Returns: + pd.DataFrame: A DataFrame containing file paths and extracted metadata, including 'version', 'year', 'zone', and 's3_prefix'. + """ + files_df = pd.DataFrame({"file_path": files}) + # Extract parent parts + parent_parts = ( + files_df["file_path"] + .apply(lambda x: Path(x).parent.parts[-5:]) + .apply(pd.Series) + ) + # Join extracted parts as new columns + files_df = files_df.join( + parent_parts.rename( + columns={0: "rest", 1: "version", 2: "_unused", 3: "year", 4: "zone"} + ) + ) + files_df = files_df.drop(columns=["rest", "_unused"]) + files_df["s3_prefix"] = files_df[["version", "year", "zone"]].apply( + lambda x: "/".join(x), axis=1 + ) + return files_df + + +def upload_files(files_df: pd.DataFrame, storage: WEED_storage, s3_root: str) -> None: + """ + Uploads files to an S3-compatible storage using the specified storage utility. Each file's destination + is determined from a DataFrame containing file paths and their respective S3 prefixes. This function + also supports options such as progress tracking, ETag validation, and existence checks for each upload. + + :param files_df: A DataFrame where each row contains details of a file to upload. Must include columns + for 'file_path' specifying the local path of the file and 's3_prefix' defining the relative destination + path in the S3-compatible storage. + :param storage: An instance of WEED_storage, which provides methods for uploading files to + S3-compatible storage backends. + :param s3_root: The root directory in the target S3 storage where files will be uploaded. The + final S3 destination of each file is formed by concatenating this root with the 's3_prefix'. + """ + for file_row in files_df.itertuples(): + s3_destination = f"{s3_root}/{file_row.s3_prefix}" + logger.info(f"Uploading {file_row.file_path} to {s3_destination}") + storage.upload_file_to_s3( + file_row.file_path, + s3_destination, + progress_bar=True, + etag_check=True, + exist_check=True, + ) + + +# def filter_files_on_s3(storage: WEED_storage, s3_root: str, df: pd.DataFrame) -> pd.DataFrame: +# """Get all files in the S3 bucket in a directory.""" +# list_all = storage.get_s3_content(s3_root) +# all_files = [f["Key"] for f in list_all if f["Key"].endswith(".tif")] +# result = [ +# {"year": f.split("/")[1], "zone": f.split("/")[2], "filename": f.split("/")[3].split("_")[0]} +# for f in all_files +# ] +# df_exist= pd.DataFrame(result) + +# # drop existing files on S3 +# # Create a boolean mask for rows to drop +# def should_drop(row): +# for rw in df_exist.itertuples(index=False): +# if int(row['year']) == int(rw.year) and rw.filename in row['path']: +# return True +# return False +# mask = df.apply(should_drop, axis=1) +# df_filtered = df[~mask].reset_index(drop=True) +# return df_filtered + + +def filter_files_on_s3(storage: WEED_storage, s3_root: str, df: pd.DataFrame) -> pd.DataFrame: + """Get all files in the S3 bucket in a directory and filter out existing ones.""" + list_all = storage.get_s3_content(s3_root) + all_files = [f["Key"] for f in list_all if f["Key"].endswith(".tif")] + # Build a list of dicts as before + result = [ + {"year": f.split("/")[-3], "zone": f.split("/")[-2], "filename": f.split("/")[-1].split("_")[0]} + for f in all_files + ] + + def should_drop(row): + for rw in result: + if int(row['year']) == int(rw['year']) and rw['filename'] in row['path']: + return True + return False + + mask = df.apply(should_drop, axis=1) + df_filtered = df[~mask].reset_index(drop=True) + return df_filtered diff --git a/src/eo_processing/utils/catalogue_check.py b/src/eo_processing/utils/catalogue_check.py index 0f0962e..2561fd9 100644 --- a/src/eo_processing/utils/catalogue_check.py +++ b/src/eo_processing/utils/catalogue_check.py @@ -2,30 +2,39 @@ import requests import json import pandas as pd -from eo_processing.utils.geoprocessing import reproj_bbox_to_ll +from eo_processing.utils.geoprocessing import reproj_bbox_to_ll, bbox_of_PointsFeatureCollection import geojson from typing import TYPE_CHECKING import pystac_client +import time if TYPE_CHECKING: from eo_processing.config.data_formats import openEO_bbox_format -def catalogue_check_S1(orbit_direction: str, start: str, end: str, bbox: openEO_bbox_format) -> str | None: +def catalogue_check_S1(orbit_direction: str, start: str, end: str, bbox: openEO_bbox_format, + messages: bool=True, stop_processing: bool=True) -> str | None: """ - Checks the availability of Sentinel-1 images based on the specified orbit direction, date range, - and bounding box. Validates the amount of data available against a predefined minimum threshold. - - :param orbit_direction: The direction of the orbit, either 'ASCENDING' or 'DESCENDING'. If not - specified, the function will check data for both orbit directions. Must comply with the - specified format. - :param start: The start date of the desired time range in ISO 8601 date format (YYYY-MM-DD) or 'YYYY-MM-DDThh:mm:ss.SSZ'.. - :param end: The end date of the desired time range in ISO 8601 date format (YYYY-MM-DD) or 'YYYY-MM-DDThh:mm:ss.SSZ'.. - :param bbox: The bounding box of the area of interest in openEO_bbox_format. It will be reprojected - to a latitude and longitude format for API queries. - :return: Returns the specified orbit direction if sufficient Sentinel-1 images for the given - direction are available. Returns `None` if the checks for both orbit directions combined are - sufficient or if orbit direction was not specified. Raises an error if the amount of data - does not meet the threshold. + Checks the availability of Sentinel-1 imagery within a specified spatiotemporal extent and orbit + direction constraints in the given catalog. + + This function determines if the number of Sentinel-1 images available in a specified bounding + box and temporal range meets an expected threshold. It supports restricting results based on + an orbit direction and provides summary messages or raises exceptions if conditions are not met. + + :param orbit_direction: (str) The orbit direction for filtering imagery. Must be 'ASCENDING' + or 'DESCENDING'. If None, both directions are checked. + :param start: (str) The start date of the query in ISO 8601 format. Time is appended as 'T00:00:00.00Z' + if not provided. + :param end: (str) The end date of the query in ISO 8601 format. Time is appended as 'T00:00:00.00Z' + if not provided. + :param bbox: (openEO_bbox_format) A bounding box specifying the geographic extent of the query + in the format supported by OpenEO. + :param messages: (bool) If True, print messages summarizing the results. Default is True. + :param stop_processing: (bool) If True, raises a ValueError when the imagery availability criteria + are not met. Default is True. + + :return: The specific orbit direction ('ASCENDING' or 'DESCENDING') if enough images are found + for that orbit direction, otherwise None. """ #standard settigns for amount of expected files per day #quickfix on dates that are in date format @@ -52,31 +61,41 @@ def catalogue_check_S1(orbit_direction: str, start: str, end: str, bbox: openEO_ json_data = json.loads(results.text) if len(json_data["value"]) < MIN_VALUE_S1*percentage*temp_extent_days: - print(f'Not enough S1 images with orbit {orbit_direction}. \n' + \ - f'Found {len(json_data["value"])} images.') - else: return orbit_direction + if messages: + print(f'Not enough S1 images with orbit {orbit_direction}. \n' + \ + f'Found {len(json_data["value"])} images.') + else: + if messages: + print(f'Found {len(json_data["value"])} images with orbit direction {orbit_direction}.') + return orbit_direction #use both orbits #check with both directions. nbr_files = count_amount_of_files('S1', latlon_box, start, end) if nbr_files < MIN_VALUE_S1*percentage*temp_extent_days: - raise ValueError(f'not enough S1 without orbit direction selection. \n'+ \ - f'Found {nbr_files} images.') + if stop_processing: + raise ValueError(f'not enough S1 without orbit direction selection. \n'+ \ + f'Found {nbr_files} images.') + if messages: + print(f'Found {nbr_files} images with orbit direction BOTH.') return None -def catalogue_check_S2(start: str, end: str, bbox: openEO_bbox_format) -> None: +def catalogue_check_S2(start: str, end: str, bbox: openEO_bbox_format, + messages: bool=True, stop_processing: bool=True) -> None: """ - Check the availability of Sentinel-2 (S2) satellite images for a given time period - and bounding box. The function calculates the expected minimum number of images - and raises a ValueError if the actual count is insufficient. - - :param start: The start date of the time period, in the format 'YYYY-MM-DD' or 'YYYY-MM-DDThh:mm:ss.SSZ'. - :param end: The end date of the time period, in the format 'YYYY-MM-DD'or 'YYYY-MM-DDThh:mm:ss.SSZ'.. - :param bbox: The bounding box defining the spatial extent, must be in openEO - bounding box format. - :return: None - :raises ValueError: If the number of available Sentinel-2 images is less than the - required minimum threshold. + Checks the availability of Sentinel-2 (S2) satellite images within a given time range and bounding box. + The function determines if the number of available images meets the minimum threshold based on the temporal + extent and year-specific conditions. If the threshold is not met, it raises an error or optionally prints + the count of found images. + + :param start: (str) The start date of the search range in ISO 8601 format. + :param end: (str) The end date of the search range in ISO 8601 format. + :param bbox: (openEO_bbox_format) The bounding box in a geographical coordinate system. + :param messages: (bool) Optional flag to print a message with the number of found images. Default is True. + :param stop_processing: (bool) Optional flag to raise an exception if the number of images is below the threshold. + Default is True. + + :raises ValueError: If the number of Sentinel-2 images does not meet the minimum threshold and stop_processing is True. """ MIN_VALUE_S2 = 1./5. percentage = 0.8 @@ -95,7 +114,10 @@ def catalogue_check_S2(start: str, end: str, bbox: openEO_bbox_format) -> None: nbr_files = count_amount_of_files('S2', latlon_box, start, end) if nbr_files < MIN_VALUE_S2*percentage*temp_extent_days: - raise ValueError(f'not enough S2 images. Found {nbr_files} images.') + if stop_processing: + raise ValueError(f'not enough S2 images. Found {nbr_files} images.') + if messages: + print(f'Found {nbr_files} S2 images.') mece_sequence = [[[9], [15], [21], [27], [34], [40], [49], [54], [57], [62], [69], [73], [82]], [[86], [89], [93], [99], [102], [110], [113], [117], [122], [126], [130], [135], [138]], @@ -132,24 +154,30 @@ def count_amount_of_files(sentinel: str, latlon_box: geojson.Feature, start: str json_data = json.loads(results.text) return len(json_data["value"]) -def catalogue_check_CDSE_S1(orbit_direction: str, start: str, end: str, bbox: openEO_bbox_format) -> str | None: +def catalogue_check_CDSE_S1(orbit_direction: str, start: str, end: str, bbox: openEO_bbox_format, + messages: bool=True, stop_processing: bool=True) -> str | None: """ - Executes a query to the Copernicus Data Space Ecosystem (CDSE) Sentinel-1 catalogue - to verify the availability of Sentinel-1 imagery over a specified temporal extent - and spatial bounding box. The function evaluates whether sufficient Sentinel-1 - images exist based on certain thresholds and optionally filters by orbit direction. - - :param orbit_direction: Direction of the orbit for filtering results. Can be - 'ASCENDING', 'DESCENDING', or None for both. If provided, it must be one of these values. - :param start: Start date of the query temporal extent in ISO 8601 format - (YYYY-MM-DD). Time will default to "T00:00:00.00Z" if not specified. - :param end: End date of the query temporal extent in ISO 8601 format - (YYYY-MM-DD). Time will default to "T00:00:00.00Z" if not specified. - :param bbox: Spatial bounding box for the query in openEO_bbox_format. Used to - define the region of interest. - - :return: The orbit direction if sufficient Sentinel-1 images are available for - the specified orbit, else None. + Checks the availability of Sentinel-1 satellite images for a given temporal and spatial extent + and optionally filters them by orbit direction. If the specified criteria are not met, the + function can optionally raise errors or display messages. VErsion to use with CDSE and pySTAC. + + :param orbit_direction: str. The orbit direction to filter on. Acceptable values are + 'ASCENDING' or 'DESCENDING'. If None, both directions are considered. + :param start: str. Start date for the temporal extent in ISO 8601 format (e.g., 'YYYY-MM-DD'). + :param end: str. End date for the temporal extent in ISO 8601 format (e.g., 'YYYY-MM-DD'). + :param bbox: openEO_bbox_format. Bounding box defining the spatial extent of the search region + in openEO format. + :param messages: bool (default=True). Whether to print messages about the results of the search. + :param stop_processing: bool (default=True). Whether to raise an error if the number of images + found does not meet the required criteria. + + :return: str | None. The orbit direction that satisfies the criteria if applicable. Returns + None if no specific orbit direction was found to meet the requirements or if no orbit + direction was specified. + + :raises ValueError: If `orbit_direction` is not one of the acceptable values ('ASCENDING' or + 'DESCENDING'), or if not enough Sentinel-1 images are found to meet the + required criteria when `stop_processing` is True. """ #quickfix on dates that are in date format if not 'Z' in start: @@ -184,64 +212,79 @@ def catalogue_check_CDSE_S1(orbit_direction: str, start: str, end: str, bbox: op datetime=f"{start}/{end}", fields=["id", "properties.datetime"], query={"sat:orbit_state": {"eq": f"{orbit_direction.lower()}"}, - #"sar:polarizations": {"eq": "VV&VH"}, + "sar:polarizations": {"eq": ["VV", "VH"]}, }, ) # get the dates of all found matches - results = [] - for item in search.items_as_dicts(): - results.append(item['properties']['datetime']) + results_single = [] + query_retry(search, results_single) # count the number of unique dates on which we have observations (resolved tile overlap) - df = pd.DataFrame(results, columns=['date']) + df = pd.DataFrame(results_single, columns=['date']) df['date'] = pd.to_datetime(df['date']) df['date'] = df['date'].apply(lambda x: x.date()) nbr_files = df['date'].nunique() if nbr_files < MIN_VALUE_S1*percentage*temp_extent_days: - print(f'Not enough S1 images with orbit {orbit_direction}. \n' + \ - f'Found {nbr_files} images.') - else: return orbit_direction - #use both orbits -> check with both directions. + if messages: + print(f'Not enough S1 images with orbit {orbit_direction}. \n' + \ + f'Found {nbr_files} images.') + # jump back to check with BOTH orbits + pass + else: + if messages: + print(f'Found {nbr_files} images with orbit direction {orbit_direction}.') + return orbit_direction + #use both orbits -> check with both directions. search = client.search( collections=['sentinel-1-grd'], bbox=list(latlon_box.bounds), datetime=f"{start}/{end}", fields=["id", "properties.datetime"], - #query={"sar:polarizations": {"eq": "VV&VH"} }, + query={"sar:polarizations": {"eq": ["VV", "VH"]} }, ) # get the dates of all found matches - results = [] - for item in search.items_as_dicts(): - results.append(item['properties']['datetime']) + results_both = [] + query_retry(search, results_both) # count the number of unique dates on which we have observations (resolved tile overlap) - df = pd.DataFrame(results, columns=['date']) + df = pd.DataFrame(results_both, columns=['date']) df['date'] = pd.to_datetime(df['date']) df['date'] = df['date'].apply(lambda x: x.date()) nbr_files = df['date'].nunique() if nbr_files < MIN_VALUE_S1*percentage*temp_extent_days: - raise ValueError(f'not enough S1 without orbit direction selection. \n'+ \ - f'Found {nbr_files} images.') + if stop_processing: + raise ValueError(f'not enough S1 without orbit direction selection. \n'+ \ + f'Found {nbr_files} images.') + if messages: + print(f'Found {nbr_files} images with orbit direction BOTH.') return None -def catalogue_check_CDSE_S2(start: str, end: str, bbox: openEO_bbox_format) -> None: +def catalogue_check_CDSE_S2(start: str, end: str, bbox: openEO_bbox_format, + messages: bool=True, stop_processing: bool=True) -> None: """ - Checks the availability of Sentinel-2 images from the Copernicus Data Space Ecosystem STAC - API within a specified temporal and spatial extent. Validates whether the minimum required - observation density is met based on temporal extent duration and predefined thresholds. - - :param start: The start date of the temporal extent in ISO 8601 format (e.g., "YYYY-MM-DD"). - :param end: The end date of the temporal extent in ISO 8601 format (e.g., "YYYY-MM-DD"). - :param bbox: A bounding box defining the spatial extent in openEO_bbox_format. - - :raises ValueError: If the number of Sentinel-2 images found does not meet the minimum - required observation density for the specified bounding box and temporal extent. + Checks the availability of Sentinel-2 images for a specified temporal and spatial extent + using the Copernicus Data Space Ecosystem STAC API. + + This function evaluates whether the available number of Sentinel-2 (S2) images falls + below a minimum threshold for an expected observation frequency based on the specified + temporal extent and spatial bounding box. The function can optionally stop further processing + if the images are insufficient or print a message about the number of images found. + + :param start: (str) The start date of the temporal extent in "YYYY-MM-DD" format. + :param end: (str) The end date of the temporal extent in "YYYY-MM-DD" format. + :param bbox: (openEO_bbox_format) The spatial bounding box in an openEO-compatible format. + :param messages: (bool) A flag to indicate whether to print the number of images found. Defaults to True. + :param stop_processing: (bool) A flag to indicate whether to raise an error and halt processing if + the number of images is insufficient. Defaults to True. + + :raises ValueError: If the number of available S2 images is less than the required threshold and + `stop_processing` is set to True. """ #quickfix on dates that are in date format if not 'Z' in start: @@ -278,8 +321,7 @@ def catalogue_check_CDSE_S2(start: str, end: str, bbox: openEO_bbox_format) -> N # get the dates of all found matches results = [] - for item in search.items_as_dicts(): - results.append(item['properties']['datetime']) + query_retry(search, results) # count the number of unique dates on which we have observations (resolved tile overlap) df = pd.DataFrame(results, columns=['date']) @@ -289,4 +331,60 @@ def catalogue_check_CDSE_S2(start: str, end: str, bbox: openEO_bbox_format) -> N # run the test if nbr_files < MIN_VALUE_S2*percentage*temp_extent_days: - raise ValueError(f'not enough S2 images. Found {nbr_files} images.') + if stop_processing: + raise ValueError(f'not enough S2 images. Found {nbr_files} images.') + if messages: + print(f'Found {nbr_files} S2 images.') + +def catalogue_check_CDSE_S1_FeatureCollection(points_geometry: geojson.FeatureCollection, start_orbit: str, + start: str, end: str) -> str | None: + """ + Checks the availability of satellite data within a defined geometry and time range. + + This function takes a geography feature collection and a time range, then determines + satellite data availability by performing an orbit check. It calculates a bounding + box from the given feature collection to perform the check. + + :param points_geometry: (geojson.FeatureCollection) The geometry as a feature + collection used to define the spatial extent. + :param start_orbit: (str) The starting orbit identifier for the data query. + :param start: (str) The start of the time range for the data query in ISO 8601 format. + :param end: (str) The end of the time range for the data query in ISO 8601 format. + + :return: Returns a string containing the result of the orbit check if data is available. + Returns None if no data is found or the check fails. + """ + # need openEO BBOX for orbit check + bbox = bbox_of_PointsFeatureCollection(points_geometry) + # now we run the orbit check and give result back + return catalogue_check_CDSE_S1(start_orbit, start, end, bbox, messages=False, stop_processing=False) + + +def query_retry(search: pystac_client.ItemSearch, results: list, base_waiting_time: int = 20, + max_attempts: int = 3) -> list: + """ + Retries a query multiple times with exponential backoff in case of failure. + + This function performs a query using the provided pystac_client.ItemSearch object, + attempting to collect datetime properties from each item's dictionary representation. + It will retry the query up to a maximum configured number of attempts, applying an + exponential backoff between retries. If the maximum number of attempts is reached + and the query still fails, it will raise the exception encountered. + + :param search: A pystac_client.ItemSearch object used to perform the item query. + :param results: A list where the datetimes from the queried items will be appended. + :param base_waiting_time: Initial waiting time in seconds for exponential backoff. + Default is 20 seconds. + :param max_attempts: Maximum number of retry attempts. Default is 3. + :return: A list containing the datetime properties of items retrieved from the query. + """ + for attempt in range(max_attempts): + try: + for item in search.items_as_dicts(): + results.append(item['properties']['datetime']) + except Exception as e: + if attempt < (max_attempts - 1): + time.sleep(base_waiting_time * (attempt+1)) # 20s, 40s, 60s... + else: + raise e + return results \ No newline at end of file diff --git a/src/eo_processing/utils/helper.py b/src/eo_processing/utils/helper.py index 7d00904..e731c63 100644 --- a/src/eo_processing/utils/helper.py +++ b/src/eo_processing/utils/helper.py @@ -13,7 +13,7 @@ from datetime import datetime import json from ast import literal_eval -from typing import Union, Dict, List, TYPE_CHECKING +from typing import Union, Dict, List, TYPE_CHECKING, Optional try: import importlib.resources as importlib_resources except: @@ -21,39 +21,80 @@ if TYPE_CHECKING: from eo_processing.config.data_formats import openEO_bbox_format + from eo_processing.utils.storage import WEED_storage -def init_connection(provider: str) -> openeo.Connection : +def init_connection(provider: str, storage: Optional[WEED_storage]=None) -> openeo.Connection : """ - Initializes a connection to the specified OpenEO backend provider and - authenticates the connection using OpenID Connect. Supports connections - to Terrascope, Development, CDSE, CDSE-Staging, and a standard OpenEO - entry point. - - :param provider: The name of the OpenEO backend provider. Supported values - are 'terrascope', 'development', 'cdse', 'cdse-staging', - or other for default OpenEO connection. - :return: An authenticated OpenEO connection to the specified provider. - - :raises ValueError: If the provider specified does not match the supported - categories and a backend-specific connection setup is - unavailable. + + Initializes a connection to an OpenEO backend based on the specified provider. Supports + two modes of authentication: OpenID Connect and client credentials when a storage object + is provided. Each provider corresponds to a specific backend URL. + + :param provider: (str) The name of the OpenEO backend provider. Supported values include: + - 'terrascope': Connects to the VITO backend. + - 'development': Connects to the VITO development backend. + - 'cdse': Connects to the Copernicus Data Space Ecosystem backend. + - 'cdse-staging': Connects to the staging environment of the Copernicus Data Space + Ecosystem backend. + - 'cdse-staging-waw4-1': Connects to the staging WAW4-1 instance. + If an unsupported provider is specified and no storage is provided, a connection is + established with the default "https://openeo.cloud" entry point. + + :param storage: (Optional[WEED_storage]) An optional storage object containing + service account credentials necessary for client credentials authentication. When + provided, attempts to authenticate using the storage's `CLIENT_ID` and `CLIENT_SECRET`. + Supported providers with this authentication mode include: + - 'cdse' + - 'cdse-staging' + - 'cdse-staging-waw4-1' + + :return: An instance of `openeo.Connection` representing the connection to the selected + backend. """ - if provider == 'terrascope': - connection = openeo.connect("https://openeo.vito.be").authenticate_oidc() - elif provider == 'development': - connection = openeo.connect("https://openeo-dev.vito.be").authenticate_oidc() - elif provider == 'cdse': - connection = openeo.connect(url="openeo.dataspace.copernicus.eu").authenticate_oidc() - elif provider == 'cdse-staging': - connection = openeo.connect(url='openeo-staging.dataspace.copernicus.eu').authenticate_oidc() - elif provider == 'cdse-staging-waw4-1': - connection = openeo.connect(url='https://openeo.stag.waw4-1.openeo-int.v1.dataspace.copernicus.eu').authenticate_oidc() + if storage is None: + if provider == 'terrascope': + url = 'https://openeo.vito.be' + elif provider == 'development': + url = 'https://openeo-dev.vito.be' + elif provider == 'cdse': + url = 'https://openeo.dataspace.copernicus.eu' + elif provider == 'cdse-staging': + url = 'https://openeo-staging.dataspace.copernicus.eu' + elif provider == 'cdse-staging-waw4-1': + url = 'https://openeo.stag.waw4-1.openeo-int.v1.dataspace.copernicus.eu' + else: + print('currently no specific connections to backends like creodias and sentinelhub are setup.') + print('use standard entry point') + url = 'https://openeo.cloud' + connection = openeo.connect(url).authenticate_oidc() else: - print('currently no specific connections to backends like creodias and sentinelhub are setup.') - print('use standard entry point') - connection = openeo.connect("https://openeo.cloud").authenticate_oidc() + if provider == 'cdse': + url = 'https://openeo.dataspace.copernicus.eu' + elif provider == 'cdse-staging': + url = 'https://openeo-staging.dataspace.copernicus.eu' + elif provider == 'cdse-staging-waw4-1': + url = 'https://openeo.stag.waw4-1.openeo-int.v1.dataspace.copernicus.eu' + else: + raise ValueError('currently no other backends are setup for WEED service account. Choose from cdse, cdse-staging, cdse-staging-waw4-1.') + connection = openeo.connect(url=url).authenticate_oidc_client_credentials( + client_id=storage.service_account['CLIENT_ID'], + client_secret=storage.service_account['CLIENT_SECRET']) return connection +def purge_jobs(conn: openeo.Connection) -> None: + """ + Purges all jobs from the given openEO connection. + + :param conn: The openEO connection object. + """ + jobs = conn.list_jobs(limit=1000) + + for job_metadata in jobs: + if job_metadata.get("status", None) in ["queued", "running", "queued_for_start"]: + job = conn.job(job_metadata["id"]) + print(f"Cancelling job {job.job_id} with status {job.status()}") + job.stop() + print("All jobs purged if needed.") def location_visu(aoi_object: openEO_bbox_format | gpd.GeoDataFrame, zoom: bool = False, region: str = 'EU', label: bool = True) -> None: diff --git a/src/eo_processing/utils/jobmanager.py b/src/eo_processing/utils/jobmanager.py index bbfa96a..086c10b 100644 --- a/src/eo_processing/utils/jobmanager.py +++ b/src/eo_processing/utils/jobmanager.py @@ -1,6 +1,7 @@ from __future__ import annotations import os import json +import geojson import logging import geopandas as gpd from threading import Thread, active_count @@ -27,7 +28,8 @@ import openeo import warnings from eo_processing.utils.helper import string_to_dict -from eo_processing.utils.geoprocessing import create_feature_extraction_processing_grid, get_point_number +from eo_processing.utils.geoprocessing import (create_feature_extraction_processing_grid, get_point_number, + bbox_of_PointsFeatureCollection, reproj_bbox_to_ll) from eo_processing.utils.mgrs import gridID_2_epsg if TYPE_CHECKING: @@ -119,25 +121,19 @@ def download_job_too_long(self, job: openeo.BatchJob, row: pd.Series) -> bool: return True else: return False - def check_finished(self, job: openeo.BatchJob) -> bool: + def check_finished(self, job: openeo.BatchJob, row: pd.Series) -> bool: """ - Check if the metadata file for a given job already exists in the filesystem, - indicating whether the job has been completed. - - This function extracts the job's metadata and determines the file path for - the associated metadata file. It then checks if this file path exists, which - would suggest that the job has been processed and the metadata has been saved. - - :param job: The job object whose completion status needs to be verified. It - must provide a 'describe' method that returns a dictionary - containing job metadata, including the job's title. - :return: A boolean value where `True` indicates that the job metadata file - exists, and thus the job is finished. `False` signifies that the - metadata file is not found, indicating that the job might not be - complete. + Determines if a batch job is finished by checking the existence of its metadata. + + :param job: The batch job for which the status needs to be checked. + :param row: A pandas Series containing job-related information, including the + file_prefix used to identify the metadata file. + :return: True if the job's metadata file exists, indicating the job is finished; + False otherwise. """ - job_metadata = job.describe() - title = os.path.splitext(job_metadata['title'])[0] + #job_metadata = job.describe() + #title = os.path.splitext(job_metadata['title'])[0] + title = row["file_prefix"] metadata_path = self.get_job_metadata_path(job.job_id, title) return os.path.exists(metadata_path) @@ -223,7 +219,8 @@ def on_job_error(self, job: openeo.BatchJob, row: pd.Series) -> Union[str, bool] """ error_logs = job.logs(level="error") job_metadata = job.describe_job() - title = os.path.splitext(job_metadata['title'])[0] + #title = os.path.splitext(job_metadata['title'])[0] + title = row["file_prefix"] error_log_path = self.get_error_log_path(job.job_id,title) job_graph_path = self.get_job_graph_path(job.job_id,title) @@ -257,7 +254,10 @@ def on_job_done(self, job: openeo.BatchJob, row: pd.Series) -> None: job_metadata = job.describe() job_dir = self.get_job_dir(job.job_id) - title = os.path.splitext(job_metadata['title'])[0] + #title = os.path.splitext(job_metadata['title'])[0] + + title = row["file_prefix"] + file_ext = job_metadata['process']['process_graph']['saveresult1']['arguments']['format'].lower() metadata_path = self.get_job_metadata_path(job.job_id,title) job_graph_path = self.get_job_graph_path(job.job_id,title) @@ -280,16 +280,15 @@ def on_job_done(self, job: openeo.BatchJob, row: pd.Series) -> None: else : s3_client = self.storage_options["WEED_storage"].get_s3_client() bucket_name= self.storage_options["WEED_storage"].get_s3_bucket_name() - s3_client.download_file(bucket_name, os.path.join(S3_prefix,f"timeseries.{file_ext}"), + s3_client.download_file(bucket_name, os.path.join(S3_prefix,f"{title}.{file_ext}"), job_dir / f"{title}.{file_ext}") - if not self.storage_options.get('workspace_export', False): #fix prefix problem for non netcdf or GTiff files if file_ext in ['netcdf','gtiff']: results.download_files(job_dir, include_stac_metadata=False) else : - results.download_file(job_dir / f"{title}.{file_ext}", name=f"timeseries.{file_ext}") + results.download_file(job_dir / f"{title}.{file_ext}") with open(metadata_path, "w", encoding='utf8') as f: @@ -370,7 +369,7 @@ def _track_statuses(self, job_db: JobDatabaseInterface, stats: Optional[Dict] = new_status = "downloading" if previous_status == "downloading": - if not self.check_finished(the_job): + if not self.check_finished(the_job, active.loc[i]): new_status = "downloading" if previous_status != "error" and new_status == "error": @@ -500,6 +499,8 @@ def _launch_job(self, start_job, df, i, backend_name, stats: Optional[Dict] = No # start job if not yet done by callback try: job_con = job.connection + # Proactively refresh bearer token (because task in thread will not be able to do that) + self._refresh_bearer_token(connection=job_con) task = _JobStartTask( root_url=job_con.root_url, bearer_token=job_con.auth.bearer if isinstance(job_con.auth, BearerAuth) else None, @@ -781,6 +782,7 @@ def create_job_dataframe(gdf: Union[gpd.GeoDataFrame, List], year: int, file_nam discriminator: Optional[str] = None, target_crs: Optional[int] = None, version: Optional[str] = None, model_ID: Optional[str] = None, + nonEO_file: Optional[str] = None, storage_options: Optional[storage_option_format] = None, organization_id : Optional[int] = None, path_global_grid: Optional[str] = None, feature_bbox: Optional[Tuple[float, float, float, float]] = None) -> gpd.GeoDataFrame: @@ -816,11 +818,11 @@ def create_job_dataframe(gdf: Union[gpd.GeoDataFrame, List], year: int, file_nam if isinstance(gdf, gpd.GeoDataFrame): # we are preparing a inference or post-processing actions columns = ['name', 'tileID', 'target_epsg', 'bbox', 'file_prefix', 'start_date', 'end_date','export_workspace', - 's3_prefix', 'organization_id', 's2_tileid_list'] + 's3_prefix', 'organization_id', 's2_tileid_list','nonEO_file'] dtypes = {'name': 'string', 'tileID': 'string', 'target_epsg': 'UInt16', 'file_prefix': 'string', 'start_date': 'string', 'end_date': 'string', 's3_prefix': 'string', 'geometry': 'geometry', 'bbox': 'string', 'organization_id':'UInt16','s2_tileid_list':'string', - 'export_workspace':'string'} + 'export_workspace':'string','nonEO_file' : 'string'} job_df = gdf.copy() @@ -861,6 +863,8 @@ def create_job_dataframe(gdf: Union[gpd.GeoDataFrame, List], year: int, file_nam job_df['s3_prefix'] = None job_df['export_workspace'] = None + # set no EO list Should be later replaced by an extraction method straight out of the model metadata. + job_df['nonEO_file'] = nonEO_file # a fix since the "name" column has to be unique job_df['tileID'] = job_df[tile_col].copy() if discriminator: @@ -959,8 +963,14 @@ def create_job_dataframe(gdf: Union[gpd.GeoDataFrame, List], year: int, file_nam # merge in the needed Polygons and convert to GeoDataFrame job_df = pd.merge(df, gdf_grid[['tile_name', 'geometry']], left_on='name', right_on='tile_name', how='left') - # remove row which have no polygon assigned due to fact that this grid is not in openEO processing extent - job_df = job_df.dropna(subset=["geometry"]) + # we have to fix rows which did not get a valid geometry + missing_geometry_row = job_df[job_df['geometry'].isna()].index + #print(missing_geometry_row) + for row in missing_geometry_row: + job_df.at[row, 'geometry'] = reproj_bbox_to_ll(bbox_of_PointsFeatureCollection(geojson.loads(job_df.loc[row, 'FeatureCollection']))) + job_df.at[row, 'tile_name'] = job_df.loc[row, 'name'] + #job_df = job_df.dropna(subset=["geometry"]) + # convert to GeoPandas GeoDataFrame job_df = gpd.GeoDataFrame(job_df, geometry='geometry') job_df.reset_index(inplace=True) @@ -970,8 +980,17 @@ def create_job_dataframe(gdf: Union[gpd.GeoDataFrame, List], year: int, file_nam job_df['end_date'] = ( pd.to_datetime(job_df['end_date'], format='%Y-%m-%d') - pd.Timedelta(seconds=1)).dt.strftime( '%Y-%m-%dT%H:%M:%SZ') - job_df['export_workspace'] = None - job_df['s3_prefix'] = None + # set the s3_prefix which is needed for the path to S3 storage relative to bucket if we export + if storage_options: + job_df['s3_prefix'] = storage_options.get('S3_prefix', None) + if storage_options.get('WEED_storage', None): + job_df['export_workspace'] = storage_options['WEED_storage'].get_export_workspace() + else: + job_df['export_workspace'] = None + else: + job_df['s3_prefix'] = None + job_df['export_workspace'] = None + job_df['organization_id'] = organization_id job_df['s2_tileid_list'] = None diff --git a/src/eo_processing/utils/messaging.py b/src/eo_processing/utils/messaging.py index 7391179..76574c0 100644 --- a/src/eo_processing/utils/messaging.py +++ b/src/eo_processing/utils/messaging.py @@ -6,8 +6,7 @@ # Constants SMTP_SERVER = 'mail.vgt.vito.be' SENDER_NAME = 'WEED openEO processing cluster' -SENDER_EMAIL = 'esa.weed.project@vito.be' - +SENDER_EMAIL = 'esa.weed.project@vgt.vito.be' def format_email_address(name_email_pair: Tuple[str, str]) -> str: """ diff --git a/src/eo_processing/utils/stac_helper.py b/src/eo_processing/utils/stac_helper.py index 638e46d..e88e2f3 100644 --- a/src/eo_processing/utils/stac_helper.py +++ b/src/eo_processing/utils/stac_helper.py @@ -1,22 +1,51 @@ import pystac_client -def get_stac_collection_url(collection_id): - """Fetches the url of the STAC collection with the given collection_id. And returns None if not found.""" - client = pystac_client.Client.open("https://catalogue.weed.apex.esa.int/") +def get_stac_collection_url(collection_id: str, catalog_url: str = "https://catalogue.weed.apex.esa.int/") -> str: + """ + Fetches the URL of a STAC (SpatioTemporal Asset Catalog) collection based on the provided + collection ID and catalog URL. + + This function utilizes the pystac_client library to interact with a STAC catalog and extract + the URL of a specific collection by its ID. If the collection cannot be found, the function + returns None. + + :param collection_id (str): The unique identifier of the STAC collection to retrieve. + :param catalog_url (str): The URL of the STAC catalog to query. Defaults to + 'https://catalogue.weed.apex.esa.int/'. + + :return: The URL (as a string) of the requested STAC collection if found, or None if the + collection does not exist in the catalog. + """ + client = pystac_client.Client.open(catalog_url) try: return client.get_collection(collection_id).self_href except: return None +def query_modelID_asset_url(model_id: str, + catalog_url: str ="https://catalogue.weed.apex.esa.int/", + collection_id: str ="model-STAC-v2") -> str: + """ + Queries the URL of a specific asset for a given modelID from a STAC catalog. + + This function connects to the specified STAC catalog URL and searches for metadata + associated with the given modelID and collectionID using CQL2 filters. It returns + the URL of the asset identified as `model_valid_geometry`. + + :param model_id: (str) The unique identifier of the model to search for. + :param catalog_url: (str) [Optional] The URL of the STAC catalog to query. + Defaults to "https://catalogue.weed.apex.esa.int/". + :param collection_id: (str) [Optional] The ID of the collection in the STAC catalog + to search within. Defaults to "model-STAC-v2". -def query_asset_url(model_id): - """Use model-id to get the parquet asset link""" - client = pystac_client.Client.open("https://catalogue.weed.apex.esa.int/") + :return: (str) The URL of the asset associated with the searched model ID. + """ + client = pystac_client.Client.open(catalog_url) search = client.search( limit=20, - collections=["model-STAC-v2"], + collections=[collection_id], filter={"op": "=", "args": [{"property": "properties.modelID"}, model_id]}, filter_lang="cql2-json", ) diff --git a/src/eo_processing/utils/storage.py b/src/eo_processing/utils/storage.py index 65977c6..c319101 100644 --- a/src/eo_processing/utils/storage.py +++ b/src/eo_processing/utils/storage.py @@ -2,6 +2,7 @@ import os import boto3 +import pandas as pd from botocore.exceptions import ClientError from getpass import getpass import geopandas as gpd @@ -760,6 +761,9 @@ def __init__(self, sql_credentials: Optional[sql_credentials_format] = None): #at the moment we initialize with hadoop false. Not sure if it's need to have? self.hadoop = False + def get_info(self): + return self.sql_credentials['host'], self.sql_credentials['port'], self.sql_credentials['dbname'], self.sql_credentials['schema'], self.sql_credentials['password'] + def create_connection(self) -> psycopg.Connection: """ Establishes a connection to a PostgreSQL database using the provided credentials. @@ -811,6 +815,7 @@ def GenericQueryWithResult(self, sql_statement: str) -> List[tuple]: print('** get data from request') # create cursor cur = conn.cursor() + cur.itersize = 10000 cur.execute(sql_statement) # get data vResult = cur.fetchall() @@ -975,7 +980,78 @@ def QueryItems(self, table: str, lcolumns: List[str]) -> List[tuple]: return lresults - def StatusUpdateTiles(self, table: str, tileid: int, lcolumns: List[str], lmsg: List[str]) -> bool: + def AddColumns(self, table: str, column_names: lst(str)) -> None: + """ + Adds a new column to a specified table in the database. The method dynamically constructs + an SQL ALTER TABLE statement to add the column with the specified name, ensuring + compatibility with the existing table structure. If the column already exists, the query + does not raise an error. + + Args: + table (str): The name of the table to which the column should be added. + column_names (str): The names of the new columns. + + Returns: + None + """ + + sql_statement = f"""ALTER TABLE {table} + {',\n'.join([f'ADD COLUMN IF NOT EXISTS "{col.lower()}" numeric' for col in column_names])};""" + + self.GenericQueryWithOUTResult(sql_statement) + + def UploadTrainingPointData(self, table: str, df: pd.DataFrame, column_names: List[str]) -> None: + """ + Uploads data to the specified database table by adding missing columns, handling existing + data updates, and inserting new data points in bulk or individually. + + This method performs the following steps: + 1. Checks and ensures all necessary columns in the table. + 2. Identifies whether existing data points need to be updated or if new points need to be + inserted. + 3. Performs a bulk data insert for new data points if any. + 4. Updates rows individually for existing data points. + + Parameters: + table: str + The name of the database table where the data should be uploaded. + df: pd.DataFrame + A pandas DataFrame containing the data to be uploaded. + column_names: List[str] + A list of column names that need to be added to the table if they do not exist. + + Raises: + None + + Returns: + None + """ + + #First check if all columns are in the table + self.AddColumns(table, column_names) + + #First check which points are new and which are not + sql_statement = f"""SELECT mgrsid10, year + FROM {table}""" + + available_keys = self.GenericQueryWithResult(sql_statement) + + #split according to PK available are not + bulk_df = df[~df.set_index(['MGRSid10', 'year']).index.isin(available_keys)] + line_df = df[df.set_index(['MGRSid10', 'year']).index.isin(available_keys)] + + if len(bulk_df) != 0: + #upload bulk + data = ReadFaker(bulk_df) + self.BulkInsert(table, data, tuple([column.lower() for column in df.columns])) + + for index, row in line_df.iterrows(): + #upload line + self.StatusUpdateTiles(table, (row['MGRSid10'], row['year']), + tuple([column.lower() for column in column_names]), + row[column_names].to_list()) + + def StatusUpdateTiles(self, table: str, PK: tuple(str, int), lcolumns: List[str], lmsg: List[str]) -> bool: """ Updates specific columns in a particular database table for a given tile ID with new values. @@ -986,7 +1062,7 @@ def StatusUpdateTiles(self, table: str, tileid: int, lcolumns: List[str], lmsg: database connection cleanup. :param table: The name of the database table to be updated. - :param tileid: The identifier of the tile for which data is to be updated. + :param PK: The identifier for which data is to be updated. :param lcolumns: A list of column names that need to be updated. :param lmsg: A list of new values corresponding to the specified columns. :return: True if the update operation completes successfully, otherwise False. @@ -1002,8 +1078,8 @@ def StatusUpdateTiles(self, table: str, tileid: int, lcolumns: List[str], lmsg: # prepare UPDATE statement print('** update the tile status...') for i in range(0, len(lcolumns)): - sql_statement = "UPDATE %s SET %s = %%s WHERE tile_id = %%s;" % (table, lcolumns[i]) - cur.execute(sql_statement, (lmsg[i], tileid)) + sql_statement = 'UPDATE %s SET "%s" = %%s WHERE (mgrsid10, year) = %%s;' % (table, lcolumns[i]) + cur.execute(sql_statement, (lmsg[i], PK)) # commit transactions conn.commit() @@ -1280,7 +1356,7 @@ def upload_collection_to_catalog(self, collection: pystac.Collection, edit_flag: #get the auth auth_token = self.get_bearer_auth() #get catalog_url - catalog_url = self.get_catalog_url() + catalog_url = self.get_catalog_url().rstrip('/') # load the collection from the created collection. coll = collection.to_dict() @@ -1296,7 +1372,7 @@ def upload_collection_to_catalog(self, collection: pystac.Collection, edit_flag: # upload a new collection collection_url = f"{catalog_url}/collections/" resp = post(collection_url, auth=auth_token, json=coll) - if resp.status_code == 201: + if resp.status_code == 200 or resp.status_code == 201: coll_id = resp.json()["id"] if edit_flag: print(f"Collection edited: {coll_id}") @@ -1326,7 +1402,7 @@ def upload_items_to_collection(self, collection: pystac.Collection, #get the auth auth_token = self.get_bearer_auth() #get catalog_url - catalog_url = self.get_catalog_url() + catalog_url = self.get_catalog_url().rstrip('/') # check if there are items that need to be uploaded only if update is False if len(items_to_upload) == 0: @@ -1588,7 +1664,7 @@ def __init__(self, username: str = 'buchhornm', S3 bucket and ML Flow uri. It also initializes necessary credentials and configurations for using Google Drive and S3 storage services. - :param username: A username string used for accessing credentials. + :param username: A username string used for accessing credentials. NOTE use "VAULT_TOKEN" to use token saved in environment variable. :param gdrive_entry_point: The entry point ID of the Google Drive folder. :param s3_bucket: The name of the S3 bucket to be used. :param stac_env: The environment name for the STAC catalog. @@ -1609,6 +1685,7 @@ def __init__(self, username: str = 'buchhornm', self._set_gdrive_credentials(gdrive_entry_point=gdrive_entry_point) self._set_stac_credentials(stac_env=stac_env) self._set_mlflow_credentials() + self._set_service_account() def _get_credentials(self): """warper to get the credentials.""" @@ -1665,6 +1742,17 @@ def _set_s3_credentials(self, bucket: str) -> None: self.s3_client.close() self._init_boto3() + def _set_service_account(self) -> None: + """ + Sets the service account based on provided credentials. + + This method retrieves the service account information from the + credentials dictionary and assigns it to the service_account attribute. + + :return: None + """ + self.service_account = string_to_dict(self.credentials['weed-service-account']) + def _set_sql_credentials(self) -> None: """ Sets SQL credentials for the database connection. @@ -1766,30 +1854,57 @@ def _set_stac_credentials(self, stac_env: str) -> None: def get_credentials(user :str ) -> Dict[str, str]: """ - Retrieves WEED access credentials from Terrascope VAULT using LDAP authentication. + Gets the credentials of a specified user from the Terrascope VAULT. + + This function prompts the user to enter their password and then + authenticates with the Terrascope VAULT service using the LDAP method. + Once authenticated, it attempts to retrieve the secret for the TAP apps + under the WEED path from the Vault. If unsuccessful, an exception is raised. + + Parameters: + :param user: (str) The username of the account to authenticate against the VAULT. NOTE use "VAULT_TOKEN" to use token saved in environment variable. - This method prompts the user to enter their password for Terrascope VAULT, authenticates - with the VAULT using LDAP, and fetches credentials from the WEED KV storage path. + Returns: + :returns: Dictionary containing the retrieved secret credentials. - :return: credentials as a dictionary + Raises: + :raises Exception: If the credentials cannot be retrieved (e.g., not connected + to the VITO VPN or authentication failure). """ - password_prompt = 'Please enter your password for the Terrascope VAULT: ' - service_account_password = getpass(prompt=password_prompt) + # first a check if username is "VAULT_TOKEN" + if user == "VAULT_TOKEN": + # we run the auth via token from environment variable + # check if TOKEN is set + vault_token = os.environ.get('VAULT_TOKEN') + if vault_token is None: + raise Exception('VAULT_TOKEN environment variable is not set. Please set it and try again. Or use a different username.') + else: + password_prompt = 'Please enter your password for the Terrascope VAULT: ' + service_account_password = getpass(prompt=password_prompt) try: client = hvac.Client(url='https://vault.vgt.vito.be') - client.auth.ldap.login( - username=user, - password=service_account_password, - mount_point='ldap' - ) + if user == "VAULT_TOKEN": + client.token = vault_token + if not client.is_authenticated(): + raise Exception('Your token is expired. Please re-authenticate.') + else: + client.auth.ldap.login( + username=user, + password=service_account_password, + mount_point='ldap' + ) + + if not client.is_authenticated(): + raise Exception('Your credentials are invalid. Please re-authenticate.') + secret_version_response = client.secrets.kv.v2.read_secret_version(mount_point='kv', path='TAP/apps/WEED', raise_on_deleted_version=True) client.logout() except: raise Exception('Could not retrieve WEED credentials from Terrascope VAULT. ' - 'Are you connected to the VITO VPN?') + 'Are you connected to the VITO VPN? Check your password or renew your token!') return secret_version_response['data']['data'] def read_credential_file(file_path: str = '~/.sonata_credentials') -> Dict[str, str]: diff --git a/tests/resources/cube_extraction/extract_S1_basic.json b/tests/resources/cube_extraction/extract_S1_basic.json index 73c6eff..36511c7 100644 --- a/tests/resources/cube_extraction/extract_S1_basic.json +++ b/tests/resources/cube_extraction/extract_S1_basic.json @@ -12,7 +12,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -20,7 +20,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true } @@ -56,7 +59,6 @@ "options": { "implementation_version": "2", "tile_size": 128, - "otb_memory": 1024, "debug": false, "elev_geoid": "/opt/openeo-vito-aux-data/egm96.tif" } diff --git a/tests/resources/cube_extraction/extract_S1_custom_crs.json b/tests/resources/cube_extraction/extract_S1_custom_crs.json index b5e8f7d..8cd3a4a 100644 --- a/tests/resources/cube_extraction/extract_S1_custom_crs.json +++ b/tests/resources/cube_extraction/extract_S1_custom_crs.json @@ -12,7 +12,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -20,7 +20,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true } @@ -56,7 +59,6 @@ "options": { "implementation_version": "2", "tile_size": 128, - "otb_memory": 1024, "debug": false, "elev_geoid": "/opt/openeo-vito-aux-data/egm96.tif" } diff --git a/tests/resources/cube_extraction/extract_S1_integration.json b/tests/resources/cube_extraction/extract_S1_integration.json index 286d570..58b67db 100644 --- a/tests/resources/cube_extraction/extract_S1_integration.json +++ b/tests/resources/cube_extraction/extract_S1_integration.json @@ -16,7 +16,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -24,7 +24,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true } diff --git a/tests/resources/cube_extraction/extract_S1_temporal_aggregation.json b/tests/resources/cube_extraction/extract_S1_temporal_aggregation.json index aa0ab2d..59a572c 100644 --- a/tests/resources/cube_extraction/extract_S1_temporal_aggregation.json +++ b/tests/resources/cube_extraction/extract_S1_temporal_aggregation.json @@ -12,7 +12,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -20,7 +20,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true } @@ -56,7 +59,6 @@ "options": { "implementation_version": "2", "tile_size": 128, - "otb_memory": 1024, "debug": false, "elev_geoid": "/opt/openeo-vito-aux-data/egm96.tif" } diff --git a/tests/resources/cube_extraction/ts_datacube_extraction_S1_interpolation.json b/tests/resources/cube_extraction/ts_datacube_extraction_S1_interpolation.json index 7212dc1..29448ea 100644 --- a/tests/resources/cube_extraction/ts_datacube_extraction_S1_interpolation.json +++ b/tests/resources/cube_extraction/ts_datacube_extraction_S1_interpolation.json @@ -141,7 +141,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -149,7 +149,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true } @@ -185,7 +188,6 @@ "options": { "implementation_version": "2", "tile_size": 128, - "otb_memory": 1024, "debug": false, "elev_geoid": "/opt/openeo-vito-aux-data/egm96.tif" } diff --git a/tests/resources/cube_extraction/ts_datacube_extraction_S2_with_masking.json b/tests/resources/cube_extraction/ts_datacube_extraction_S2_with_masking.json index cad2077..eacf802 100644 --- a/tests/resources/cube_extraction/ts_datacube_extraction_S2_with_masking.json +++ b/tests/resources/cube_extraction/ts_datacube_extraction_S2_with_masking.json @@ -197,7 +197,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -205,7 +205,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true } @@ -241,7 +244,6 @@ "options": { "implementation_version": "2", "tile_size": 128, - "otb_memory": 1024, "debug": false, "elev_geoid": "/opt/openeo-vito-aux-data/egm96.tif" } diff --git a/tests/resources/cube_extraction/ts_datacube_extraction_combined.json b/tests/resources/cube_extraction/ts_datacube_extraction_combined.json index bbfae6a..d0e2143 100644 --- a/tests/resources/cube_extraction/ts_datacube_extraction_combined.json +++ b/tests/resources/cube_extraction/ts_datacube_extraction_combined.json @@ -97,7 +97,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -105,7 +105,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true } @@ -141,7 +144,6 @@ "options": { "implementation_version": "2", "tile_size": 128, - "otb_memory": 1024, "debug": false, "elev_geoid": "/opt/openeo-vito-aux-data/egm96.tif" } diff --git a/tests/resources/cube_extraction/ts_datacube_extraction_combined_custom_crs.json b/tests/resources/cube_extraction/ts_datacube_extraction_combined_custom_crs.json index c19956b..81a443b 100644 --- a/tests/resources/cube_extraction/ts_datacube_extraction_combined_custom_crs.json +++ b/tests/resources/cube_extraction/ts_datacube_extraction_combined_custom_crs.json @@ -97,7 +97,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -105,7 +105,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true } @@ -141,7 +144,6 @@ "options": { "implementation_version": "2", "tile_size": 128, - "otb_memory": 1024, "debug": false, "elev_geoid": "/opt/openeo-vito-aux-data/egm96.tif" } diff --git a/tests/resources/cube_extraction/ts_datacube_extraction_combined_integration.json b/tests/resources/cube_extraction/ts_datacube_extraction_combined_integration.json index 852acef..6bf449c 100644 --- a/tests/resources/cube_extraction/ts_datacube_extraction_combined_integration.json +++ b/tests/resources/cube_extraction/ts_datacube_extraction_combined_integration.json @@ -101,7 +101,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -109,7 +109,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true } diff --git a/tests/resources/cube_generation/generate_S1_feature_cube.json b/tests/resources/cube_generation/generate_S1_feature_cube.json index 3704b40..930a43a 100644 --- a/tests/resources/cube_generation/generate_S1_feature_cube.json +++ b/tests/resources/cube_generation/generate_S1_feature_cube.json @@ -12,7 +12,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -20,7 +20,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true } @@ -56,7 +59,6 @@ "options": { "implementation_version": "2", "tile_size": 128, - "otb_memory": 1024, "debug": false, "elev_geoid": "/opt/openeo-vito-aux-data/egm96.tif" } @@ -516,7 +518,6 @@ "process_id": "apply_dimension", "arguments": { "context": { - "parallel": true, "TileSize": 128 }, "data": { @@ -671,66 +672,66 @@ }, "dimension": "bands", "target": [ - "VH_p2", - "VH_p5", - "VH_p25", - "VH_median", - "VH_p75", - "VH_p95", - "VH_p98", - "VH_mean", - "VH_sd", - "VH_sum", - "VH_iqr", - "VH_iqr0595", - "VV_p2", - "VV_p5", - "VV_p25", - "VV_median", - "VV_p75", - "VV_p95", - "VV_p98", - "VV_mean", - "VV_sd", - "VV_sum", - "VV_iqr", - "VV_iqr0595", - "VHVVD_p2", - "VHVVD_p5", - "VHVVD_p25", - "VHVVD_median", - "VHVVD_p75", - "VHVVD_p95", - "VHVVD_p98", - "VHVVD_mean", - "VHVVD_sd", - "VHVVD_sum", - "VHVVD_iqr", - "VHVVD_iqr0595", - "VHVVR_p2", - "VHVVR_p5", - "VHVVR_p25", - "VHVVR_median", - "VHVVR_p75", - "VHVVR_p95", - "VHVVR_p98", - "VHVVR_mean", - "VHVVR_sd", - "VHVVR_sum", - "VHVVR_iqr", - "VHVVR_iqr0595", - "DpRVIVV_p2", - "DpRVIVV_p5", - "DpRVIVV_p25", - "DpRVIVV_median", - "DpRVIVV_p75", - "DpRVIVV_p95", - "DpRVIVV_p98", - "DpRVIVV_mean", - "DpRVIVV_sd", - "DpRVIVV_sum", - "DpRVIVV_iqr", - "DpRVIVV_iqr0595" + "vh_p2", + "vh_p5", + "vh_p25", + "vh_median", + "vh_p75", + "vh_p95", + "vh_p98", + "vh_mean", + "vh_sd", + "vh_sum", + "vh_iqr", + "vh_iqr0595", + "vv_p2", + "vv_p5", + "vv_p25", + "vv_median", + "vv_p75", + "vv_p95", + "vv_p98", + "vv_mean", + "vv_sd", + "vv_sum", + "vv_iqr", + "vv_iqr0595", + "vhvvd_p2", + "vhvvd_p5", + "vhvvd_p25", + "vhvvd_median", + "vhvvd_p75", + "vhvvd_p95", + "vhvvd_p98", + "vhvvd_mean", + "vhvvd_sd", + "vhvvd_sum", + "vhvvd_iqr", + "vhvvd_iqr0595", + "vhvvr_p2", + "vhvvr_p5", + "vhvvr_p25", + "vhvvr_median", + "vhvvr_p75", + "vhvvr_p95", + "vhvvr_p98", + "vhvvr_mean", + "vhvvr_sd", + "vhvvr_sum", + "vhvvr_iqr", + "vhvvr_iqr0595", + "dprvivv_p2", + "dprvivv_p5", + "dprvivv_p25", + "dprvivv_median", + "dprvivv_p75", + "dprvivv_p95", + "dprvivv_p98", + "dprvivv_mean", + "dprvivv_sd", + "dprvivv_sum", + "dprvivv_iqr", + "dprvivv_iqr0595" ] } }, @@ -738,63 +739,63 @@ "process_id": "filter_bands", "arguments": { "bands": [ - "VH_p2", - "VH_p5", - "VH_p25", - "VH_median", - "VH_p75", - "VH_p95", - "VH_p98", - "VH_mean", - "VH_sd", - "VH_iqr", - "VH_iqr0595", - "VV_p2", - "VV_p5", - "VV_p25", - "VV_median", - "VV_p75", - "VV_p95", - "VV_p98", - "VV_mean", - "VV_sd", - "VV_iqr", - "VV_iqr0595", - "VHVVD_p2", - "VHVVD_p5", - "VHVVD_p25", - "VHVVD_median", - "VHVVD_p75", - "VHVVD_p95", - "VHVVD_p98", - "VHVVD_mean", - "VHVVD_sd", - "VHVVD_iqr", - "VHVVD_iqr0595", - "VHVVR_p2", - "VHVVR_p5", - "VHVVR_p25", - "VHVVR_median", - "VHVVR_p75", - "VHVVR_p95", - "VHVVR_p98", - "VHVVR_mean", - "VHVVR_sd", - "VHVVR_sum", - "VHVVR_iqr", - "VHVVR_iqr0595", - "DpRVIVV_p2", - "DpRVIVV_p5", - "DpRVIVV_p25", - "DpRVIVV_median", - "DpRVIVV_p75", - "DpRVIVV_p95", - "DpRVIVV_p98", - "DpRVIVV_mean", - "DpRVIVV_sd", - "DpRVIVV_sum", - "DpRVIVV_iqr", - "DpRVIVV_iqr0595" + "vh_p2", + "vh_p5", + "vh_p25", + "vh_median", + "vh_p75", + "vh_p95", + "vh_p98", + "vh_mean", + "vh_sd", + "vh_iqr", + "vh_iqr0595", + "vv_p2", + "vv_p5", + "vv_p25", + "vv_median", + "vv_p75", + "vv_p95", + "vv_p98", + "vv_mean", + "vv_sd", + "vv_iqr", + "vv_iqr0595", + "vhvvd_p2", + "vhvvd_p5", + "vhvvd_p25", + "vhvvd_median", + "vhvvd_p75", + "vhvvd_p95", + "vhvvd_p98", + "vhvvd_mean", + "vhvvd_sd", + "vhvvd_iqr", + "vhvvd_iqr0595", + "vhvvr_p2", + "vhvvr_p5", + "vhvvr_p25", + "vhvvr_median", + "vhvvr_p75", + "vhvvr_p95", + "vhvvr_p98", + "vhvvr_mean", + "vhvvr_sd", + "vhvvr_sum", + "vhvvr_iqr", + "vhvvr_iqr0595", + "dprvivv_p2", + "dprvivv_p5", + "dprvivv_p25", + "dprvivv_median", + "dprvivv_p75", + "dprvivv_p95", + "dprvivv_p98", + "dprvivv_mean", + "dprvivv_sd", + "dprvivv_sum", + "dprvivv_iqr", + "dprvivv_iqr0595" ], "data": { "from_node": "renamelabels2" diff --git a/tests/resources/cube_generation/generate_S1_feature_cube_integration.json b/tests/resources/cube_generation/generate_S1_feature_cube_integration.json index d6014e7..1536b1d 100644 --- a/tests/resources/cube_generation/generate_S1_feature_cube_integration.json +++ b/tests/resources/cube_generation/generate_S1_feature_cube_integration.json @@ -16,7 +16,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -24,7 +24,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true } diff --git a/tests/resources/cube_generation/generate_S2_feature_cube.json b/tests/resources/cube_generation/generate_S2_feature_cube.json index 345ad04..84d8709 100644 --- a/tests/resources/cube_generation/generate_S2_feature_cube.json +++ b/tests/resources/cube_generation/generate_S2_feature_cube.json @@ -1269,7 +1269,6 @@ "process_id": "apply_dimension", "arguments": { "context": { - "parallel": true, "TileSize": 128 }, "data": { @@ -1424,366 +1423,366 @@ }, "dimension": "bands", "target": [ - "B02_p2", - "B02_p5", - "B02_p25", - "B02_median", - "B02_p75", - "B02_p95", - "B02_p98", - "B02_mean", - "B02_sd", - "B02_sum", - "B02_iqr", - "B02_iqr0595", - "B03_p2", - "B03_p5", - "B03_p25", - "B03_median", - "B03_p75", - "B03_p95", - "B03_p98", - "B03_mean", - "B03_sd", - "B03_sum", - "B03_iqr", - "B03_iqr0595", - "B04_p2", - "B04_p5", - "B04_p25", - "B04_median", - "B04_p75", - "B04_p95", - "B04_p98", - "B04_mean", - "B04_sd", - "B04_sum", - "B04_iqr", - "B04_iqr0595", - "B05_p2", - "B05_p5", - "B05_p25", - "B05_median", - "B05_p75", - "B05_p95", - "B05_p98", - "B05_mean", - "B05_sd", - "B05_sum", - "B05_iqr", - "B05_iqr0595", - "B06_p2", - "B06_p5", - "B06_p25", - "B06_median", - "B06_p75", - "B06_p95", - "B06_p98", - "B06_mean", - "B06_sd", - "B06_sum", - "B06_iqr", - "B06_iqr0595", - "B07_p2", - "B07_p5", - "B07_p25", - "B07_median", - "B07_p75", - "B07_p95", - "B07_p98", - "B07_mean", - "B07_sd", - "B07_sum", - "B07_iqr", - "B07_iqr0595", - "B08_p2", - "B08_p5", - "B08_p25", - "B08_median", - "B08_p75", - "B08_p95", - "B08_p98", - "B08_mean", - "B08_sd", - "B08_sum", - "B08_iqr", - "B08_iqr0595", - "B8A_p2", - "B8A_p5", - "B8A_p25", - "B8A_median", - "B8A_p75", - "B8A_p95", - "B8A_p98", - "B8A_mean", - "B8A_sd", - "B8A_sum", - "B8A_iqr", - "B8A_iqr0595", - "B11_p2", - "B11_p5", - "B11_p25", - "B11_median", - "B11_p75", - "B11_p95", - "B11_p98", - "B11_mean", - "B11_sd", - "B11_sum", - "B11_iqr", - "B11_iqr0595", - "B12_p2", - "B12_p5", - "B12_p25", - "B12_median", - "B12_p75", - "B12_p95", - "B12_p98", - "B12_mean", - "B12_sd", - "B12_sum", - "B12_iqr", - "B12_iqr0595", - "ABDI1_p2", - "ABDI1_p5", - "ABDI1_p25", - "ABDI1_median", - "ABDI1_p75", - "ABDI1_p95", - "ABDI1_p98", - "ABDI1_mean", - "ABDI1_sd", - "ABDI1_sum", - "ABDI1_iqr", - "ABDI1_iqr0595", - "ABDI2_p2", - "ABDI2_p5", - "ABDI2_p25", - "ABDI2_median", - "ABDI2_p75", - "ABDI2_p95", - "ABDI2_p98", - "ABDI2_mean", - "ABDI2_sd", - "ABDI2_sum", - "ABDI2_iqr", - "ABDI2_iqr0595", - "AWEInsh_p2", - "AWEInsh_p5", - "AWEInsh_p25", - "AWEInsh_median", - "AWEInsh_p75", - "AWEInsh_p95", - "AWEInsh_p98", - "AWEInsh_mean", - "AWEInsh_sd", - "AWEInsh_sum", - "AWEInsh_iqr", - "AWEInsh_iqr0595", - "AVI_p2", - "AVI_p5", - "AVI_p25", - "AVI_median", - "AVI_p75", - "AVI_p95", - "AVI_p98", - "AVI_mean", - "AVI_sd", - "AVI_sum", - "AVI_iqr", - "AVI_iqr0595", - "BLFEI_p2", - "BLFEI_p5", - "BLFEI_p25", - "BLFEI_median", - "BLFEI_p75", - "BLFEI_p95", - "BLFEI_p98", - "BLFEI_mean", - "BLFEI_sd", - "BLFEI_sum", - "BLFEI_iqr", - "BLFEI_iqr0595", - "CIRE_p2", - "CIRE_p5", - "CIRE_p25", - "CIRE_median", - "CIRE_p75", - "CIRE_p95", - "CIRE_p98", - "CIRE_mean", - "CIRE_sd", - "CIRE_sum", - "CIRE_iqr", - "CIRE_iqr0595", - "EVI_p2", - "EVI_p5", - "EVI_p25", - "EVI_median", - "EVI_p75", - "EVI_p95", - "EVI_p98", - "EVI_mean", - "EVI_sd", - "EVI_sum", - "EVI_iqr", - "EVI_iqr0595", - "IRECI_p2", - "IRECI_p5", - "IRECI_p25", - "IRECI_median", - "IRECI_p75", - "IRECI_p95", - "IRECI_p98", - "IRECI_mean", - "IRECI_sd", - "IRECI_sum", - "IRECI_iqr", - "IRECI_iqr0595", - "MBWI_p2", - "MBWI_p5", - "MBWI_p25", - "MBWI_median", - "MBWI_p75", - "MBWI_p95", - "MBWI_p98", - "MBWI_mean", - "MBWI_sd", - "MBWI_sum", - "MBWI_iqr", - "MBWI_iqr0595", - "MNDWI_p2", - "MNDWI_p5", - "MNDWI_p25", - "MNDWI_median", - "MNDWI_p75", - "MNDWI_p95", - "MNDWI_p98", - "MNDWI_mean", - "MNDWI_sd", - "MNDWI_sum", - "MNDWI_iqr", - "MNDWI_iqr0595", - "MNDVI_p2", - "MNDVI_p5", - "MNDVI_p25", - "MNDVI_median", - "MNDVI_p75", - "MNDVI_p95", - "MNDVI_p98", - "MNDVI_mean", - "MNDVI_sd", - "MNDVI_sum", - "MNDVI_iqr", - "MNDVI_iqr0595", - "NDMI_p2", - "NDMI_p5", - "NDMI_p25", - "NDMI_median", - "NDMI_p75", - "NDMI_p95", - "NDMI_p98", - "NDMI_mean", - "NDMI_sd", - "NDMI_sum", - "NDMI_iqr", - "NDMI_iqr0595", - "NDVI_p2", - "NDVI_p5", - "NDVI_p25", - "NDVI_median", - "NDVI_p75", - "NDVI_p95", - "NDVI_p98", - "NDVI_mean", - "NDVI_sd", - "NDVI_sum", - "NDVI_iqr", - "NDVI_iqr0595", - "NDVIMNDWI_p2", - "NDVIMNDWI_p5", - "NDVIMNDWI_p25", - "NDVIMNDWI_median", - "NDVIMNDWI_p75", - "NDVIMNDWI_p95", - "NDVIMNDWI_p98", - "NDVIMNDWI_mean", - "NDVIMNDWI_sd", - "NDVIMNDWI_sum", - "NDVIMNDWI_iqr", - "NDVIMNDWI_iqr0595", - "NDWI_p2", - "NDWI_p5", - "NDWI_p25", - "NDWI_median", - "NDWI_p75", - "NDWI_p95", - "NDWI_p98", - "NDWI_mean", - "NDWI_sd", - "NDWI_sum", - "NDWI_iqr", - "NDWI_iqr0595", - "NMDI_p2", - "NMDI_p5", - "NMDI_p25", - "NMDI_median", - "NMDI_p75", - "NMDI_p95", - "NMDI_p98", - "NMDI_mean", - "NMDI_sd", - "NMDI_sum", - "NMDI_iqr", - "NMDI_iqr0595", - "NIRv_p2", - "NIRv_p5", - "NIRv_p25", - "NIRv_median", - "NIRv_p75", - "NIRv_p95", - "NIRv_p98", - "NIRv_mean", - "NIRv_sd", - "NIRv_sum", - "NIRv_iqr", - "NIRv_iqr0595", - "S2WI_p2", - "S2WI_p5", - "S2WI_p25", - "S2WI_median", - "S2WI_p75", - "S2WI_p95", - "S2WI_p98", - "S2WI_mean", - "S2WI_sd", - "S2WI_sum", - "S2WI_iqr", - "S2WI_iqr0595", - "S2REP_p2", - "S2REP_p5", - "S2REP_p25", - "S2REP_median", - "S2REP_p75", - "S2REP_p95", - "S2REP_p98", - "S2REP_mean", - "S2REP_sd", - "S2REP_sum", - "S2REP_iqr", - "S2REP_iqr0595", - "WRI_p2", - "WRI_p5", - "WRI_p25", - "WRI_median", - "WRI_p75", - "WRI_p95", - "WRI_p98", - "WRI_mean", - "WRI_sd", - "WRI_sum", - "WRI_iqr", - "WRI_iqr0595" + "b02_p2", + "b02_p5", + "b02_p25", + "b02_median", + "b02_p75", + "b02_p95", + "b02_p98", + "b02_mean", + "b02_sd", + "b02_sum", + "b02_iqr", + "b02_iqr0595", + "b03_p2", + "b03_p5", + "b03_p25", + "b03_median", + "b03_p75", + "b03_p95", + "b03_p98", + "b03_mean", + "b03_sd", + "b03_sum", + "b03_iqr", + "b03_iqr0595", + "b04_p2", + "b04_p5", + "b04_p25", + "b04_median", + "b04_p75", + "b04_p95", + "b04_p98", + "b04_mean", + "b04_sd", + "b04_sum", + "b04_iqr", + "b04_iqr0595", + "b05_p2", + "b05_p5", + "b05_p25", + "b05_median", + "b05_p75", + "b05_p95", + "b05_p98", + "b05_mean", + "b05_sd", + "b05_sum", + "b05_iqr", + "b05_iqr0595", + "b06_p2", + "b06_p5", + "b06_p25", + "b06_median", + "b06_p75", + "b06_p95", + "b06_p98", + "b06_mean", + "b06_sd", + "b06_sum", + "b06_iqr", + "b06_iqr0595", + "b07_p2", + "b07_p5", + "b07_p25", + "b07_median", + "b07_p75", + "b07_p95", + "b07_p98", + "b07_mean", + "b07_sd", + "b07_sum", + "b07_iqr", + "b07_iqr0595", + "b08_p2", + "b08_p5", + "b08_p25", + "b08_median", + "b08_p75", + "b08_p95", + "b08_p98", + "b08_mean", + "b08_sd", + "b08_sum", + "b08_iqr", + "b08_iqr0595", + "b8a_p2", + "b8a_p5", + "b8a_p25", + "b8a_median", + "b8a_p75", + "b8a_p95", + "b8a_p98", + "b8a_mean", + "b8a_sd", + "b8a_sum", + "b8a_iqr", + "b8a_iqr0595", + "b11_p2", + "b11_p5", + "b11_p25", + "b11_median", + "b11_p75", + "b11_p95", + "b11_p98", + "b11_mean", + "b11_sd", + "b11_sum", + "b11_iqr", + "b11_iqr0595", + "b12_p2", + "b12_p5", + "b12_p25", + "b12_median", + "b12_p75", + "b12_p95", + "b12_p98", + "b12_mean", + "b12_sd", + "b12_sum", + "b12_iqr", + "b12_iqr0595", + "abdi1_p2", + "abdi1_p5", + "abdi1_p25", + "abdi1_median", + "abdi1_p75", + "abdi1_p95", + "abdi1_p98", + "abdi1_mean", + "abdi1_sd", + "abdi1_sum", + "abdi1_iqr", + "abdi1_iqr0595", + "abdi2_p2", + "abdi2_p5", + "abdi2_p25", + "abdi2_median", + "abdi2_p75", + "abdi2_p95", + "abdi2_p98", + "abdi2_mean", + "abdi2_sd", + "abdi2_sum", + "abdi2_iqr", + "abdi2_iqr0595", + "aweinsh_p2", + "aweinsh_p5", + "aweinsh_p25", + "aweinsh_median", + "aweinsh_p75", + "aweinsh_p95", + "aweinsh_p98", + "aweinsh_mean", + "aweinsh_sd", + "aweinsh_sum", + "aweinsh_iqr", + "aweinsh_iqr0595", + "avi_p2", + "avi_p5", + "avi_p25", + "avi_median", + "avi_p75", + "avi_p95", + "avi_p98", + "avi_mean", + "avi_sd", + "avi_sum", + "avi_iqr", + "avi_iqr0595", + "blfei_p2", + "blfei_p5", + "blfei_p25", + "blfei_median", + "blfei_p75", + "blfei_p95", + "blfei_p98", + "blfei_mean", + "blfei_sd", + "blfei_sum", + "blfei_iqr", + "blfei_iqr0595", + "cire_p2", + "cire_p5", + "cire_p25", + "cire_median", + "cire_p75", + "cire_p95", + "cire_p98", + "cire_mean", + "cire_sd", + "cire_sum", + "cire_iqr", + "cire_iqr0595", + "evi_p2", + "evi_p5", + "evi_p25", + "evi_median", + "evi_p75", + "evi_p95", + "evi_p98", + "evi_mean", + "evi_sd", + "evi_sum", + "evi_iqr", + "evi_iqr0595", + "ireci_p2", + "ireci_p5", + "ireci_p25", + "ireci_median", + "ireci_p75", + "ireci_p95", + "ireci_p98", + "ireci_mean", + "ireci_sd", + "ireci_sum", + "ireci_iqr", + "ireci_iqr0595", + "mbwi_p2", + "mbwi_p5", + "mbwi_p25", + "mbwi_median", + "mbwi_p75", + "mbwi_p95", + "mbwi_p98", + "mbwi_mean", + "mbwi_sd", + "mbwi_sum", + "mbwi_iqr", + "mbwi_iqr0595", + "mndwi_p2", + "mndwi_p5", + "mndwi_p25", + "mndwi_median", + "mndwi_p75", + "mndwi_p95", + "mndwi_p98", + "mndwi_mean", + "mndwi_sd", + "mndwi_sum", + "mndwi_iqr", + "mndwi_iqr0595", + "mndvi_p2", + "mndvi_p5", + "mndvi_p25", + "mndvi_median", + "mndvi_p75", + "mndvi_p95", + "mndvi_p98", + "mndvi_mean", + "mndvi_sd", + "mndvi_sum", + "mndvi_iqr", + "mndvi_iqr0595", + "ndmi_p2", + "ndmi_p5", + "ndmi_p25", + "ndmi_median", + "ndmi_p75", + "ndmi_p95", + "ndmi_p98", + "ndmi_mean", + "ndmi_sd", + "ndmi_sum", + "ndmi_iqr", + "ndmi_iqr0595", + "ndvi_p2", + "ndvi_p5", + "ndvi_p25", + "ndvi_median", + "ndvi_p75", + "ndvi_p95", + "ndvi_p98", + "ndvi_mean", + "ndvi_sd", + "ndvi_sum", + "ndvi_iqr", + "ndvi_iqr0595", + "ndvimndwi_p2", + "ndvimndwi_p5", + "ndvimndwi_p25", + "ndvimndwi_median", + "ndvimndwi_p75", + "ndvimndwi_p95", + "ndvimndwi_p98", + "ndvimndwi_mean", + "ndvimndwi_sd", + "ndvimndwi_sum", + "ndvimndwi_iqr", + "ndvimndwi_iqr0595", + "ndwi_p2", + "ndwi_p5", + "ndwi_p25", + "ndwi_median", + "ndwi_p75", + "ndwi_p95", + "ndwi_p98", + "ndwi_mean", + "ndwi_sd", + "ndwi_sum", + "ndwi_iqr", + "ndwi_iqr0595", + "nmdi_p2", + "nmdi_p5", + "nmdi_p25", + "nmdi_median", + "nmdi_p75", + "nmdi_p95", + "nmdi_p98", + "nmdi_mean", + "nmdi_sd", + "nmdi_sum", + "nmdi_iqr", + "nmdi_iqr0595", + "nirv_p2", + "nirv_p5", + "nirv_p25", + "nirv_median", + "nirv_p75", + "nirv_p95", + "nirv_p98", + "nirv_mean", + "nirv_sd", + "nirv_sum", + "nirv_iqr", + "nirv_iqr0595", + "s2wi_p2", + "s2wi_p5", + "s2wi_p25", + "s2wi_median", + "s2wi_p75", + "s2wi_p95", + "s2wi_p98", + "s2wi_mean", + "s2wi_sd", + "s2wi_sum", + "s2wi_iqr", + "s2wi_iqr0595", + "s2rep_p2", + "s2rep_p5", + "s2rep_p25", + "s2rep_median", + "s2rep_p75", + "s2rep_p95", + "s2rep_p98", + "s2rep_mean", + "s2rep_sd", + "s2rep_sum", + "s2rep_iqr", + "s2rep_iqr0595", + "wri_p2", + "wri_p5", + "wri_p25", + "wri_median", + "wri_p75", + "wri_p95", + "wri_p98", + "wri_mean", + "wri_sd", + "wri_sum", + "wri_iqr", + "wri_iqr0595" ] } }, @@ -1791,362 +1790,362 @@ "process_id": "filter_bands", "arguments": { "bands": [ - "B02_p2", - "B02_p5", - "B02_p25", - "B02_median", - "B02_p75", - "B02_p95", - "B02_p98", - "B02_mean", - "B02_sd", - "B02_sum", - "B02_iqr", - "B02_iqr0595", - "B03_p2", - "B03_p5", - "B03_p25", - "B03_median", - "B03_p75", - "B03_p95", - "B03_p98", - "B03_mean", - "B03_sd", - "B03_sum", - "B03_iqr", - "B03_iqr0595", - "B04_p2", - "B04_p5", - "B04_p25", - "B04_median", - "B04_p75", - "B04_p95", - "B04_p98", - "B04_mean", - "B04_sd", - "B04_sum", - "B04_iqr", - "B04_iqr0595", - "B05_p2", - "B05_p5", - "B05_p25", - "B05_median", - "B05_p75", - "B05_p95", - "B05_p98", - "B05_mean", - "B05_sd", - "B05_sum", - "B05_iqr", - "B05_iqr0595", - "B06_p2", - "B06_p5", - "B06_p25", - "B06_median", - "B06_p75", - "B06_p95", - "B06_p98", - "B06_mean", - "B06_sd", - "B06_sum", - "B06_iqr", - "B06_iqr0595", - "B07_p2", - "B07_p5", - "B07_p25", - "B07_median", - "B07_p75", - "B07_p95", - "B07_p98", - "B07_mean", - "B07_sd", - "B07_sum", - "B07_iqr", - "B07_iqr0595", - "B08_p2", - "B08_p5", - "B08_p25", - "B08_median", - "B08_p75", - "B08_p95", - "B08_p98", - "B08_mean", - "B08_sd", - "B08_sum", - "B08_iqr", - "B08_iqr0595", - "B8A_p2", - "B8A_p5", - "B8A_p25", - "B8A_median", - "B8A_p75", - "B8A_p95", - "B8A_p98", - "B8A_mean", - "B8A_sd", - "B8A_sum", - "B8A_iqr", - "B8A_iqr0595", - "B11_p2", - "B11_p5", - "B11_p25", - "B11_median", - "B11_p75", - "B11_p95", - "B11_p98", - "B11_mean", - "B11_sd", - "B11_sum", - "B11_iqr", - "B11_iqr0595", - "B12_p2", - "B12_p5", - "B12_p25", - "B12_median", - "B12_p75", - "B12_p95", - "B12_p98", - "B12_mean", - "B12_sd", - "B12_sum", - "B12_iqr", - "B12_iqr0595", - "ABDI1_p2", - "ABDI1_p5", - "ABDI1_p25", - "ABDI1_median", - "ABDI1_p75", - "ABDI1_p95", - "ABDI1_p98", - "ABDI1_mean", - "ABDI1_sd", - "ABDI1_sum", - "ABDI1_iqr", - "ABDI1_iqr0595", - "ABDI2_p2", - "ABDI2_p5", - "ABDI2_p25", - "ABDI2_median", - "ABDI2_p75", - "ABDI2_p95", - "ABDI2_p98", - "ABDI2_mean", - "ABDI2_sd", - "ABDI2_sum", - "ABDI2_iqr", - "ABDI2_iqr0595", - "AWEInsh_p2", - "AWEInsh_p5", - "AWEInsh_p25", - "AWEInsh_median", - "AWEInsh_p75", - "AWEInsh_p95", - "AWEInsh_p98", - "AWEInsh_mean", - "AWEInsh_sd", - "AWEInsh_sum", - "AWEInsh_iqr", - "AWEInsh_iqr0595", - "AVI_p2", - "AVI_p5", - "AVI_p25", - "AVI_median", - "AVI_p75", - "AVI_p95", - "AVI_p98", - "AVI_mean", - "AVI_sd", - "AVI_sum", - "AVI_iqr", - "AVI_iqr0595", - "BLFEI_p2", - "BLFEI_p5", - "BLFEI_p25", - "BLFEI_median", - "BLFEI_p75", - "BLFEI_p95", - "BLFEI_p98", - "BLFEI_mean", - "BLFEI_sd", - "BLFEI_sum", - "BLFEI_iqr", - "BLFEI_iqr0595", - "CIRE_p2", - "CIRE_p5", - "CIRE_p25", - "CIRE_median", - "CIRE_p75", - "CIRE_p95", - "CIRE_p98", - "CIRE_mean", - "CIRE_sd", - "CIRE_sum", - "CIRE_iqr", - "CIRE_iqr0595", - "EVI_p2", - "EVI_p5", - "EVI_p25", - "EVI_median", - "EVI_p75", - "EVI_p95", - "EVI_p98", - "EVI_mean", - "EVI_sd", - "EVI_sum", - "EVI_iqr", - "EVI_iqr0595", - "IRECI_p2", - "IRECI_p5", - "IRECI_p25", - "IRECI_median", - "IRECI_p75", - "IRECI_p95", - "IRECI_p98", - "IRECI_mean", - "IRECI_sd", - "IRECI_sum", - "IRECI_iqr", - "IRECI_iqr0595", - "MBWI_p2", - "MBWI_p5", - "MBWI_p25", - "MBWI_median", - "MBWI_p75", - "MBWI_p95", - "MBWI_p98", - "MBWI_mean", - "MBWI_sd", - "MBWI_sum", - "MBWI_iqr", - "MBWI_iqr0595", - "MNDWI_p2", - "MNDWI_p5", - "MNDWI_p25", - "MNDWI_median", - "MNDWI_p75", - "MNDWI_p95", - "MNDWI_p98", - "MNDWI_mean", - "MNDWI_sd", - "MNDWI_sum", - "MNDWI_iqr", - "MNDWI_iqr0595", - "MNDVI_p2", - "MNDVI_p5", - "MNDVI_p25", - "MNDVI_median", - "MNDVI_p75", - "MNDVI_p95", - "MNDVI_p98", - "MNDVI_mean", - "MNDVI_sd", - "MNDVI_sum", - "MNDVI_iqr", - "MNDVI_iqr0595", - "NDMI_p2", - "NDMI_p5", - "NDMI_p25", - "NDMI_median", - "NDMI_p75", - "NDMI_p95", - "NDMI_p98", - "NDMI_mean", - "NDMI_sd", - "NDMI_sum", - "NDMI_iqr", - "NDMI_iqr0595", - "NDVI_p2", - "NDVI_p5", - "NDVI_p25", - "NDVI_median", - "NDVI_p75", - "NDVI_p95", - "NDVI_p98", - "NDVI_mean", - "NDVI_sd", - "NDVI_sum", - "NDVI_iqr", - "NDVI_iqr0595", - "NDVIMNDWI_p2", - "NDVIMNDWI_p5", - "NDVIMNDWI_p25", - "NDVIMNDWI_median", - "NDVIMNDWI_p75", - "NDVIMNDWI_p95", - "NDVIMNDWI_p98", - "NDVIMNDWI_mean", - "NDVIMNDWI_sd", - "NDVIMNDWI_sum", - "NDVIMNDWI_iqr", - "NDVIMNDWI_iqr0595", - "NDWI_p2", - "NDWI_p5", - "NDWI_p25", - "NDWI_median", - "NDWI_p75", - "NDWI_p95", - "NDWI_p98", - "NDWI_mean", - "NDWI_sd", - "NDWI_sum", - "NDWI_iqr", - "NDWI_iqr0595", - "NMDI_p2", - "NMDI_p5", - "NMDI_p25", - "NMDI_median", - "NMDI_p75", - "NMDI_p95", - "NMDI_p98", - "NMDI_mean", - "NMDI_sd", - "NMDI_sum", - "NMDI_iqr", - "NMDI_iqr0595", - "NIRv_p2", - "NIRv_p5", - "NIRv_p25", - "NIRv_median", - "NIRv_p75", - "NIRv_p95", - "NIRv_p98", - "NIRv_mean", - "NIRv_sd", - "NIRv_sum", - "NIRv_iqr", - "NIRv_iqr0595", - "S2WI_p2", - "S2WI_p5", - "S2WI_p25", - "S2WI_median", - "S2WI_p75", - "S2WI_p95", - "S2WI_p98", - "S2WI_mean", - "S2WI_sd", - "S2WI_sum", - "S2WI_iqr", - "S2WI_iqr0595", - "S2REP_p2", - "S2REP_p5", - "S2REP_p25", - "S2REP_median", - "S2REP_p75", - "S2REP_p95", - "S2REP_p98", - "S2REP_mean", - "WRI_p2", - "WRI_p5", - "WRI_p25", - "WRI_median", - "WRI_p75", - "WRI_p95", - "WRI_p98", - "WRI_mean", - "WRI_sd", - "WRI_sum", - "WRI_iqr", - "WRI_iqr0595" + "b02_p2", + "b02_p5", + "b02_p25", + "b02_median", + "b02_p75", + "b02_p95", + "b02_p98", + "b02_mean", + "b02_sd", + "b02_sum", + "b02_iqr", + "b02_iqr0595", + "b03_p2", + "b03_p5", + "b03_p25", + "b03_median", + "b03_p75", + "b03_p95", + "b03_p98", + "b03_mean", + "b03_sd", + "b03_sum", + "b03_iqr", + "b03_iqr0595", + "b04_p2", + "b04_p5", + "b04_p25", + "b04_median", + "b04_p75", + "b04_p95", + "b04_p98", + "b04_mean", + "b04_sd", + "b04_sum", + "b04_iqr", + "b04_iqr0595", + "b05_p2", + "b05_p5", + "b05_p25", + "b05_median", + "b05_p75", + "b05_p95", + "b05_p98", + "b05_mean", + "b05_sd", + "b05_sum", + "b05_iqr", + "b05_iqr0595", + "b06_p2", + "b06_p5", + "b06_p25", + "b06_median", + "b06_p75", + "b06_p95", + "b06_p98", + "b06_mean", + "b06_sd", + "b06_sum", + "b06_iqr", + "b06_iqr0595", + "b07_p2", + "b07_p5", + "b07_p25", + "b07_median", + "b07_p75", + "b07_p95", + "b07_p98", + "b07_mean", + "b07_sd", + "b07_sum", + "b07_iqr", + "b07_iqr0595", + "b08_p2", + "b08_p5", + "b08_p25", + "b08_median", + "b08_p75", + "b08_p95", + "b08_p98", + "b08_mean", + "b08_sd", + "b08_sum", + "b08_iqr", + "b08_iqr0595", + "b8a_p2", + "b8a_p5", + "b8a_p25", + "b8a_median", + "b8a_p75", + "b8a_p95", + "b8a_p98", + "b8a_mean", + "b8a_sd", + "b8a_sum", + "b8a_iqr", + "b8a_iqr0595", + "b11_p2", + "b11_p5", + "b11_p25", + "b11_median", + "b11_p75", + "b11_p95", + "b11_p98", + "b11_mean", + "b11_sd", + "b11_sum", + "b11_iqr", + "b11_iqr0595", + "b12_p2", + "b12_p5", + "b12_p25", + "b12_median", + "b12_p75", + "b12_p95", + "b12_p98", + "b12_mean", + "b12_sd", + "b12_sum", + "b12_iqr", + "b12_iqr0595", + "abdi1_p2", + "abdi1_p5", + "abdi1_p25", + "abdi1_median", + "abdi1_p75", + "abdi1_p95", + "abdi1_p98", + "abdi1_mean", + "abdi1_sd", + "abdi1_sum", + "abdi1_iqr", + "abdi1_iqr0595", + "abdi2_p2", + "abdi2_p5", + "abdi2_p25", + "abdi2_median", + "abdi2_p75", + "abdi2_p95", + "abdi2_p98", + "abdi2_mean", + "abdi2_sd", + "abdi2_sum", + "abdi2_iqr", + "abdi2_iqr0595", + "aweinsh_p2", + "aweinsh_p5", + "aweinsh_p25", + "aweinsh_median", + "aweinsh_p75", + "aweinsh_p95", + "aweinsh_p98", + "aweinsh_mean", + "aweinsh_sd", + "aweinsh_sum", + "aweinsh_iqr", + "aweinsh_iqr0595", + "avi_p2", + "avi_p5", + "avi_p25", + "avi_median", + "avi_p75", + "avi_p95", + "avi_p98", + "avi_mean", + "avi_sd", + "avi_sum", + "avi_iqr", + "avi_iqr0595", + "blfei_p2", + "blfei_p5", + "blfei_p25", + "blfei_median", + "blfei_p75", + "blfei_p95", + "blfei_p98", + "blfei_mean", + "blfei_sd", + "blfei_sum", + "blfei_iqr", + "blfei_iqr0595", + "cire_p2", + "cire_p5", + "cire_p25", + "cire_median", + "cire_p75", + "cire_p95", + "cire_p98", + "cire_mean", + "cire_sd", + "cire_sum", + "cire_iqr", + "cire_iqr0595", + "evi_p2", + "evi_p5", + "evi_p25", + "evi_median", + "evi_p75", + "evi_p95", + "evi_p98", + "evi_mean", + "evi_sd", + "evi_sum", + "evi_iqr", + "evi_iqr0595", + "ireci_p2", + "ireci_p5", + "ireci_p25", + "ireci_median", + "ireci_p75", + "ireci_p95", + "ireci_p98", + "ireci_mean", + "ireci_sd", + "ireci_sum", + "ireci_iqr", + "ireci_iqr0595", + "mbwi_p2", + "mbwi_p5", + "mbwi_p25", + "mbwi_median", + "mbwi_p75", + "mbwi_p95", + "mbwi_p98", + "mbwi_mean", + "mbwi_sd", + "mbwi_sum", + "mbwi_iqr", + "mbwi_iqr0595", + "mndwi_p2", + "mndwi_p5", + "mndwi_p25", + "mndwi_median", + "mndwi_p75", + "mndwi_p95", + "mndwi_p98", + "mndwi_mean", + "mndwi_sd", + "mndwi_sum", + "mndwi_iqr", + "mndwi_iqr0595", + "mndvi_p2", + "mndvi_p5", + "mndvi_p25", + "mndvi_median", + "mndvi_p75", + "mndvi_p95", + "mndvi_p98", + "mndvi_mean", + "mndvi_sd", + "mndvi_sum", + "mndvi_iqr", + "mndvi_iqr0595", + "ndmi_p2", + "ndmi_p5", + "ndmi_p25", + "ndmi_median", + "ndmi_p75", + "ndmi_p95", + "ndmi_p98", + "ndmi_mean", + "ndmi_sd", + "ndmi_sum", + "ndmi_iqr", + "ndmi_iqr0595", + "ndvi_p2", + "ndvi_p5", + "ndvi_p25", + "ndvi_median", + "ndvi_p75", + "ndvi_p95", + "ndvi_p98", + "ndvi_mean", + "ndvi_sd", + "ndvi_sum", + "ndvi_iqr", + "ndvi_iqr0595", + "ndvimndwi_p2", + "ndvimndwi_p5", + "ndvimndwi_p25", + "ndvimndwi_median", + "ndvimndwi_p75", + "ndvimndwi_p95", + "ndvimndwi_p98", + "ndvimndwi_mean", + "ndvimndwi_sd", + "ndvimndwi_sum", + "ndvimndwi_iqr", + "ndvimndwi_iqr0595", + "ndwi_p2", + "ndwi_p5", + "ndwi_p25", + "ndwi_median", + "ndwi_p75", + "ndwi_p95", + "ndwi_p98", + "ndwi_mean", + "ndwi_sd", + "ndwi_sum", + "ndwi_iqr", + "ndwi_iqr0595", + "nmdi_p2", + "nmdi_p5", + "nmdi_p25", + "nmdi_median", + "nmdi_p75", + "nmdi_p95", + "nmdi_p98", + "nmdi_mean", + "nmdi_sd", + "nmdi_sum", + "nmdi_iqr", + "nmdi_iqr0595", + "nirv_p2", + "nirv_p5", + "nirv_p25", + "nirv_median", + "nirv_p75", + "nirv_p95", + "nirv_p98", + "nirv_mean", + "nirv_sd", + "nirv_sum", + "nirv_iqr", + "nirv_iqr0595", + "s2wi_p2", + "s2wi_p5", + "s2wi_p25", + "s2wi_median", + "s2wi_p75", + "s2wi_p95", + "s2wi_p98", + "s2wi_mean", + "s2wi_sd", + "s2wi_sum", + "s2wi_iqr", + "s2wi_iqr0595", + "s2rep_p2", + "s2rep_p5", + "s2rep_p25", + "s2rep_median", + "s2rep_p75", + "s2rep_p95", + "s2rep_p98", + "s2rep_mean", + "wri_p2", + "wri_p5", + "wri_p25", + "wri_median", + "wri_p75", + "wri_p95", + "wri_p98", + "wri_mean", + "wri_sd", + "wri_sum", + "wri_iqr", + "wri_iqr0595" ], "data": { "from_node": "renamelabels2" diff --git a/tests/resources/cube_generation/generate_master_feature_cube_with_catboost_inference.json b/tests/resources/cube_generation/generate_master_feature_cube_with_catboost_inference.json index d04e877..9b88482 100644 --- a/tests/resources/cube_generation/generate_master_feature_cube_with_catboost_inference.json +++ b/tests/resources/cube_generation/generate_master_feature_cube_with_catboost_inference.json @@ -1265,6 +1265,893 @@ ] } }, + "applydimension2": { + "process_id": "apply_dimension", + "arguments": { + "context": { + "TileSize": 128 + }, + "data": { + "from_node": "renamelabels1" + }, + "dimension": "t", + "process": { + "process_graph": { + "quantiles1": { + "process_id": "quantiles", + "arguments": { + "data": { + "from_parameter": "data" + }, + "probabilities": [ + 0.02, + 0.05, + 0.25, + 0.5, + 0.75, + 0.95, + 0.98 + ] + } + }, + "mean1": { + "process_id": "mean", + "arguments": { + "data": { + "from_parameter": "data" + } + } + }, + "sd1": { + "process_id": "sd", + "arguments": { + "data": { + "from_parameter": "data" + } + } + }, + "sum1": { + "process_id": "sum", + "arguments": { + "data": { + "from_parameter": "data" + } + } + }, + "quantiles2": { + "process_id": "quantiles", + "arguments": { + "data": { + "from_parameter": "data" + }, + "probabilities": [ + 0.75 + ] + } + }, + "quantiles3": { + "process_id": "quantiles", + "arguments": { + "data": { + "from_parameter": "data" + }, + "probabilities": [ + 0.25 + ] + } + }, + "subtract31": { + "process_id": "subtract", + "arguments": { + "x": { + "from_node": "quantiles2" + }, + "y": { + "from_node": "quantiles3" + } + } + }, + "quantiles4": { + "process_id": "quantiles", + "arguments": { + "data": { + "from_parameter": "data" + }, + "probabilities": [ + 0.95 + ] + } + }, + "quantiles5": { + "process_id": "quantiles", + "arguments": { + "data": { + "from_parameter": "data" + }, + "probabilities": [ + 0.05 + ] + } + }, + "subtract32": { + "process_id": "subtract", + "arguments": { + "x": { + "from_node": "quantiles4" + }, + "y": { + "from_node": "quantiles5" + } + } + }, + "arrayconcat1": { + "process_id": "array_concat", + "arguments": { + "array1": { + "from_node": "quantiles1" + }, + "array2": [ + { + "from_node": "mean1" + }, + { + "from_node": "sd1" + }, + { + "from_node": "sum1" + }, + { + "from_node": "subtract31" + }, + { + "from_node": "subtract32" + } + ] + }, + "result": true + } + } + }, + "target_dimension": "bands" + } + }, + "renamelabels2": { + "process_id": "rename_labels", + "arguments": { + "data": { + "from_node": "applydimension2" + }, + "dimension": "bands", + "target": [ + "b02_p2", + "b02_p5", + "b02_p25", + "b02_median", + "b02_p75", + "b02_p95", + "b02_p98", + "b02_mean", + "b02_sd", + "b02_sum", + "b02_iqr", + "b02_iqr0595", + "b03_p2", + "b03_p5", + "b03_p25", + "b03_median", + "b03_p75", + "b03_p95", + "b03_p98", + "b03_mean", + "b03_sd", + "b03_sum", + "b03_iqr", + "b03_iqr0595", + "b04_p2", + "b04_p5", + "b04_p25", + "b04_median", + "b04_p75", + "b04_p95", + "b04_p98", + "b04_mean", + "b04_sd", + "b04_sum", + "b04_iqr", + "b04_iqr0595", + "b05_p2", + "b05_p5", + "b05_p25", + "b05_median", + "b05_p75", + "b05_p95", + "b05_p98", + "b05_mean", + "b05_sd", + "b05_sum", + "b05_iqr", + "b05_iqr0595", + "b06_p2", + "b06_p5", + "b06_p25", + "b06_median", + "b06_p75", + "b06_p95", + "b06_p98", + "b06_mean", + "b06_sd", + "b06_sum", + "b06_iqr", + "b06_iqr0595", + "b07_p2", + "b07_p5", + "b07_p25", + "b07_median", + "b07_p75", + "b07_p95", + "b07_p98", + "b07_mean", + "b07_sd", + "b07_sum", + "b07_iqr", + "b07_iqr0595", + "b08_p2", + "b08_p5", + "b08_p25", + "b08_median", + "b08_p75", + "b08_p95", + "b08_p98", + "b08_mean", + "b08_sd", + "b08_sum", + "b08_iqr", + "b08_iqr0595", + "b8a_p2", + "b8a_p5", + "b8a_p25", + "b8a_median", + "b8a_p75", + "b8a_p95", + "b8a_p98", + "b8a_mean", + "b8a_sd", + "b8a_sum", + "b8a_iqr", + "b8a_iqr0595", + "b11_p2", + "b11_p5", + "b11_p25", + "b11_median", + "b11_p75", + "b11_p95", + "b11_p98", + "b11_mean", + "b11_sd", + "b11_sum", + "b11_iqr", + "b11_iqr0595", + "b12_p2", + "b12_p5", + "b12_p25", + "b12_median", + "b12_p75", + "b12_p95", + "b12_p98", + "b12_mean", + "b12_sd", + "b12_sum", + "b12_iqr", + "b12_iqr0595", + "abdi1_p2", + "abdi1_p5", + "abdi1_p25", + "abdi1_median", + "abdi1_p75", + "abdi1_p95", + "abdi1_p98", + "abdi1_mean", + "abdi1_sd", + "abdi1_sum", + "abdi1_iqr", + "abdi1_iqr0595", + "abdi2_p2", + "abdi2_p5", + "abdi2_p25", + "abdi2_median", + "abdi2_p75", + "abdi2_p95", + "abdi2_p98", + "abdi2_mean", + "abdi2_sd", + "abdi2_sum", + "abdi2_iqr", + "abdi2_iqr0595", + "aweinsh_p2", + "aweinsh_p5", + "aweinsh_p25", + "aweinsh_median", + "aweinsh_p75", + "aweinsh_p95", + "aweinsh_p98", + "aweinsh_mean", + "aweinsh_sd", + "aweinsh_sum", + "aweinsh_iqr", + "aweinsh_iqr0595", + "avi_p2", + "avi_p5", + "avi_p25", + "avi_median", + "avi_p75", + "avi_p95", + "avi_p98", + "avi_mean", + "avi_sd", + "avi_sum", + "avi_iqr", + "avi_iqr0595", + "blfei_p2", + "blfei_p5", + "blfei_p25", + "blfei_median", + "blfei_p75", + "blfei_p95", + "blfei_p98", + "blfei_mean", + "blfei_sd", + "blfei_sum", + "blfei_iqr", + "blfei_iqr0595", + "cire_p2", + "cire_p5", + "cire_p25", + "cire_median", + "cire_p75", + "cire_p95", + "cire_p98", + "cire_mean", + "cire_sd", + "cire_sum", + "cire_iqr", + "cire_iqr0595", + "evi_p2", + "evi_p5", + "evi_p25", + "evi_median", + "evi_p75", + "evi_p95", + "evi_p98", + "evi_mean", + "evi_sd", + "evi_sum", + "evi_iqr", + "evi_iqr0595", + "ireci_p2", + "ireci_p5", + "ireci_p25", + "ireci_median", + "ireci_p75", + "ireci_p95", + "ireci_p98", + "ireci_mean", + "ireci_sd", + "ireci_sum", + "ireci_iqr", + "ireci_iqr0595", + "mbwi_p2", + "mbwi_p5", + "mbwi_p25", + "mbwi_median", + "mbwi_p75", + "mbwi_p95", + "mbwi_p98", + "mbwi_mean", + "mbwi_sd", + "mbwi_sum", + "mbwi_iqr", + "mbwi_iqr0595", + "mndwi_p2", + "mndwi_p5", + "mndwi_p25", + "mndwi_median", + "mndwi_p75", + "mndwi_p95", + "mndwi_p98", + "mndwi_mean", + "mndwi_sd", + "mndwi_sum", + "mndwi_iqr", + "mndwi_iqr0595", + "mndvi_p2", + "mndvi_p5", + "mndvi_p25", + "mndvi_median", + "mndvi_p75", + "mndvi_p95", + "mndvi_p98", + "mndvi_mean", + "mndvi_sd", + "mndvi_sum", + "mndvi_iqr", + "mndvi_iqr0595", + "ndmi_p2", + "ndmi_p5", + "ndmi_p25", + "ndmi_median", + "ndmi_p75", + "ndmi_p95", + "ndmi_p98", + "ndmi_mean", + "ndmi_sd", + "ndmi_sum", + "ndmi_iqr", + "ndmi_iqr0595", + "ndvi_p2", + "ndvi_p5", + "ndvi_p25", + "ndvi_median", + "ndvi_p75", + "ndvi_p95", + "ndvi_p98", + "ndvi_mean", + "ndvi_sd", + "ndvi_sum", + "ndvi_iqr", + "ndvi_iqr0595", + "ndvimndwi_p2", + "ndvimndwi_p5", + "ndvimndwi_p25", + "ndvimndwi_median", + "ndvimndwi_p75", + "ndvimndwi_p95", + "ndvimndwi_p98", + "ndvimndwi_mean", + "ndvimndwi_sd", + "ndvimndwi_sum", + "ndvimndwi_iqr", + "ndvimndwi_iqr0595", + "ndwi_p2", + "ndwi_p5", + "ndwi_p25", + "ndwi_median", + "ndwi_p75", + "ndwi_p95", + "ndwi_p98", + "ndwi_mean", + "ndwi_sd", + "ndwi_sum", + "ndwi_iqr", + "ndwi_iqr0595", + "nmdi_p2", + "nmdi_p5", + "nmdi_p25", + "nmdi_median", + "nmdi_p75", + "nmdi_p95", + "nmdi_p98", + "nmdi_mean", + "nmdi_sd", + "nmdi_sum", + "nmdi_iqr", + "nmdi_iqr0595", + "nirv_p2", + "nirv_p5", + "nirv_p25", + "nirv_median", + "nirv_p75", + "nirv_p95", + "nirv_p98", + "nirv_mean", + "nirv_sd", + "nirv_sum", + "nirv_iqr", + "nirv_iqr0595", + "s2wi_p2", + "s2wi_p5", + "s2wi_p25", + "s2wi_median", + "s2wi_p75", + "s2wi_p95", + "s2wi_p98", + "s2wi_mean", + "s2wi_sd", + "s2wi_sum", + "s2wi_iqr", + "s2wi_iqr0595", + "s2rep_p2", + "s2rep_p5", + "s2rep_p25", + "s2rep_median", + "s2rep_p75", + "s2rep_p95", + "s2rep_p98", + "s2rep_mean", + "s2rep_sd", + "s2rep_sum", + "s2rep_iqr", + "s2rep_iqr0595", + "wri_p2", + "wri_p5", + "wri_p25", + "wri_median", + "wri_p75", + "wri_p95", + "wri_p98", + "wri_mean", + "wri_sd", + "wri_sum", + "wri_iqr", + "wri_iqr0595" + ] + } + }, + "filterbands1": { + "process_id": "filter_bands", + "arguments": { + "bands": [ + "b02_p2", + "b02_p5", + "b02_p25", + "b02_median", + "b02_p75", + "b02_p95", + "b02_p98", + "b02_mean", + "b02_sd", + "b02_sum", + "b02_iqr", + "b02_iqr0595", + "b03_p2", + "b03_p5", + "b03_p25", + "b03_median", + "b03_p75", + "b03_p95", + "b03_p98", + "b03_mean", + "b03_sd", + "b03_sum", + "b03_iqr", + "b03_iqr0595", + "b04_p2", + "b04_p5", + "b04_p25", + "b04_median", + "b04_p75", + "b04_p95", + "b04_p98", + "b04_mean", + "b04_sd", + "b04_sum", + "b04_iqr", + "b04_iqr0595", + "b05_p2", + "b05_p5", + "b05_p25", + "b05_median", + "b05_p75", + "b05_p95", + "b05_p98", + "b05_mean", + "b05_sd", + "b05_sum", + "b05_iqr", + "b05_iqr0595", + "b06_p2", + "b06_p5", + "b06_p25", + "b06_median", + "b06_p75", + "b06_p95", + "b06_p98", + "b06_mean", + "b06_sd", + "b06_sum", + "b06_iqr", + "b06_iqr0595", + "b07_p2", + "b07_p5", + "b07_p25", + "b07_median", + "b07_p75", + "b07_p95", + "b07_p98", + "b07_mean", + "b07_sd", + "b07_sum", + "b07_iqr", + "b07_iqr0595", + "b08_p2", + "b08_p5", + "b08_p25", + "b08_median", + "b08_p75", + "b08_p95", + "b08_p98", + "b08_mean", + "b08_sd", + "b08_sum", + "b08_iqr", + "b08_iqr0595", + "b8a_p2", + "b8a_p5", + "b8a_p25", + "b8a_median", + "b8a_p75", + "b8a_p95", + "b8a_p98", + "b8a_mean", + "b8a_sd", + "b8a_sum", + "b8a_iqr", + "b8a_iqr0595", + "b11_p2", + "b11_p5", + "b11_p25", + "b11_median", + "b11_p75", + "b11_p95", + "b11_p98", + "b11_mean", + "b11_sd", + "b11_sum", + "b11_iqr", + "b11_iqr0595", + "b12_p2", + "b12_p5", + "b12_p25", + "b12_median", + "b12_p75", + "b12_p95", + "b12_p98", + "b12_mean", + "b12_sd", + "b12_sum", + "b12_iqr", + "b12_iqr0595", + "abdi1_p2", + "abdi1_p5", + "abdi1_p25", + "abdi1_median", + "abdi1_p75", + "abdi1_p95", + "abdi1_p98", + "abdi1_mean", + "abdi1_sd", + "abdi1_sum", + "abdi1_iqr", + "abdi1_iqr0595", + "abdi2_p2", + "abdi2_p5", + "abdi2_p25", + "abdi2_median", + "abdi2_p75", + "abdi2_p95", + "abdi2_p98", + "abdi2_mean", + "abdi2_sd", + "abdi2_sum", + "abdi2_iqr", + "abdi2_iqr0595", + "aweinsh_p2", + "aweinsh_p5", + "aweinsh_p25", + "aweinsh_median", + "aweinsh_p75", + "aweinsh_p95", + "aweinsh_p98", + "aweinsh_mean", + "aweinsh_sd", + "aweinsh_sum", + "aweinsh_iqr", + "aweinsh_iqr0595", + "avi_p2", + "avi_p5", + "avi_p25", + "avi_median", + "avi_p75", + "avi_p95", + "avi_p98", + "avi_mean", + "avi_sd", + "avi_sum", + "avi_iqr", + "avi_iqr0595", + "blfei_p2", + "blfei_p5", + "blfei_p25", + "blfei_median", + "blfei_p75", + "blfei_p95", + "blfei_p98", + "blfei_mean", + "blfei_sd", + "blfei_sum", + "blfei_iqr", + "blfei_iqr0595", + "cire_p2", + "cire_p5", + "cire_p25", + "cire_median", + "cire_p75", + "cire_p95", + "cire_p98", + "cire_mean", + "cire_sd", + "cire_sum", + "cire_iqr", + "cire_iqr0595", + "evi_p2", + "evi_p5", + "evi_p25", + "evi_median", + "evi_p75", + "evi_p95", + "evi_p98", + "evi_mean", + "evi_sd", + "evi_sum", + "evi_iqr", + "evi_iqr0595", + "ireci_p2", + "ireci_p5", + "ireci_p25", + "ireci_median", + "ireci_p75", + "ireci_p95", + "ireci_p98", + "ireci_mean", + "ireci_sd", + "ireci_sum", + "ireci_iqr", + "ireci_iqr0595", + "mbwi_p2", + "mbwi_p5", + "mbwi_p25", + "mbwi_median", + "mbwi_p75", + "mbwi_p95", + "mbwi_p98", + "mbwi_mean", + "mbwi_sd", + "mbwi_sum", + "mbwi_iqr", + "mbwi_iqr0595", + "mndwi_p2", + "mndwi_p5", + "mndwi_p25", + "mndwi_median", + "mndwi_p75", + "mndwi_p95", + "mndwi_p98", + "mndwi_mean", + "mndwi_sd", + "mndwi_sum", + "mndwi_iqr", + "mndwi_iqr0595", + "mndvi_p2", + "mndvi_p5", + "mndvi_p25", + "mndvi_median", + "mndvi_p75", + "mndvi_p95", + "mndvi_p98", + "mndvi_mean", + "mndvi_sd", + "mndvi_sum", + "mndvi_iqr", + "mndvi_iqr0595", + "ndmi_p2", + "ndmi_p5", + "ndmi_p25", + "ndmi_median", + "ndmi_p75", + "ndmi_p95", + "ndmi_p98", + "ndmi_mean", + "ndmi_sd", + "ndmi_sum", + "ndmi_iqr", + "ndmi_iqr0595", + "ndvi_p2", + "ndvi_p5", + "ndvi_p25", + "ndvi_median", + "ndvi_p75", + "ndvi_p95", + "ndvi_p98", + "ndvi_mean", + "ndvi_sd", + "ndvi_sum", + "ndvi_iqr", + "ndvi_iqr0595", + "ndvimndwi_p2", + "ndvimndwi_p5", + "ndvimndwi_p25", + "ndvimndwi_median", + "ndvimndwi_p75", + "ndvimndwi_p95", + "ndvimndwi_p98", + "ndvimndwi_mean", + "ndvimndwi_sd", + "ndvimndwi_sum", + "ndvimndwi_iqr", + "ndvimndwi_iqr0595", + "ndwi_p2", + "ndwi_p5", + "ndwi_p25", + "ndwi_median", + "ndwi_p75", + "ndwi_p95", + "ndwi_p98", + "ndwi_mean", + "ndwi_sd", + "ndwi_sum", + "ndwi_iqr", + "ndwi_iqr0595", + "nmdi_p2", + "nmdi_p5", + "nmdi_p25", + "nmdi_median", + "nmdi_p75", + "nmdi_p95", + "nmdi_p98", + "nmdi_mean", + "nmdi_sd", + "nmdi_sum", + "nmdi_iqr", + "nmdi_iqr0595", + "nirv_p2", + "nirv_p5", + "nirv_p25", + "nirv_median", + "nirv_p75", + "nirv_p95", + "nirv_p98", + "nirv_mean", + "nirv_sd", + "nirv_sum", + "nirv_iqr", + "nirv_iqr0595", + "s2wi_p2", + "s2wi_p5", + "s2wi_p25", + "s2wi_median", + "s2wi_p75", + "s2wi_p95", + "s2wi_p98", + "s2wi_mean", + "s2wi_sd", + "s2wi_sum", + "s2wi_iqr", + "s2wi_iqr0595", + "s2rep_p2", + "s2rep_p5", + "s2rep_p25", + "s2rep_median", + "s2rep_p75", + "s2rep_p95", + "s2rep_p98", + "s2rep_mean", + "wri_p2", + "wri_p5", + "wri_p25", + "wri_median", + "wri_p75", + "wri_p95", + "wri_p98", + "wri_mean", + "wri_sd", + "wri_sum", + "wri_iqr", + "wri_iqr0595" + ], + "data": { + "from_node": "renamelabels2" + } + } + }, "loadcollection2": { "process_id": "load_collection", "arguments": { @@ -1277,7 +2164,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -1285,7 +2172,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true } @@ -1321,7 +2211,6 @@ "options": { "implementation_version": "2", "tile_size": 128, - "otb_memory": 1024, "debug": false, "elev_geoid": "/opt/openeo-vito-aux-data/egm96.tif" } @@ -1339,7 +2228,7 @@ "resolution": 10.0 } }, - "applydimension2": { + "applydimension3": { "process_id": "apply_dimension", "arguments": { "data": { @@ -1536,7 +2425,7 @@ "process_id": "apply", "arguments": { "data": { - "from_node": "applydimension2" + "from_node": "applydimension3" }, "process": { "process_graph": { @@ -1557,7 +2446,7 @@ } } }, - "applydimension3": { + "applydimension4": { "process_id": "apply_dimension", "arguments": { "data": { @@ -1593,7 +2482,7 @@ } } }, - "subtract31": { + "subtract33": { "process_id": "subtract", "arguments": { "x": { @@ -1629,7 +2518,7 @@ } } }, - "subtract32": { + "subtract34": { "process_id": "subtract", "arguments": { "x": { @@ -1643,10 +2532,10 @@ "arguments": { "data": [ { - "from_node": "subtract31" + "from_node": "subtract33" }, { - "from_node": "subtract32" + "from_node": "subtract34" } ] }, @@ -1656,11 +2545,11 @@ } } }, - "applydimension4": { + "applydimension5": { "process_id": "apply_dimension", "arguments": { "data": { - "from_node": "applydimension3" + "from_node": "applydimension4" }, "dimension": "bands", "process": { @@ -1683,7 +2572,7 @@ "index": 1 } }, - "subtract33": { + "subtract35": { "process_id": "subtract", "arguments": { "x": { @@ -1745,7 +2634,7 @@ "index": 2, "values": [ { - "from_node": "subtract33" + "from_node": "subtract35" }, { "from_node": "divide25" @@ -1761,11 +2650,11 @@ } } }, - "renamelabels2": { + "renamelabels3": { "process_id": "rename_labels", "arguments": { "data": { - "from_node": "applydimension4" + "from_node": "applydimension5" }, "dimension": "bands", "target": [ @@ -1777,31 +2666,19 @@ ] } }, - "mergecubes1": { - "process_id": "merge_cubes", - "arguments": { - "cube1": { - "from_node": "renamelabels1" - }, - "cube2": { - "from_node": "renamelabels2" - } - } - }, - "applydimension5": { + "applydimension6": { "process_id": "apply_dimension", "arguments": { "context": { - "parallel": true, "TileSize": 128 }, "data": { - "from_node": "mergecubes1" + "from_node": "renamelabels3" }, "dimension": "t", "process": { "process_graph": { - "quantiles1": { + "quantiles6": { "process_id": "quantiles", "arguments": { "data": { @@ -1818,7 +2695,7 @@ ] } }, - "mean1": { + "mean2": { "process_id": "mean", "arguments": { "data": { @@ -1826,7 +2703,7 @@ } } }, - "sd1": { + "sd2": { "process_id": "sd", "arguments": { "data": { @@ -1834,7 +2711,7 @@ } } }, - "sum1": { + "sum2": { "process_id": "sum", "arguments": { "data": { @@ -1842,7 +2719,7 @@ } } }, - "quantiles2": { + "quantiles7": { "process_id": "quantiles", "arguments": { "data": { @@ -1853,7 +2730,7 @@ ] } }, - "quantiles3": { + "quantiles8": { "process_id": "quantiles", "arguments": { "data": { @@ -1864,18 +2741,18 @@ ] } }, - "subtract34": { + "subtract36": { "process_id": "subtract", "arguments": { "x": { - "from_node": "quantiles2" + "from_node": "quantiles7" }, "y": { - "from_node": "quantiles3" + "from_node": "quantiles8" } } }, - "quantiles4": { + "quantiles9": { "process_id": "quantiles", "arguments": { "data": { @@ -1886,7 +2763,7 @@ ] } }, - "quantiles5": { + "quantiles10": { "process_id": "quantiles", "arguments": { "data": { @@ -1897,38 +2774,38 @@ ] } }, - "subtract35": { + "subtract37": { "process_id": "subtract", "arguments": { "x": { - "from_node": "quantiles4" + "from_node": "quantiles9" }, "y": { - "from_node": "quantiles5" + "from_node": "quantiles10" } } }, - "arrayconcat1": { + "arrayconcat2": { "process_id": "array_concat", "arguments": { "array1": { - "from_node": "quantiles1" + "from_node": "quantiles6" }, "array2": [ { - "from_node": "mean1" + "from_node": "mean2" }, { - "from_node": "sd1" + "from_node": "sd2" }, { - "from_node": "sum1" + "from_node": "sum2" }, { - "from_node": "subtract34" + "from_node": "subtract36" }, { - "from_node": "subtract35" + "from_node": "subtract37" } ] }, @@ -1939,857 +2816,152 @@ "target_dimension": "bands" } }, - "renamelabels3": { + "renamelabels4": { "process_id": "rename_labels", "arguments": { "data": { - "from_node": "applydimension5" + "from_node": "applydimension6" }, "dimension": "bands", "target": [ - "B02_p2", - "B02_p5", - "B02_p25", - "B02_median", - "B02_p75", - "B02_p95", - "B02_p98", - "B02_mean", - "B02_sd", - "B02_sum", - "B02_iqr", - "B02_iqr0595", - "B03_p2", - "B03_p5", - "B03_p25", - "B03_median", - "B03_p75", - "B03_p95", - "B03_p98", - "B03_mean", - "B03_sd", - "B03_sum", - "B03_iqr", - "B03_iqr0595", - "B04_p2", - "B04_p5", - "B04_p25", - "B04_median", - "B04_p75", - "B04_p95", - "B04_p98", - "B04_mean", - "B04_sd", - "B04_sum", - "B04_iqr", - "B04_iqr0595", - "B05_p2", - "B05_p5", - "B05_p25", - "B05_median", - "B05_p75", - "B05_p95", - "B05_p98", - "B05_mean", - "B05_sd", - "B05_sum", - "B05_iqr", - "B05_iqr0595", - "B06_p2", - "B06_p5", - "B06_p25", - "B06_median", - "B06_p75", - "B06_p95", - "B06_p98", - "B06_mean", - "B06_sd", - "B06_sum", - "B06_iqr", - "B06_iqr0595", - "B07_p2", - "B07_p5", - "B07_p25", - "B07_median", - "B07_p75", - "B07_p95", - "B07_p98", - "B07_mean", - "B07_sd", - "B07_sum", - "B07_iqr", - "B07_iqr0595", - "B08_p2", - "B08_p5", - "B08_p25", - "B08_median", - "B08_p75", - "B08_p95", - "B08_p98", - "B08_mean", - "B08_sd", - "B08_sum", - "B08_iqr", - "B08_iqr0595", - "B8A_p2", - "B8A_p5", - "B8A_p25", - "B8A_median", - "B8A_p75", - "B8A_p95", - "B8A_p98", - "B8A_mean", - "B8A_sd", - "B8A_sum", - "B8A_iqr", - "B8A_iqr0595", - "B11_p2", - "B11_p5", - "B11_p25", - "B11_median", - "B11_p75", - "B11_p95", - "B11_p98", - "B11_mean", - "B11_sd", - "B11_sum", - "B11_iqr", - "B11_iqr0595", - "B12_p2", - "B12_p5", - "B12_p25", - "B12_median", - "B12_p75", - "B12_p95", - "B12_p98", - "B12_mean", - "B12_sd", - "B12_sum", - "B12_iqr", - "B12_iqr0595", - "ABDI1_p2", - "ABDI1_p5", - "ABDI1_p25", - "ABDI1_median", - "ABDI1_p75", - "ABDI1_p95", - "ABDI1_p98", - "ABDI1_mean", - "ABDI1_sd", - "ABDI1_sum", - "ABDI1_iqr", - "ABDI1_iqr0595", - "ABDI2_p2", - "ABDI2_p5", - "ABDI2_p25", - "ABDI2_median", - "ABDI2_p75", - "ABDI2_p95", - "ABDI2_p98", - "ABDI2_mean", - "ABDI2_sd", - "ABDI2_sum", - "ABDI2_iqr", - "ABDI2_iqr0595", - "AWEInsh_p2", - "AWEInsh_p5", - "AWEInsh_p25", - "AWEInsh_median", - "AWEInsh_p75", - "AWEInsh_p95", - "AWEInsh_p98", - "AWEInsh_mean", - "AWEInsh_sd", - "AWEInsh_sum", - "AWEInsh_iqr", - "AWEInsh_iqr0595", - "AVI_p2", - "AVI_p5", - "AVI_p25", - "AVI_median", - "AVI_p75", - "AVI_p95", - "AVI_p98", - "AVI_mean", - "AVI_sd", - "AVI_sum", - "AVI_iqr", - "AVI_iqr0595", - "BLFEI_p2", - "BLFEI_p5", - "BLFEI_p25", - "BLFEI_median", - "BLFEI_p75", - "BLFEI_p95", - "BLFEI_p98", - "BLFEI_mean", - "BLFEI_sd", - "BLFEI_sum", - "BLFEI_iqr", - "BLFEI_iqr0595", - "CIRE_p2", - "CIRE_p5", - "CIRE_p25", - "CIRE_median", - "CIRE_p75", - "CIRE_p95", - "CIRE_p98", - "CIRE_mean", - "CIRE_sd", - "CIRE_sum", - "CIRE_iqr", - "CIRE_iqr0595", - "EVI_p2", - "EVI_p5", - "EVI_p25", - "EVI_median", - "EVI_p75", - "EVI_p95", - "EVI_p98", - "EVI_mean", - "EVI_sd", - "EVI_sum", - "EVI_iqr", - "EVI_iqr0595", - "IRECI_p2", - "IRECI_p5", - "IRECI_p25", - "IRECI_median", - "IRECI_p75", - "IRECI_p95", - "IRECI_p98", - "IRECI_mean", - "IRECI_sd", - "IRECI_sum", - "IRECI_iqr", - "IRECI_iqr0595", - "MBWI_p2", - "MBWI_p5", - "MBWI_p25", - "MBWI_median", - "MBWI_p75", - "MBWI_p95", - "MBWI_p98", - "MBWI_mean", - "MBWI_sd", - "MBWI_sum", - "MBWI_iqr", - "MBWI_iqr0595", - "MNDWI_p2", - "MNDWI_p5", - "MNDWI_p25", - "MNDWI_median", - "MNDWI_p75", - "MNDWI_p95", - "MNDWI_p98", - "MNDWI_mean", - "MNDWI_sd", - "MNDWI_sum", - "MNDWI_iqr", - "MNDWI_iqr0595", - "MNDVI_p2", - "MNDVI_p5", - "MNDVI_p25", - "MNDVI_median", - "MNDVI_p75", - "MNDVI_p95", - "MNDVI_p98", - "MNDVI_mean", - "MNDVI_sd", - "MNDVI_sum", - "MNDVI_iqr", - "MNDVI_iqr0595", - "NDMI_p2", - "NDMI_p5", - "NDMI_p25", - "NDMI_median", - "NDMI_p75", - "NDMI_p95", - "NDMI_p98", - "NDMI_mean", - "NDMI_sd", - "NDMI_sum", - "NDMI_iqr", - "NDMI_iqr0595", - "NDVI_p2", - "NDVI_p5", - "NDVI_p25", - "NDVI_median", - "NDVI_p75", - "NDVI_p95", - "NDVI_p98", - "NDVI_mean", - "NDVI_sd", - "NDVI_sum", - "NDVI_iqr", - "NDVI_iqr0595", - "NDVIMNDWI_p2", - "NDVIMNDWI_p5", - "NDVIMNDWI_p25", - "NDVIMNDWI_median", - "NDVIMNDWI_p75", - "NDVIMNDWI_p95", - "NDVIMNDWI_p98", - "NDVIMNDWI_mean", - "NDVIMNDWI_sd", - "NDVIMNDWI_sum", - "NDVIMNDWI_iqr", - "NDVIMNDWI_iqr0595", - "NDWI_p2", - "NDWI_p5", - "NDWI_p25", - "NDWI_median", - "NDWI_p75", - "NDWI_p95", - "NDWI_p98", - "NDWI_mean", - "NDWI_sd", - "NDWI_sum", - "NDWI_iqr", - "NDWI_iqr0595", - "NMDI_p2", - "NMDI_p5", - "NMDI_p25", - "NMDI_median", - "NMDI_p75", - "NMDI_p95", - "NMDI_p98", - "NMDI_mean", - "NMDI_sd", - "NMDI_sum", - "NMDI_iqr", - "NMDI_iqr0595", - "NIRv_p2", - "NIRv_p5", - "NIRv_p25", - "NIRv_median", - "NIRv_p75", - "NIRv_p95", - "NIRv_p98", - "NIRv_mean", - "NIRv_sd", - "NIRv_sum", - "NIRv_iqr", - "NIRv_iqr0595", - "S2WI_p2", - "S2WI_p5", - "S2WI_p25", - "S2WI_median", - "S2WI_p75", - "S2WI_p95", - "S2WI_p98", - "S2WI_mean", - "S2WI_sd", - "S2WI_sum", - "S2WI_iqr", - "S2WI_iqr0595", - "S2REP_p2", - "S2REP_p5", - "S2REP_p25", - "S2REP_median", - "S2REP_p75", - "S2REP_p95", - "S2REP_p98", - "S2REP_mean", - "S2REP_sd", - "S2REP_sum", - "S2REP_iqr", - "S2REP_iqr0595", - "WRI_p2", - "WRI_p5", - "WRI_p25", - "WRI_median", - "WRI_p75", - "WRI_p95", - "WRI_p98", - "WRI_mean", - "WRI_sd", - "WRI_sum", - "WRI_iqr", - "WRI_iqr0595", - "VH_p2", - "VH_p5", - "VH_p25", - "VH_median", - "VH_p75", - "VH_p95", - "VH_p98", - "VH_mean", - "VH_sd", - "VH_sum", - "VH_iqr", - "VH_iqr0595", - "VV_p2", - "VV_p5", - "VV_p25", - "VV_median", - "VV_p75", - "VV_p95", - "VV_p98", - "VV_mean", - "VV_sd", - "VV_sum", - "VV_iqr", - "VV_iqr0595", - "VHVVD_p2", - "VHVVD_p5", - "VHVVD_p25", - "VHVVD_median", - "VHVVD_p75", - "VHVVD_p95", - "VHVVD_p98", - "VHVVD_mean", - "VHVVD_sd", - "VHVVD_sum", - "VHVVD_iqr", - "VHVVD_iqr0595", - "VHVVR_p2", - "VHVVR_p5", - "VHVVR_p25", - "VHVVR_median", - "VHVVR_p75", - "VHVVR_p95", - "VHVVR_p98", - "VHVVR_mean", - "VHVVR_sd", - "VHVVR_sum", - "VHVVR_iqr", - "VHVVR_iqr0595", - "DpRVIVV_p2", - "DpRVIVV_p5", - "DpRVIVV_p25", - "DpRVIVV_median", - "DpRVIVV_p75", - "DpRVIVV_p95", - "DpRVIVV_p98", - "DpRVIVV_mean", - "DpRVIVV_sd", - "DpRVIVV_sum", - "DpRVIVV_iqr", - "DpRVIVV_iqr0595" + "vh_p2", + "vh_p5", + "vh_p25", + "vh_median", + "vh_p75", + "vh_p95", + "vh_p98", + "vh_mean", + "vh_sd", + "vh_sum", + "vh_iqr", + "vh_iqr0595", + "vv_p2", + "vv_p5", + "vv_p25", + "vv_median", + "vv_p75", + "vv_p95", + "vv_p98", + "vv_mean", + "vv_sd", + "vv_sum", + "vv_iqr", + "vv_iqr0595", + "vhvvd_p2", + "vhvvd_p5", + "vhvvd_p25", + "vhvvd_median", + "vhvvd_p75", + "vhvvd_p95", + "vhvvd_p98", + "vhvvd_mean", + "vhvvd_sd", + "vhvvd_sum", + "vhvvd_iqr", + "vhvvd_iqr0595", + "vhvvr_p2", + "vhvvr_p5", + "vhvvr_p25", + "vhvvr_median", + "vhvvr_p75", + "vhvvr_p95", + "vhvvr_p98", + "vhvvr_mean", + "vhvvr_sd", + "vhvvr_sum", + "vhvvr_iqr", + "vhvvr_iqr0595", + "dprvivv_p2", + "dprvivv_p5", + "dprvivv_p25", + "dprvivv_median", + "dprvivv_p75", + "dprvivv_p95", + "dprvivv_p98", + "dprvivv_mean", + "dprvivv_sd", + "dprvivv_sum", + "dprvivv_iqr", + "dprvivv_iqr0595" ] } }, - "filterbands1": { + "filterbands2": { "process_id": "filter_bands", "arguments": { "bands": [ - "B02_p2", - "B02_p5", - "B02_p25", - "B02_median", - "B02_p75", - "B02_p95", - "B02_p98", - "B02_mean", - "B02_sd", - "B02_sum", - "B02_iqr", - "B02_iqr0595", - "B03_p2", - "B03_p5", - "B03_p25", - "B03_median", - "B03_p75", - "B03_p95", - "B03_p98", - "B03_mean", - "B03_sd", - "B03_sum", - "B03_iqr", - "B03_iqr0595", - "B04_p2", - "B04_p5", - "B04_p25", - "B04_median", - "B04_p75", - "B04_p95", - "B04_p98", - "B04_mean", - "B04_sd", - "B04_sum", - "B04_iqr", - "B04_iqr0595", - "B05_p2", - "B05_p5", - "B05_p25", - "B05_median", - "B05_p75", - "B05_p95", - "B05_p98", - "B05_mean", - "B05_sd", - "B05_sum", - "B05_iqr", - "B05_iqr0595", - "B06_p2", - "B06_p5", - "B06_p25", - "B06_median", - "B06_p75", - "B06_p95", - "B06_p98", - "B06_mean", - "B06_sd", - "B06_sum", - "B06_iqr", - "B06_iqr0595", - "B07_p2", - "B07_p5", - "B07_p25", - "B07_median", - "B07_p75", - "B07_p95", - "B07_p98", - "B07_mean", - "B07_sd", - "B07_sum", - "B07_iqr", - "B07_iqr0595", - "B08_p2", - "B08_p5", - "B08_p25", - "B08_median", - "B08_p75", - "B08_p95", - "B08_p98", - "B08_mean", - "B08_sd", - "B08_sum", - "B08_iqr", - "B08_iqr0595", - "B8A_p2", - "B8A_p5", - "B8A_p25", - "B8A_median", - "B8A_p75", - "B8A_p95", - "B8A_p98", - "B8A_mean", - "B8A_sd", - "B8A_sum", - "B8A_iqr", - "B8A_iqr0595", - "B11_p2", - "B11_p5", - "B11_p25", - "B11_median", - "B11_p75", - "B11_p95", - "B11_p98", - "B11_mean", - "B11_sd", - "B11_sum", - "B11_iqr", - "B11_iqr0595", - "B12_p2", - "B12_p5", - "B12_p25", - "B12_median", - "B12_p75", - "B12_p95", - "B12_p98", - "B12_mean", - "B12_sd", - "B12_sum", - "B12_iqr", - "B12_iqr0595", - "ABDI1_p2", - "ABDI1_p5", - "ABDI1_p25", - "ABDI1_median", - "ABDI1_p75", - "ABDI1_p95", - "ABDI1_p98", - "ABDI1_mean", - "ABDI1_sd", - "ABDI1_sum", - "ABDI1_iqr", - "ABDI1_iqr0595", - "ABDI2_p2", - "ABDI2_p5", - "ABDI2_p25", - "ABDI2_median", - "ABDI2_p75", - "ABDI2_p95", - "ABDI2_p98", - "ABDI2_mean", - "ABDI2_sd", - "ABDI2_sum", - "ABDI2_iqr", - "ABDI2_iqr0595", - "AWEInsh_p2", - "AWEInsh_p5", - "AWEInsh_p25", - "AWEInsh_median", - "AWEInsh_p75", - "AWEInsh_p95", - "AWEInsh_p98", - "AWEInsh_mean", - "AWEInsh_sd", - "AWEInsh_sum", - "AWEInsh_iqr", - "AWEInsh_iqr0595", - "AVI_p2", - "AVI_p5", - "AVI_p25", - "AVI_median", - "AVI_p75", - "AVI_p95", - "AVI_p98", - "AVI_mean", - "AVI_sd", - "AVI_sum", - "AVI_iqr", - "AVI_iqr0595", - "BLFEI_p2", - "BLFEI_p5", - "BLFEI_p25", - "BLFEI_median", - "BLFEI_p75", - "BLFEI_p95", - "BLFEI_p98", - "BLFEI_mean", - "BLFEI_sd", - "BLFEI_sum", - "BLFEI_iqr", - "BLFEI_iqr0595", - "CIRE_p2", - "CIRE_p5", - "CIRE_p25", - "CIRE_median", - "CIRE_p75", - "CIRE_p95", - "CIRE_p98", - "CIRE_mean", - "CIRE_sd", - "CIRE_sum", - "CIRE_iqr", - "CIRE_iqr0595", - "EVI_p2", - "EVI_p5", - "EVI_p25", - "EVI_median", - "EVI_p75", - "EVI_p95", - "EVI_p98", - "EVI_mean", - "EVI_sd", - "EVI_sum", - "EVI_iqr", - "EVI_iqr0595", - "IRECI_p2", - "IRECI_p5", - "IRECI_p25", - "IRECI_median", - "IRECI_p75", - "IRECI_p95", - "IRECI_p98", - "IRECI_mean", - "IRECI_sd", - "IRECI_sum", - "IRECI_iqr", - "IRECI_iqr0595", - "MBWI_p2", - "MBWI_p5", - "MBWI_p25", - "MBWI_median", - "MBWI_p75", - "MBWI_p95", - "MBWI_p98", - "MBWI_mean", - "MBWI_sd", - "MBWI_sum", - "MBWI_iqr", - "MBWI_iqr0595", - "MNDWI_p2", - "MNDWI_p5", - "MNDWI_p25", - "MNDWI_median", - "MNDWI_p75", - "MNDWI_p95", - "MNDWI_p98", - "MNDWI_mean", - "MNDWI_sd", - "MNDWI_sum", - "MNDWI_iqr", - "MNDWI_iqr0595", - "MNDVI_p2", - "MNDVI_p5", - "MNDVI_p25", - "MNDVI_median", - "MNDVI_p75", - "MNDVI_p95", - "MNDVI_p98", - "MNDVI_mean", - "MNDVI_sd", - "MNDVI_sum", - "MNDVI_iqr", - "MNDVI_iqr0595", - "NDMI_p2", - "NDMI_p5", - "NDMI_p25", - "NDMI_median", - "NDMI_p75", - "NDMI_p95", - "NDMI_p98", - "NDMI_mean", - "NDMI_sd", - "NDMI_sum", - "NDMI_iqr", - "NDMI_iqr0595", - "NDVI_p2", - "NDVI_p5", - "NDVI_p25", - "NDVI_median", - "NDVI_p75", - "NDVI_p95", - "NDVI_p98", - "NDVI_mean", - "NDVI_sd", - "NDVI_sum", - "NDVI_iqr", - "NDVI_iqr0595", - "NDVIMNDWI_p2", - "NDVIMNDWI_p5", - "NDVIMNDWI_p25", - "NDVIMNDWI_median", - "NDVIMNDWI_p75", - "NDVIMNDWI_p95", - "NDVIMNDWI_p98", - "NDVIMNDWI_mean", - "NDVIMNDWI_sd", - "NDVIMNDWI_sum", - "NDVIMNDWI_iqr", - "NDVIMNDWI_iqr0595", - "NDWI_p2", - "NDWI_p5", - "NDWI_p25", - "NDWI_median", - "NDWI_p75", - "NDWI_p95", - "NDWI_p98", - "NDWI_mean", - "NDWI_sd", - "NDWI_sum", - "NDWI_iqr", - "NDWI_iqr0595", - "NMDI_p2", - "NMDI_p5", - "NMDI_p25", - "NMDI_median", - "NMDI_p75", - "NMDI_p95", - "NMDI_p98", - "NMDI_mean", - "NMDI_sd", - "NMDI_sum", - "NMDI_iqr", - "NMDI_iqr0595", - "NIRv_p2", - "NIRv_p5", - "NIRv_p25", - "NIRv_median", - "NIRv_p75", - "NIRv_p95", - "NIRv_p98", - "NIRv_mean", - "NIRv_sd", - "NIRv_sum", - "NIRv_iqr", - "NIRv_iqr0595", - "S2WI_p2", - "S2WI_p5", - "S2WI_p25", - "S2WI_median", - "S2WI_p75", - "S2WI_p95", - "S2WI_p98", - "S2WI_mean", - "S2WI_sd", - "S2WI_sum", - "S2WI_iqr", - "S2WI_iqr0595", - "S2REP_p2", - "S2REP_p5", - "S2REP_p25", - "S2REP_median", - "S2REP_p75", - "S2REP_p95", - "S2REP_p98", - "S2REP_mean", - "WRI_p2", - "WRI_p5", - "WRI_p25", - "WRI_median", - "WRI_p75", - "WRI_p95", - "WRI_p98", - "WRI_mean", - "WRI_sd", - "WRI_sum", - "WRI_iqr", - "WRI_iqr0595", - "VH_p2", - "VH_p5", - "VH_p25", - "VH_median", - "VH_p75", - "VH_p95", - "VH_p98", - "VH_mean", - "VH_sd", - "VH_iqr", - "VH_iqr0595", - "VV_p2", - "VV_p5", - "VV_p25", - "VV_median", - "VV_p75", - "VV_p95", - "VV_p98", - "VV_mean", - "VV_sd", - "VV_iqr", - "VV_iqr0595", - "VHVVD_p2", - "VHVVD_p5", - "VHVVD_p25", - "VHVVD_median", - "VHVVD_p75", - "VHVVD_p95", - "VHVVD_p98", - "VHVVD_mean", - "VHVVD_sd", - "VHVVD_iqr", - "VHVVD_iqr0595", - "VHVVR_p2", - "VHVVR_p5", - "VHVVR_p25", - "VHVVR_median", - "VHVVR_p75", - "VHVVR_p95", - "VHVVR_p98", - "VHVVR_mean", - "VHVVR_sd", - "VHVVR_sum", - "VHVVR_iqr", - "VHVVR_iqr0595", - "DpRVIVV_p2", - "DpRVIVV_p5", - "DpRVIVV_p25", - "DpRVIVV_median", - "DpRVIVV_p75", - "DpRVIVV_p95", - "DpRVIVV_p98", - "DpRVIVV_mean", - "DpRVIVV_sd", - "DpRVIVV_sum", - "DpRVIVV_iqr", - "DpRVIVV_iqr0595" + "vh_p2", + "vh_p5", + "vh_p25", + "vh_median", + "vh_p75", + "vh_p95", + "vh_p98", + "vh_mean", + "vh_sd", + "vh_iqr", + "vh_iqr0595", + "vv_p2", + "vv_p5", + "vv_p25", + "vv_median", + "vv_p75", + "vv_p95", + "vv_p98", + "vv_mean", + "vv_sd", + "vv_iqr", + "vv_iqr0595", + "vhvvd_p2", + "vhvvd_p5", + "vhvvd_p25", + "vhvvd_median", + "vhvvd_p75", + "vhvvd_p95", + "vhvvd_p98", + "vhvvd_mean", + "vhvvd_sd", + "vhvvd_iqr", + "vhvvd_iqr0595", + "vhvvr_p2", + "vhvvr_p5", + "vhvvr_p25", + "vhvvr_median", + "vhvvr_p75", + "vhvvr_p95", + "vhvvr_p98", + "vhvvr_mean", + "vhvvr_sd", + "vhvvr_sum", + "vhvvr_iqr", + "vhvvr_iqr0595", + "dprvivv_p2", + "dprvivv_p5", + "dprvivv_p25", + "dprvivv_median", + "dprvivv_p75", + "dprvivv_p95", + "dprvivv_p98", + "dprvivv_mean", + "dprvivv_sd", + "dprvivv_sum", + "dprvivv_iqr", + "dprvivv_iqr0595" ], "data": { - "from_node": "renamelabels3" + "from_node": "renamelabels4" + } + } + }, + "mergecubes1": { + "process_id": "merge_cubes", + "arguments": { + "cube1": { + "from_node": "filterbands1" + }, + "cube2": { + "from_node": "filterbands2" } } }, @@ -2858,7 +3030,7 @@ "process_id": "merge_cubes", "arguments": { "cube1": { - "from_node": "filterbands1" + "from_node": "mergecubes1" }, "cube2": { "from_node": "filterbbox1" @@ -2918,7 +3090,7 @@ } } }, - "applydimension6": { + "applydimension7": { "process_id": "apply_dimension", "arguments": { "data": { @@ -2950,7 +3122,7 @@ "process_id": "apply", "arguments": { "data": { - "from_node": "applydimension6" + "from_node": "applydimension7" }, "process": { "process_graph": { diff --git a/tests/resources/cube_generation/generate_master_feature_cube_with_catboost_inference_integration.json b/tests/resources/cube_generation/generate_master_feature_cube_with_catboost_inference_integration.json index cafa0c0..862ee3b 100644 --- a/tests/resources/cube_generation/generate_master_feature_cube_with_catboost_inference_integration.json +++ b/tests/resources/cube_generation/generate_master_feature_cube_with_catboost_inference_integration.json @@ -848,7 +848,7 @@ }, "id": "SENTINEL1_GRD", "properties": { - "polarisation": { + "sar:polarizations": { "process_graph": { "eq1": { "process_id": "eq", @@ -856,7 +856,10 @@ "x": { "from_parameter": "value" }, - "y": "VV&VH" + "y": [ + "VV", + "VH" + ] }, "result": true }