diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2dd31f4..1fd318a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,13 +17,13 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.11] + python-version: [3.11, 3.12] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} @@ -39,13 +39,7 @@ jobs: - name: Run tests only env: - NHI_GWO_USERNAME: ${{ secrets.NHI_GWO_USERNAME}} - NHI_GWO_PASSWORD: ${{ secrets.NHI_GWO_PASSWORD}} + NHI_GWO_USERNAME: ${{ secrets.NHI_GWO_USERNAME}} + NHI_GWO_PASSWORD: ${{ secrets.NHI_GWO_PASSWORD}} run: | py.test ./tests -m "not notebooks" - - - name: Run codacy-coverage-reporter - uses: codacy/codacy-coverage-reporter-action@master - with: - project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} - coverage-reports: coverage.xml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..86d0f2bb --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,38 @@ +name: Lint + +on: + push: + branches: + - main + - dev + pull_request: + branches: + - main + - dev + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Install uv with Python + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d + with: + enable-cache: true + python-version: 3.13 + + - name: Install the project + run: uv sync --extra lint + + - name: ruff-lint + run: uv run ruff check + + - name: ruff-format + run: uv run ruff format --check diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 1965aa1f..f41d8e74 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -11,24 +11,24 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.11" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build setuptools wheel + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build setuptools wheel - - name: build binary wheel and a source tarball - run: | - python -m build --sdist --wheel --outdir dist/ + - name: build binary wheel and a source tarball + run: | + python -m build --sdist --wheel --outdir dist/ - - name: Publish a Python distribution to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} + - name: Publish a Python distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore index 9772658e..ba292c75 100644 --- a/.gitignore +++ b/.gitignore @@ -147,14 +147,18 @@ nlmod/bin/* !nlmod/bin/mp7_2_002_provisional nlmod/data/bofek/* -tests/data/* -!tests/data/**/ -tests/data/mfoutput/* -!tests/data/mfoutput/vertex/* -!tests/data/mfoutput/structured/* - docs/examples/*/ !docs/examples/data/ !docs/examples/data/chloride_hbossche.nc -docs/data_sources/* \ No newline at end of file +docs/data_sources/* + +docs/advanced_stress_packages/01_lake/ +docs/advanced_stress_packages/model17/ + +# Notebook-generated model directories (named like notebook stems, e.g. 01_lake) +docs/advanced_stress_packages/[0-9][0-9]_*/ +docs/data_sources/[0-9][0-9]_*/ +docs/examples/[0-9][0-9]_*/ +docs/utilities/[0-9][0-9]_*/ +docs/workflows/[0-9][0-9]_*/ diff --git a/.readthedocs.yml b/.readthedocs.yml index 6a831c43..cb50917c 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ version: 2 build: os: ubuntu-lts-latest tools: - python: "3.11" + python: "3.12" # Build documentation in the docs/ directory with Sphinx sphinx: @@ -26,4 +26,3 @@ python: extra_requirements: - nbtest - rtd - diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 23de1940..65dbfbb0 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,13 +1,91 @@ -/* --- Make pandas DataFrames scrollable --- */ -.dataframe { - max-width: 100%; /* Prevent table from exceeding container width */ - display: block; /* Enables scrolling */ - overflow-x: auto; /* Horizontal scroll for wide tables */ -} - -/* make text smaller so more fits */ -.dataframe td, -.dataframe th { - font-size: 0.7em; - padding: 0.3em 0.5em; +/* --------------------------------------------- + Documentation-style pandas dataframe tables + Compatible with Sphinx RTD theme + MyST-NB +--------------------------------------------- */ + +table.dataframe { + border-collapse: collapse; + border: none; + display: block; + overflow-x: auto; + max-width: 100%; + font-size: 0.7em; + margin-top: 0.4em; + margin-bottom: 0.6em; +} + +/* Column headers */ +table.dataframe thead th { + text-align: right; + padding: 4px 8px; + font-weight: bold; + border: none; + border-bottom: 2px solid #888; /* single horizontal line */ + background: transparent; + white-space: nowrap; + vertical-align: bottom; + line-height: 1.2; +} + +/* Remove empty header cells from multi-index (top-left corner) */ +table.dataframe thead th:empty { + padding: 0; + border: none; +} + +/* Index columns */ +table.dataframe tbody th { + text-align: right; + padding: 4px 8px; + font-weight: bold; + border: none; /* remove all vertical borders */ + white-space: nowrap; + line-height: 1.2; +} + +/* Only the rightmost index column gets the vertical line */ +table.dataframe tbody th:last-of-type { + border-right: 1px solid #bbb; +} + +/* Data cells */ +table.dataframe td { + text-align: right; + padding: 4px 8px; + border: none; + line-height: 1.2; +} + +/* Zebra striping */ +table.dataframe tbody tr:nth-child(odd) { + background-color: #f5f5f5; +} + +table.dataframe tbody tr:nth-child(even) { + background-color: transparent; +} + +/* Hover highlight */ +table.dataframe tbody tr:hover { + background-color: #dcdcdc; +} + +/* Multi-index column headers: only bottom row gets horizontal line */ +table.dataframe thead tr:not(:last-child) th { + border-bottom: none; +} +table.dataframe thead tr:last-child th { + border-bottom: 2px solid #888; +} + +/* Notebook-like output container */ +div.output_area table.dataframe { + display: block; + overflow-x: auto; + max-width: 100%; +} + +/* Space around notebook outputs */ +div.output_area { + padding: 0.35em 0; } \ No newline at end of file diff --git a/docs/advanced_stress_packages/01_lake.ipynb b/docs/advanced_stress_packages/01_lake.ipynb index 8edd47e6..9c2b139a 100644 --- a/docs/advanced_stress_packages/01_lake.ipynb +++ b/docs/advanced_stress_packages/01_lake.ipynb @@ -19,11 +19,13 @@ "outputs": [], "source": [ "import os\n", + "\n", + "import flopy\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", "import pandas as pd\n", "import xarray as xr\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib\n", - "import flopy\n", + "\n", "import nlmod" ] }, @@ -120,9 +122,9 @@ ")\n", "\n", "bgt = nlmod.gwf.surface_water.add_min_ahn_to_gdf(bgt, ahn, buffer=5.0)\n", - "bgt[\"summer_stage\"] = bgt[[\"summer_stage\", \"ahn_min\"]].max(1)\n", - "bgt[\"winter_stage\"] = bgt[[\"winter_stage\", \"ahn_min\"]].max(1)\n", - "bgt[\"stage\"] = bgt[[\"summer_stage\", \"winter_stage\"]].mean(1)\n", + "bgt[\"summer_stage\"] = bgt[[\"summer_stage\", \"ahn_min\"]].max(axis=1)\n", + "bgt[\"winter_stage\"] = bgt[[\"winter_stage\", \"ahn_min\"]].max(axis=1)\n", + "bgt[\"stage\"] = bgt[[\"summer_stage\", \"winter_stage\"]].mean(axis=1)\n", "\n", "bgt[\"lake\"] = None\n", "henschotermeer = bgt.area.idxmax() # 'W0662.ce4627405cb242ee8533946265644bb3'\n", @@ -282,9 +284,9 @@ "ims = nlmod.sim.ims(\n", " sim,\n", " complexity=\"COMPLEX\",\n", - " inner_dvclose=0.01, # needed so lake balance is correct\n", - " outer_dvclose=0.01, # needed so lake balance is correct\n", - " #rcloserecord=[0.01, \"STRICT\"],\n", + " inner_dvclose=0.01, # needed so lake balance is correct\n", + " outer_dvclose=0.01, # needed so lake balance is correct\n", + " # rcloserecord=[0.01, \"STRICT\"],\n", ") # ims solver\n", "gwf = nlmod.gwf.gwf(ds, sim, under_relaxation=True) # groundwater flow model\n", "dis = nlmod.gwf.dis(ds, gwf) # spatial discretization\n", @@ -328,7 +330,7 @@ "metadata": {}, "outputs": [], "source": [ - "# %% \n", + "# %%\n", "lak_resistance = 10.0 # days\n", "lak_grid[\"clake\"] = lak_resistance\n", "lak_grid[\"strt\"] = 6.0\n", @@ -341,8 +343,8 @@ " rainfall=ds[\"recharge\"].to_pandas().iloc[:, 0],\n", " evaporation=ds[\"evaporation\"].to_pandas().iloc[:, 0] * 1.25,\n", " boundname_column=\"lake\",\n", - " #maximum_stage_change=0.000001,\n", - " #maximum_iterations=200,\n", + " # maximum_stage_change=0.000001,\n", + " # maximum_iterations=200,\n", ");" ] }, @@ -404,7 +406,7 @@ "source": [ "head = nlmod.gwf.get_heads_da(ds)\n", "ax = nlmod.plot.map_array(head.sel(layer=\"BXz2\").mean(\"time\"), ds=ds)\n", - "bgt.plot(edgecolor='k', facecolor='none', ax=ax);" + "bgt.plot(edgecolor=\"k\", facecolor=\"none\", ax=ax);" ] }, { @@ -424,10 +426,10 @@ "outputs": [], "source": [ "lak_stage = pd.read_csv(os.path.join(ds.model_ws, \"lak_STAGE.csv\"), index_col=0)\n", - "lak_stage.index = pd.to_datetime(ds.time.start) + pd.to_timedelta(lak_stage.index, \"d\")\n", + "lak_stage.index = pd.to_datetime(ds.time.start) + pd.to_timedelta(lak_stage.index, \"D\")\n", "lak_stage.columns = [x.capitalize() for x in lak_stage.columns]\n", "\n", - "f, ax = plt.subplots(figsize=(10, 6), layout=\"constrained\")\n", + "f, ax = plt.subplots(figsize=(10, 3), layout=\"constrained\")\n", "lak_stage.plot(ax=ax)\n", "ax.set_xlabel(\"\")\n", "ax.set_ylabel(\"lake stage (m MSL)\")\n", @@ -463,11 +465,10 @@ "def plot_lake_in_cs(linestring, dcs, height, ax, color=\"C0\", zorder=0):\n", " xy = (dcs.line.project(linestring.boundary.geoms[0]), dcs.zmin)\n", " width = dcs.line.project(linestring.boundary.geoms[1]) - xy[0]\n", - " rect = matplotlib.patches.Rectangle(\n", - " xy, width, height, color=color, zorder=zorder\n", - " )\n", + " rect = matplotlib.patches.Rectangle(xy, width, height, color=color, zorder=zorder)\n", " ax.add_patch(rect)\n", "\n", + "\n", "# plot the lake\n", "for lake in bgt.loc[~bgt.lake.isna(), \"lake\"].unique():\n", " mean_lake_stage = lak_stage[lake].mean()\n", @@ -532,7 +533,7 @@ " for key in lake_bgt.keys():\n", " data[key] = [x[x[\"node\"] == lakeno + 1][\"q\"].sum() for x in lake_bgt[key]]\n", "\n", - " index = pd.to_datetime(ds.time.start) + pd.to_timedelta(cbf_lak.get_times(), \"d\")\n", + " index = pd.to_datetime(ds.time.start) + pd.to_timedelta(cbf_lak.get_times(), \"D\")\n", " df = pd.DataFrame(data, index=index)\n", "\n", " f, ax = plt.subplots(figsize=(10, 6), layout=\"constrained\")\n", @@ -547,7 +548,7 @@ " outflow = df_block.where(df_block < 0, 0.0)\n", " outflow = outflow.loc[:, ~(outflow == 0).all()]\n", "\n", - " ymax = max(inflow.sum(1).max(), outflow.sum(1).max())\n", + " ymax = max(inflow.sum(axis=1).max(), outflow.sum(axis=1).max())\n", "\n", " inflow.plot.area(color=colors, ax=ax, linewidth=0)\n", " handles_in, labels_in = ax.get_legend_handles_labels()\n", @@ -569,7 +570,7 @@ " index = labels.index(\"STORAGE\")\n", " handles.append(handles.pop(index))\n", " labels.append(labels.pop(index))\n", - " for handle, label in zip(handles_all, labels_all):\n", + " for handle, label in zip(handles_all, labels_all, strict=False):\n", " if label not in labels:\n", " handles.append(handle)\n", " labels.append(label)\n", diff --git a/docs/advanced_stress_packages/17_unsaturated_zone_flow.ipynb b/docs/advanced_stress_packages/17_unsaturated_zone_flow.ipynb index a6855bb0..da4040c7 100644 --- a/docs/advanced_stress_packages/17_unsaturated_zone_flow.ipynb +++ b/docs/advanced_stress_packages/17_unsaturated_zone_flow.ipynb @@ -64,7 +64,8 @@ "metadata": {}, "outputs": [], "source": [ - "model_name = model_ws = \"model17\"\n", + "model_name = \"model17\"\n", + "model_ws = \"17_unsaturated_zone_flow\"\n", "figdir, cachedir = nlmod.util.get_model_dirs(model_ws)" ] }, @@ -159,7 +160,11 @@ "\n", "ds.update(\n", " nlmod.read.knmi.get_recharge(\n", - " ds, method=\"separate\", cachename=\"recharge.nc\", cachedir=ds.cachedir\n", + " ds,\n", + " hourly_precision=True,\n", + " method=\"separate\",\n", + " cachename=\"recharge.nc\",\n", + " cachedir=ds.cachedir,\n", " )\n", ")" ] @@ -350,7 +355,7 @@ "fname = os.path.join(ds.model_ws, f\"{ds.model_name}.uzf.obs.csv\")\n", "obs = pd.read_csv(fname, index_col=0)\n", "obs.index = pd.to_datetime(ds.time.start) + pd.to_timedelta(obs.index, \"D\")\n", - "kind, lay, row, col, z = zip(*[x.split(\"_\") for x in obs.columns])\n", + "kind, lay, row, col, z = zip(*[x.split(\"_\") for x in obs.columns], strict=False)\n", "lays = np.array([int(x) for x in lay])\n", "rows = np.array([int(x) for x in row])\n", "cols = np.array([int(x) for x in col])\n", @@ -379,7 +384,11 @@ "col = 0\n", "\n", "f, ax = plt.subplot_mosaic(\n", - " [[\"P\"], [\"WC\"], [\"H\"]], figsize=(10, 8), height_ratios=[2, 5, 3], sharex=True\n", + " [[\"P\"], [\"WC\"], [\"H\"]],\n", + " figsize=(10, 8),\n", + " height_ratios=[2, 5, 3],\n", + " sharex=True,\n", + " layout=\"constrained\",\n", ")\n", "\n", "# top ax\n", @@ -410,9 +419,7 @@ "ax[\"H\"].plot(s.index, s.values, color=\"C0\", linestyle=\"-\", label=\"GWL UZF\")\n", "ax[\"H\"].set_ylabel(\"z [m NAP]\")\n", "ax[\"H\"].legend(loc=(0, 0.98), ncol=2, frameon=False)\n", - "ax[\"H\"].grid()\n", - "\n", - "f.tight_layout(pad=0.8)" + "ax[\"H\"].grid()" ] } ], diff --git a/docs/clear_all_notebooks.py b/docs/clear_all_notebooks.py index bf2e0e7a..521ca163 100644 --- a/docs/clear_all_notebooks.py +++ b/docs/clear_all_notebooks.py @@ -3,12 +3,13 @@ from glob import glob import nbformat -import nlmod from nbconvert.preprocessors import ( - ClearOutputPreprocessor, ClearMetadataPreprocessor, + ClearOutputPreprocessor, ) +import nlmod + logger = nlmod.util.get_color_logger("INFO") # %% get a list of all notebooks diff --git a/docs/conf.py b/docs/conf.py index 0d13434f..30bcaab8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,9 +12,6 @@ # import os import sys -import tqdm -import tqdm.auto -import tqdm.notebook from nlmod import __version__ @@ -88,7 +85,7 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] html_css_files = [ - 'custom.css', + "custom.css", ] # add a logo diff --git a/docs/data_sources/02_surface_water.ipynb b/docs/data_sources/02_surface_water.ipynb index 0041e76c..6dea086b 100644 --- a/docs/data_sources/02_surface_water.ipynb +++ b/docs/data_sources/02_surface_water.ipynb @@ -50,7 +50,7 @@ "outputs": [], "source": [ "model_name = \"steady\"\n", - "model_ws = \"schoonhoven\"\n", + "model_ws = \"02_surface_water\"\n", "figdir, cachedir = nlmod.util.get_model_dirs(model_ws)\n", "extent = [116_500, 120_000, 439_000, 442_000]" ] @@ -133,7 +133,7 @@ "outputs": [], "source": [ "f, ax = nlmod.plot.get_map(extent)\n", - "bgt.plot(\"bronhouder\", legend=True, ax=ax)" + "bgt.plot(\"bronhouder\", legend=True, ax=ax);" ] }, { diff --git a/docs/data_sources/15_geotop.ipynb b/docs/data_sources/15_geotop.ipynb index d6423cc0..665d3454 100644 --- a/docs/data_sources/15_geotop.ipynb +++ b/docs/data_sources/15_geotop.ipynb @@ -16,6 +16,7 @@ "metadata": {}, "outputs": [], "source": [ + "import numpy as np\n", "import matplotlib\n", "import matplotlib.pyplot as plt\n", "from shapely.geometry import LineString\n", @@ -37,7 +38,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d102e9d8", + "id": "603caa14", "metadata": {}, "outputs": [], "source": [ @@ -117,7 +118,7 @@ "id": "ace59ae0", "metadata": {}, "source": [ - "We plot the lithoclass (soil types) in a cross-section using the method `nlmod.plot.geotop_lithok_in_cross_section`." + "We plot the lithoclass (soil types) and the stratigraphy (geological units) in two cross-sections using the methods `nlmod.plot.geotop_lithok_in_cross_section` & `nlmod.plot.geotop_strat_in_cross_section`." ] }, { @@ -132,6 +133,18 @@ "f.tight_layout(pad=0.0)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "a314f836", + "metadata": {}, + "outputs": [], + "source": [ + "f, ax = plt.subplots(figsize=(10, 5))\n", + "nlmod.plot.geotop_strat_in_cross_section(line, gt, ax=ax)\n", + "f.tight_layout(pad=0.0)" + ] + }, { "cell_type": "markdown", "id": "5577c352", @@ -213,8 +226,8 @@ "metadata": {}, "outputs": [], "source": [ - "df = nlmod.read.geotop.get_kh_kv_table(kind=\"Brabant\")\n", - "df" + "df_brab = nlmod.read.geotop.get_kh_kv_table(kind=\"Brabant\")\n", + "df_brab" ] }, { @@ -232,8 +245,8 @@ "metadata": {}, "outputs": [], "source": [ - "gt = nlmod.read.geotop.add_kh_and_kv(gt, df)\n", - "gt" + "gt_brab = nlmod.read.geotop.add_kh_and_kv(gt.copy(), df_brab)\n", + "gt_brab" ] }, { @@ -251,7 +264,7 @@ "metadata": {}, "outputs": [], "source": [ - "plot_kh_kv(gt, layer=\"z\")" + "plot_kh_kv(gt_brab, layer=\"z\")" ] }, { @@ -268,7 +281,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30b334c3", + "id": "a39f8e71", "metadata": {}, "outputs": [], "source": [ @@ -278,7 +291,7 @@ }, { "cell_type": "markdown", - "id": "6f328978", + "id": "9917260a", "metadata": {}, "source": [ "We can plot the kh and kv value for each of the layers with the same method we used for the voxel-data." @@ -287,7 +300,7 @@ { "cell_type": "code", "execution_count": null, - "id": "b572bd90", + "id": "500ffa1d", "metadata": {}, "outputs": [], "source": [ @@ -433,6 +446,82 @@ "source": [ "plot_kh_kv(gt, layer=\"z\")" ] + }, + { + "cell_type": "markdown", + "id": "864daed4", + "metadata": {}, + "source": [ + "## Geulen\n", + "\n", + "Most of the stratigraphic units in Geotop are layered on top of each other. Meaning that these stratipgrahic units are always completely on top or completely below the other units. The so-called \"geulen\" are the exception to this, they can occur above, in-between and below other units. This makes it harderd to construct a layered groundwatermodel from the stratigraphic units.\n", + "\n", + "Currently there are three options to create a layer model based on the stratigraphic units in geotop:\n", + "1. `add_to_layer_below`, a geul is added to the layer directly below the geul\n", + "2. `add_to_layer_above`, a geul is added to the layer directly above the geul\n", + "3. `split_layers`, the geul is added as seperate layers in the layer model. If a geul occurs both above and below another unit this unit is also split into two layers.\n", + "\n", + "Below an example to showcase the differences between these methods. For this we use a different extent with 2 geulen in the stratigraphic geotop data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "af2b02ce", + "metadata": {}, + "outputs": [], + "source": [ + "# This extent has two Geulen in it\n", + "extent = (106000.0, 114000.0, 490500.0, 490700.0)\n", + "line1 = LineString([(extent[0], extent[2]+50), (extent[1], extent[2]+50)])\n", + "line2 = LineString([(extent[0], extent[2]+150), (extent[1], extent[2]+150)])\n", + "\n", + "# download geotop\n", + "gt = nlmod.read.geotop.download_geotop(extent)\n", + "\n", + "# add hydrological properties\n", + "df = nlmod.read.geotop.get_lithok_props()\n", + "gt = nlmod.read.geotop.add_kh_and_kv(gt, df)\n", + "\n", + "# set the logger to debug to get all the information\n", + "logger = nlmod.util.get_color_logger(\"DEBUG\")\n", + "gtl1 = nlmod.read.geotop.to_model_layers(gt, method_geulen='add_to_layer_below')\n", + "gtl2 = nlmod.read.geotop.to_model_layers(gt, method_geulen='add_to_layer_above')\n", + "gtl3 = nlmod.read.geotop.to_model_layers(gt, method_geulen='split_layers')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a5145e6", + "metadata": {}, + "outputs": [], + "source": [ + "# plot layer models\n", + "logger = nlmod.util.get_color_logger(\"INFO\")\n", + "stratdf = nlmod.read.geotop.get_strat_props().set_index('code')\n", + "f, axes = plt.subplots(figsize=(10, 15), nrows=4, sharex=True)\n", + "\n", + "nlmod.plot.geotop_strat_in_cross_section(line1, gt, ax=axes[0], legend=False)\n", + "axes[0].set_title('original stratigraphy Geotop')\n", + "\n", + "cs1 = nlmod.plot.DatasetCrossSection(gtl1, line1, layer='layer', ax=axes[1])\n", + "out1 = cs1.plot_layers(colors=stratdf, edgecolor='k', min_label_area=3000)\n", + "axes[1].set_title('add_to_layer_below')\n", + "\n", + "cs2 = nlmod.plot.DatasetCrossSection(gtl2, line1, layer='layer', ax=axes[2])\n", + "out2 = cs2.plot_layers(colors=stratdf, edgecolor='k', min_label_area=3000)\n", + "axes[2].set_title('add_to_layer_above')\n", + "\n", + "cs3 = nlmod.plot.DatasetCrossSection(gtl3, line1, layer='layer', ax=axes[3])\n", + "out3 = cs3.plot_layers(colors=stratdf, edgecolor='k', min_label_area=3000)\n", + "axes[3].set_title('split_layers');\n", + "\n", + "print('units in original gt:', sum(~np.isnan(np.unique(gt['strat']))))\n", + "print('layers in gtl1:', len(gtl1.layer))\n", + "print('layers in gtl2:', len(gtl2.layer))\n", + "print('layers in gtl3:', len(gtl3.layer))" + ] } ], "metadata": { diff --git a/docs/data_sources/21_meteorological_data.ipynb b/docs/data_sources/21_meteorological_data.ipynb index d14c2bba..b5d1060a 100644 --- a/docs/data_sources/21_meteorological_data.ipynb +++ b/docs/data_sources/21_meteorological_data.ipynb @@ -137,7 +137,7 @@ "metadata": {}, "outputs": [], "source": [ - "meteo = nlmod.read.knmi.get_recharge(ds, method=\"separate\")\n", + "meteo = nlmod.read.knmi.get_recharge(ds, method=\"separate\", hourly_precision=False)\n", "# show the contents of the returned dataset\n", "meteo" ] @@ -172,7 +172,9 @@ "metadata": {}, "outputs": [], "source": [ - "meteo = nlmod.read.knmi.get_recharge(ds, add_stn_dimensions=True)\n", + "meteo = nlmod.read.knmi.get_recharge(\n", + " ds, add_stn_dimensions=True, hourly_precision=False\n", + ")\n", "# show the contents of the returned dataset\n", "meteo" ] diff --git a/docs/examples/00_model_from_scratch.ipynb b/docs/examples/00_model_from_scratch.ipynb index 39aeef84..db0f16e1 100644 --- a/docs/examples/00_model_from_scratch.ipynb +++ b/docs/examples/00_model_from_scratch.ipynb @@ -80,7 +80,7 @@ " botm=botm,\n", " kh=kh,\n", " kv=kv,\n", - " model_ws=\"./scratch_model\",\n", + " model_ws=\"00_model_from_scratch\",\n", " model_name=\"from_scratch\",\n", ")\n", "ds" diff --git a/docs/examples/01_basic_model.ipynb b/docs/examples/01_basic_model.ipynb index 6667e9d6..190cdb3e 100644 --- a/docs/examples/01_basic_model.ipynb +++ b/docs/examples/01_basic_model.ipynb @@ -47,7 +47,7 @@ "outputs": [], "source": [ "# model settings\n", - "model_ws = \"ijmuiden\"\n", + "model_ws = \"01_basic_model\"\n", "model_name = \"IJmuiden\"\n", "figdir, cachedir = nlmod.util.get_model_dirs(model_ws)\n", "extent = [95000.0, 105000.0, 494000.0, 500000.0]\n", @@ -80,15 +80,13 @@ "outputs": [], "source": [ "# layer models\n", - "regis_ds = nlmod.read.regis.download_regis(extent, botm_layer=\"MSz1\",\n", - " cachedir=cachedir,\n", - " cachename=\"regis.nc\")\n", + "regis_ds = nlmod.read.regis.download_regis(\n", + " extent, botm_layer=\"MSz1\", cachedir=cachedir, cachename=\"regis.nc\"\n", + ")\n", "\n", - "geotop_ds = nlmod.read.geotop.download_geotop(extent,\n", - " cachedir=cachedir,\n", - " cachename=\"geotop.nc\",\n", - " chunks=None)\n", - "\n" + "geotop_ds = nlmod.read.geotop.download_geotop(\n", + " extent, cachedir=cachedir, cachename=\"geotop.nc\", chunks=None\n", + ")" ] }, { @@ -103,10 +101,9 @@ ")\n", "\n", "# bathymetry\n", - "bathymetry_da = nlmod.read.jarkus.download_bathymetry(extent=extent,\n", - " cachedir=cachedir,\n", - " cachename=\"bathymetry.nc\")\n", - "\n" + "bathymetry_da = nlmod.read.jarkus.download_bathymetry(\n", + " extent=extent, cachedir=cachedir, cachename=\"bathymetry.nc\"\n", + ")" ] }, { @@ -126,7 +123,9 @@ "outputs": [], "source": [ "# knmi data\n", - "oc_knmi = nlmod.read.knmi.download_knmi(extent=extent, delr=delr, delc=delc, start='2000-1-1', end='2020-1-1')" + "oc_knmi = nlmod.read.knmi.download_knmi(\n", + " extent=extent, delr=delr, delc=delc, start=\"2000-1-1\", end=\"2020-1-1\"\n", + ")" ] }, { @@ -152,9 +151,9 @@ "outputs": [], "source": [ "# create layer model\n", - "layer_model = nlmod.read.regis.add_geotop_to_regis_layers(regis_ds,\n", - " geotop_ds,\n", - " layers=\"HLc\")\n", + "layer_model = nlmod.read.regis.add_geotop_to_regis_layers(\n", + " regis_ds, geotop_ds, layers=\"HLc\"\n", + ")\n", "\n", "# create a dataset from they layer model\n", "ds = nlmod.to_model_ds(layer_model, model_name, model_ws, delr=delr, delc=delc)\n", @@ -164,7 +163,14 @@ "\n", "# add northsea and bathymetry to modelgrid\n", "ds.update(nlmod.read.rws.discretize_northsea(ds, gdf=gdf_surface_water))\n", - "ds.update(nlmod.read.jarkus.discretize_bathymetry(ds, bathymetry_da.drop_vars(\"time\"), cachedir=cachedir, cachename=\"bathymetry.nc\"))\n", + "ds.update(\n", + " nlmod.read.jarkus.discretize_bathymetry(\n", + " ds,\n", + " bathymetry_da.drop_vars(\"time\"),\n", + " cachedir=cachedir,\n", + " cachename=\"bathymetry.nc\",\n", + " )\n", + ")\n", "ds = nlmod.dims.add_bathymetry_to_layer_model(ds)" ] }, @@ -251,7 +257,9 @@ "outputs": [], "source": [ "# discretize knmi recharge data\n", - "knmi_ds = nlmod.read.knmi.discretize_knmi(ds, oc_knmi, cachedir=cachedir, cachename=\"recharge\")\n", + "knmi_ds = nlmod.read.knmi.discretize_knmi(\n", + " ds, oc_knmi, hourly_precision=True, cachedir=cachedir, cachename=\"recharge\"\n", + ")\n", "\n", "# update model dataset\n", "ds.update(knmi_ds)\n", diff --git a/docs/examples/03_local_grid_refinement.ipynb b/docs/examples/03_local_grid_refinement.ipynb index 7c3c8f1c..9a7d42c6 100644 --- a/docs/examples/03_local_grid_refinement.ipynb +++ b/docs/examples/03_local_grid_refinement.ipynb @@ -15,6 +15,7 @@ "outputs": [], "source": [ "import os\n", + "\n", "import geopandas as gpd\n", "\n", "import nlmod" @@ -45,7 +46,7 @@ "outputs": [], "source": [ "# model settings vertex\n", - "model_ws = \"ijmuiden\"\n", + "model_ws = \"03_local_grid_refinement\"\n", "model_name = \"IJm_planeten\"\n", "figdir, cachedir = nlmod.util.get_model_dirs(model_ws)\n", "refine_shp_fname = os.path.abspath(os.path.join(\"data\", \"planetenweg_ijmuiden\"))\n", @@ -84,13 +85,13 @@ "outputs": [], "source": [ "# layer models\n", - "regis_ds = nlmod.read.regis.download_regis(extent, botm_layer=\"MSz1\",\n", - " cachedir=cachedir,\n", - " cachename=\"regis.nc\")\n", + "regis_ds = nlmod.read.regis.download_regis(\n", + " extent, botm_layer=\"MSz1\", cachedir=cachedir, cachename=\"regis.nc\"\n", + ")\n", "\n", - "geotop_ds = nlmod.read.geotop.download_geotop(extent,\n", - " cachedir=cachedir,\n", - " cachename=\"geotop.nc\")" + "geotop_ds = nlmod.read.geotop.download_geotop(\n", + " extent, chunks=None, cachedir=cachedir, cachename=\"geotop.nc\"\n", + ")" ] }, { @@ -100,14 +101,14 @@ "outputs": [], "source": [ "# read surface water data\n", - "gdf_surface_water = nlmod.read.rws.get_gdf_surface_water(extent=extent,\n", - " cachedir=cachedir,\n", - " cachename='rws_surface_water.pklz')\n", + "gdf_surface_water = nlmod.read.rws.get_gdf_surface_water(\n", + " extent=extent, cachedir=cachedir, cachename=\"rws_surface_water.pklz\"\n", + ")\n", "\n", "# bathymetry\n", - "bathymetry_da = nlmod.read.jarkus.download_bathymetry(extent=extent,\n", - " cachedir=cachedir,\n", - " cachename=\"bathymetry.nc\")\n" + "bathymetry_da = nlmod.read.jarkus.download_bathymetry(\n", + " extent=extent, cachedir=cachedir, cachename=\"bathymetry.nc\"\n", + ")" ] }, { @@ -127,7 +128,9 @@ "outputs": [], "source": [ "# knmi data\n", - "oc_knmi = nlmod.read.knmi.download_knmi(extent=extent, delr=delr, delc=delc, start='2000-1-1', end='2020-1-1')" + "oc_knmi = nlmod.read.knmi.download_knmi(\n", + " extent=extent, delr=delr, delc=delc, start=\"2000-1-1\", end=\"2020-1-1\"\n", + ")" ] }, { @@ -145,9 +148,9 @@ "outputs": [], "source": [ "# create layer model\n", - "layer_model = nlmod.read.regis.add_geotop_to_regis_layers(regis_ds,\n", - " geotop_ds,\n", - " layers=\"HLc\")\n", + "layer_model = nlmod.read.regis.add_geotop_to_regis_layers(\n", + " regis_ds, geotop_ds, layers=\"HLc\"\n", + ")\n", "\n", "# create a model ds by changing grid of layer_model\n", "ds = nlmod.to_model_ds(layer_model, model_name, model_ws, delr=delr, delc=delc)\n", @@ -189,14 +192,18 @@ "ds = nlmod.grid.refine(ds, refinement_features=[(refine_shp_fname, \"line\", levels)])\n", "\n", "# read surface water data\n", - "gdf_surface_water = nlmod.read.rws.get_gdf_surface_water(ds=ds,\n", - " cachedir=cachedir,\n", - " cachename='rws_surface_water.pklz')\n", + "gdf_surface_water = nlmod.read.rws.get_gdf_surface_water(\n", + " ds=ds, cachedir=cachedir, cachename=\"rws_surface_water.pklz\"\n", + ")\n", "\n", "# add northsea to modelgrid\n", "if add_northsea:\n", " ds.update(nlmod.read.rws.discretize_northsea(ds, gdf=gdf_surface_water))\n", - " ds.update(nlmod.read.jarkus.discretize_bathymetry(ds, bathymetry_da, cachedir=cachedir, cachename=\"bathymetry.nc\"))\n", + " ds.update(\n", + " nlmod.read.jarkus.discretize_bathymetry(\n", + " ds, bathymetry_da, cachedir=cachedir, cachename=\"bathymetry.nc\"\n", + " )\n", + " )\n", " ds = nlmod.dims.add_bathymetry_to_layer_model(ds)" ] }, @@ -270,7 +277,9 @@ "outputs": [], "source": [ "# discretize knmi recharge data\n", - "knmi_ds = nlmod.read.knmi.discretize_knmi(ds, oc_knmi, cachedir=cachedir, cachename=\"recharge\")\n", + "knmi_ds = nlmod.read.knmi.discretize_knmi(\n", + " ds, oc_knmi, hourly_precision=True, cachedir=cachedir, cachename=\"recharge\"\n", + ")\n", "\n", "# update model dataset\n", "ds.update(knmi_ds)\n", diff --git a/docs/examples/09_schoonhoven.ipynb b/docs/examples/09_schoonhoven.ipynb index 12049fad..40d7e48b 100644 --- a/docs/examples/09_schoonhoven.ipynb +++ b/docs/examples/09_schoonhoven.ipynb @@ -59,7 +59,7 @@ "outputs": [], "source": [ "model_name = \"Schoonhoven\"\n", - "model_ws = os.path.join(\"..\", \"data_sources\", \"schoonhoven\")\n", + "model_ws = \"09_schoonhoven\"\n", "figdir, cachedir = nlmod.util.get_model_dirs(model_ws)\n", "extent = [116_500, 120_000, 439_000, 442_000]\n", "time = pd.date_range(\"2020\", \"2023\", freq=\"MS\") # monthly timestep" @@ -89,7 +89,7 @@ "metadata": {}, "outputs": [], "source": [ - "fname_bgt = os.path.join(cachedir, \"bgt.gpkg\")\n", + "fname_bgt = os.path.join(\"..\", \"data_sources\", \"02_surface_water\", \"cache\", \"bgt.gpkg\")\n", "if not os.path.isfile(fname_bgt):\n", " raise (\n", " Exception(\n", @@ -159,7 +159,9 @@ "outputs": [], "source": [ "# knmi data\n", - "oc_knmi = nlmod.read.knmi.download_knmi(extent=extent, delr=100., delc=100., start='2000-1-1', end='2025-1-1')" + "oc_knmi = nlmod.read.knmi.download_knmi(\n", + " extent=extent, delr=100.0, delc=100.0, start=\"2000-1-1\", end=\"2025-1-1\"\n", + ")" ] }, { @@ -262,7 +264,9 @@ "metadata": {}, "outputs": [], "source": [ - "knmi_ds = nlmod.read.knmi.discretize_knmi(ds, oc_knmi, cachedir=cachedir, cachename=\"recharge\")\n", + "knmi_ds = nlmod.read.knmi.discretize_knmi(\n", + " ds, oc_knmi, hourly_precision=True, cachedir=cachedir, cachename=\"recharge\"\n", + ")\n", "ds.update(knmi_ds)" ] }, @@ -543,15 +547,14 @@ "x = 118228.0\n", "line = [(x, 439000), (x, 442000)]\n", "\n", - "f, ax = plt.subplots(figsize=(10, 6))\n", + "f, ax = plt.subplots(figsize=(10, 6), layout=\"constrained\")\n", "dcs = DatasetCrossSection(ds, line, ax=ax, zmin=-100.0, zmax=10.0)\n", "pc = dcs.plot_array(head.mean(\"time\"), norm=norm, head=head.mean(\"time\"))\n", "\n", "# add labels with layer names\n", "cbar = nlmod.plot.colorbar_inside(pc)\n", "dcs.plot_grid()\n", - "dcs.plot_layers(colors=\"none\", min_label_area=1000)\n", - "f.tight_layout(pad=0.0)" + "dcs.plot_layers(colors=\"none\", min_label_area=1000)" ] }, { @@ -836,11 +839,11 @@ " pf = pdata[pdata[\"particleid\"] == pid]\n", " d = line.distance(Point(pf[\"x\"][0], pf[\"y\"][0]))\n", " if d < 200.0:\n", - " x = [line.project(Point(x, y)) for x, y in zip(pf[\"x\"], pf[\"y\"])]\n", + " x = [line.project(Point(x, y)) for x, y in zip(pf[\"x\"], pf[\"y\"], strict=False)]\n", " segments.extend(get_segments(x, pf[\"z\"]))\n", " array.extend(get_array(pf[\"time\"]))\n", "\n", - "f, ax = plt.subplots(figsize=(10, 6))\n", + "f, ax = plt.subplots(figsize=(10, 6), layout=\"constrained\")\n", "ax.grid()\n", "dcs = DatasetCrossSection(ds, line, ax=ax, zmin=-100.0, zmax=10.0)\n", "lc = matplotlib.collections.LineCollection(\n", @@ -851,28 +854,19 @@ "# add grid\n", "dcs.plot_grid()\n", "# add labels with layer names\n", - "dcs.plot_layers(alpha=0.0, min_label_area=1000)\n", - "f.tight_layout(pad=0.0)" + "dcs.plot_layers(alpha=0.0, min_label_area=1000);" ] } ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "nlmod (3.13.11)", "language": "python", "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.10" + "version": "3.13.11" } }, "nbformat": 4, diff --git a/docs/examples/14_stromingen_example.ipynb b/docs/examples/14_stromingen_example.ipynb index 822e9fd5..76618963 100644 --- a/docs/examples/14_stromingen_example.ipynb +++ b/docs/examples/14_stromingen_example.ipynb @@ -92,7 +92,7 @@ "outputs": [], "source": [ "# define a model workspace for caching downloaded data\n", - "model_ws = os.path.join(\"..\", \"data_sources\", \"schoonhoven\")\n", + "model_ws = \"14_stromingen_example\"\n", "figdir, cachedir = nlmod.util.get_model_dirs(model_ws)" ] }, @@ -113,7 +113,7 @@ "# wells = nlmod.read.get_wells(extent) # no well data is available just yet\n", "\n", "# surface water features and levels\n", - "fname_bgt = os.path.join(cachedir, \"bgt.gpkg\")\n", + "fname_bgt = os.path.join(\"..\", \"data_sources\", \"02_surface_water\", \"cache\", \"bgt.gpkg\")\n", "if not os.path.isfile(fname_bgt):\n", " raise (\n", " Exception(\n", @@ -191,7 +191,7 @@ "outputs": [], "source": [ "# download knmi recharge data\n", - "knmi_ds = nlmod.read.knmi.get_recharge(ds)\n", + "knmi_ds = nlmod.read.knmi.get_recharge(ds, hourly_precision=True)\n", "\n", "# update model dataset\n", "ds.update(knmi_ds)\n", @@ -225,7 +225,7 @@ "sw.loc[mask, \"rbot\"] = -3.0\n", "\n", "# we need to mask out the NaNs\n", - "sw.drop(sw.index[sw[\"rbot\"].isna()], inplace=True)" + "sw = sw.drop(sw.index[sw[\"rbot\"].isna()])" ] }, { @@ -240,7 +240,7 @@ "# add bed resistance to calculate conductance\n", "bed_resistance = 1.0 # days\n", "sfw_grid[\"cond\"] = sfw_grid.area / bed_resistance\n", - "sfw_grid.set_index(\"cellid\", inplace=True)\n", + "sfw_grid = sfw_grid.set_index(\"cellid\")\n", "\n", "# build stress period data for RIV package\n", "riv_spd = nlmod.gwf.surface_water.build_spd(sfw_grid, \"RIV\", ds)" diff --git a/docs/examples/16_groundwater_transport.ipynb b/docs/examples/16_groundwater_transport.ipynb index 0e704a77..17298620 100644 --- a/docs/examples/16_groundwater_transport.ipynb +++ b/docs/examples/16_groundwater_transport.ipynb @@ -56,7 +56,7 @@ "outputs": [], "source": [ "# model settings\n", - "model_ws = \"hondsbossche\"\n", + "model_ws = \"16_groundwater_transport\"\n", "model_name = \"hondsbossche\"\n", "\n", "figdir, cachedir = nlmod.util.get_model_dirs(model_ws)\n", @@ -72,7 +72,9 @@ "start_time = \"2010-1-1\"\n", "starting_head = 1.0\n", "\n", - "municipalities = nlmod.read.administrative.get_municipalities(extent=extent_hbossche)" + "municipalities = nlmod.read.administrative.download_municipalities_gdf(\n", + " extent=extent_hbossche\n", + ")" ] }, { @@ -88,7 +90,7 @@ "metadata": {}, "outputs": [], "source": [ - "fig, ax = nlmod.plot.get_map(extent_hbossche, background=\"OpenStreetMap.Mapnik\")" + "fig, ax = nlmod.plot.get_map(extent_hbossche, background=True)" ] }, { @@ -252,7 +254,9 @@ "outputs": [], "source": [ "# aggregate chloride to our layer model using weighted mean\n", - "cli_da = nlmod.layers.aggregate_by_weighted_mean_to_ds(ds, cli, \"3d-chloride\")\n", + "cli_da = nlmod.layers.aggregate_by_weighted_mean_to_ds(\n", + " ds, cli, \"3d-chloride\", source_botm_name=\"bottom\"\n", + ")\n", "\n", "# interpolate NaNs nearest\n", "for ilay in range(cli_da.shape[0]):\n", @@ -395,7 +399,9 @@ "outputs": [], "source": [ "# download knmi recharge data\n", - "knmi_ds = nlmod.read.knmi.get_recharge(ds, cachedir=ds.cachedir, cachename=\"recharge\")\n", + "knmi_ds = nlmod.read.knmi.get_recharge(\n", + " ds, hourly_precision=True, cachedir=ds.cachedir, cachename=\"recharge\"\n", + ")\n", "# update model dataset\n", "ds.update(knmi_ds)\n", "\n", @@ -494,7 +500,7 @@ "outputs": [], "source": [ "# plot using flopy\n", - "fig, ax = nlmod.plot.get_map(extent_hbossche, background=\"OpenStreetMap.Mapnik\")\n", + "fig, ax = nlmod.plot.get_map(extent_hbossche, background=True)\n", "pmv = fp.plot.PlotMapView(model=gwf, layer=0, ax=ax)\n", "# pc = pmv.plot_array(c.isel(time=0), cmap=\"Spectral_r\")\n", "pmv.plot_bc(\"GHB\", plotAll=True, alpha=0.1, label=\"GHB\")\n", @@ -606,8 +612,14 @@ } ], "metadata": { + "kernelspec": { + "display_name": "nlmod (3.13.11)", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "name": "python", + "version": "3.13.11" } }, "nbformat": 4, diff --git a/docs/examples/nested_models.ipynb b/docs/examples/nested_models.ipynb new file mode 100644 index 00000000..2aaaec5e --- /dev/null +++ b/docs/examples/nested_models.ipynb @@ -0,0 +1,533 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Nested Groundwater Flow Models\n", + "\n", + "This notebook shows how to build a nested model: a parent model with a child model\n", + "nested inside of it. The parent and child models are connected through a GWF-GWF\n", + "exchange.\n", + "\n", + "First a single reference model is created. Then a nested model is created with a parent\n", + "and a child model. The results of the two are compared." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import shutil\n", + "\n", + "import matplotlib as mpl\n", + "import numpy as np\n", + "import pandas as pd\n", + "import xarray as xr\n", + "\n", + "import nlmod" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nlmod.util.get_color_logger(\"INFO\")\n", + "nlmod.show_versions()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 0. Configuration\n", + "\n", + "Define a workspace for the models and the model grid properties." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define a workspace for the models\n", + "model_ws = \"./model_comparison\"\n", + "\n", + "if os.path.exists(model_ws):\n", + " shutil.rmtree(model_ws)\n", + "os.makedirs(model_ws, exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define model grid properties\n", + "extent = [0, 10_000, 0, 10_000]\n", + "delr = 250.0\n", + "delc = 250.0\n", + "refine_child = 1 # refinement factor for child model\n", + "nlay = 3\n", + "botm = [-10, -20, -40]\n", + "ghb_head = 0.0 # head m+ref\n", + "ghb_resistance = 400.0 # days\n", + "well_q = -500.0 # Pumping rate m3/day\n", + "\n", + "# some solver settings\n", + "ims_settings = {\n", + " \"complexity\": \"simple\",\n", + " \"inner_maximum\": 100,\n", + " \"outer_maximum\": 50,\n", + " \"outer_dvclose\": 1e-9,\n", + " \"rcloserecord\": \"1e-6 strict\",\n", + " \"linear_acceleration\": \"bicgstab\",\n", + "}\n", + "\n", + "# optional xt3d settings, do more harm than good in this example, but should be\n", + "# beneficial with significant refinement is performed.\n", + "xt3d_model = False\n", + "xt3d_exch = False" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Create the single, uniform grid model\n", + "\n", + "First, we create a single model that covers the whole extent. This will serve as a reference." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_single = nlmod.get_ds(\n", + " extent,\n", + " delr=delr,\n", + " delc=delc,\n", + " layer=nlay,\n", + " botm=botm,\n", + " model_name=\"single\",\n", + " model_ws=os.path.join(model_ws, \"single\"),\n", + ")\n", + "ds_single = nlmod.grid.refine(ds_single)\n", + "ds_single = nlmod.time.set_ds_time(\n", + " ds_single, time=pd.Timestamp.today().normalize(), start=1\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Build the MODFLOW 6 simulation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create simulation and gwf model\n", + "sim_single = nlmod.sim.sim(ds_single)\n", + "tdis_single = nlmod.sim.tdis(ds_single, sim_single)\n", + "gwf_single = nlmod.gwf.gwf(ds_single, sim_single)\n", + "disv_single = nlmod.gwf.disv(ds_single, gwf_single)\n", + "ims_single = nlmod.sim.ims(sim_single, model=gwf_single, **ims_settings)\n", + "npf_single = nlmod.gwf.npf(\n", + " ds_single, gwf_single, save_flows=True, xt3doptions=xt3d_model\n", + ")\n", + "ic_single = nlmod.gwf.ic(ds_single, gwf_single, starting_head=0.0)\n", + "oc_single = nlmod.gwf.oc(ds_single, gwf_single)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Add boundary conditions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Add GHB package to the top layer\n", + "ghb_cond = ds_single[\"area\"] / ghb_resistance\n", + "ghb_single = nlmod.gwf.ghb(ds_single, gwf_single, bhead=ghb_head, cond=ghb_cond)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Add a well in the center of the bottom layer\n", + "well_df = pd.DataFrame(\n", + " {\n", + " \"x\": [(extent[0] + extent[1]) / 2],\n", + " \"y\": [(extent[2] + extent[3]) / 2],\n", + " \"Q\": [well_q],\n", + " \"top\": [botm[-2]],\n", + " \"botm\": [botm[-1]],\n", + " }\n", + ")\n", + "wel_single = nlmod.gwf.wells.wel_from_df(well_df, gwf_single)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Write and run the single model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nlmod.sim.write_and_run(sim_single, ds_single, silent=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get output from the single model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "head_single = nlmod.gwf.output.get_heads_da(ds=ds_single)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Create the nested parent-child model\n", + "\n", + "Now, create a nested model with a parent model and a more refined child model in the center." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "child_extent = [3750, 6250, 3750, 6250]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Parent Model\n", + "\n", + "The parent model covers the same extent as the single model, but is inactivated where\n", + "the child model will be." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_parent = nlmod.get_ds(\n", + " extent,\n", + " delr=delr,\n", + " delc=delc,\n", + " layer=nlay,\n", + " botm=botm,\n", + " model_name=\"parent\",\n", + " model_ws=os.path.join(model_ws, \"nested\"),\n", + ")\n", + "ds_parent = nlmod.grid.refine(ds_parent)\n", + "ds_parent = nlmod.time.set_ds_time(\n", + " ds_parent, time=pd.Timestamp.today().normalize(), start=1\n", + ")\n", + "\n", + "# deactivate parent cells that are within the child model extent\n", + "idomain = nlmod.grid.get_idomain(ds_parent)\n", + "inactive = (\n", + " (ds_parent.x >= child_extent[0])\n", + " & (ds_parent.x <= child_extent[1])\n", + " & (ds_parent.y >= child_extent[2])\n", + " & (ds_parent.y <= child_extent[3])\n", + ")\n", + "ds_parent[\"active_domain\"] = xr.ones_like(ds_parent.layer, dtype=bool) * (~inactive)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Build the parent model simulation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sim_nested = nlmod.sim.sim(ds_parent)\n", + "tdis_nested = nlmod.sim.tdis(ds_parent, sim_nested)\n", + "gwf_parent = nlmod.gwf.gwf(ds_parent, sim_nested)\n", + "disv_parent = nlmod.gwf.disv(ds_parent, gwf_parent)\n", + "ims_nested = nlmod.sim.ims(sim_nested, **ims_settings) # IMS for the whole simulation\n", + "npf_parent = nlmod.gwf.npf(\n", + " ds_parent, gwf_parent, save_flows=True, xt3doptions=xt3d_model\n", + ")\n", + "ic_parent = nlmod.gwf.ic(ds_parent, gwf_parent, starting_head=0.0)\n", + "oc_parent = nlmod.gwf.oc(ds_parent, gwf_parent)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Add boundary conditions to the parent model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ghb_cond_parent = ds_parent[\"area\"] / ghb_resistance\n", + "ghb_cond_parent = ghb_cond_parent.where(ds_parent[\"active_domain\"].isel(layer=0), 0.0)\n", + "ghb_parent = nlmod.gwf.ghb(ds_parent, gwf_parent, bhead=ghb_head, cond=ghb_cond_parent)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Child Model\n", + "\n", + "The child model replaces the inactive zone in the parent model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_child = nlmod.get_ds(\n", + " child_extent,\n", + " delr=delr / refine_child,\n", + " delc=delc / refine_child,\n", + " layer=nlay,\n", + " botm=botm,\n", + " model_name=\"child\",\n", + " model_ws=os.path.join(model_ws, \"nested\"),\n", + ")\n", + "ds_child = nlmod.grid.refine(ds_child)\n", + "ds_child = nlmod.time.set_ds_time(\n", + " ds_child, time=pd.Timestamp.today().normalize(), start=1\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Build the child model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "gwf_child = nlmod.gwf.gwf(ds_child, sim_nested)\n", + "disv_child = nlmod.gwf.disv(ds_child, gwf_child)\n", + "npf_child = nlmod.gwf.npf(ds_child, gwf_child, save_flows=True, xt3doptions=xt3d_model)\n", + "ic_child = nlmod.gwf.ic(ds_child, gwf_child, starting_head=0.0)\n", + "oc_child = nlmod.gwf.oc(ds_child, gwf_child)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Add boundary conditions to the child model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ghb_child = nlmod.gwf.ghb(\n", + " ds_child,\n", + " gwf_child,\n", + " bhead=ghb_head * xr.ones_like(ds_child[\"top\"]),\n", + " cond=nlmod.grid.get_area(ds_child) / ghb_resistance,\n", + ")\n", + "wel_child = nlmod.gwf.wells.wel_from_df(well_df, gwf_child)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### GWF-GWF Exchange and run" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "gwfgwf = nlmod.sim.gwfgwf(sim_nested, ds_parent, ds_child, xt3d=xt3d_exch)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nlmod.sim.write_and_run(sim_nested, ds_parent, silent=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get output from the nested model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "head_single = nlmod.gwf.output.get_heads_da(ds=ds_single)\n", + "head_parent = nlmod.gwf.output.get_heads_da(ds=ds_parent)\n", + "head_child = nlmod.gwf.output.get_heads_da(ds=ds_child)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Compare results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig, (ax0, ax1) = nlmod.plot.get_map(\n", + " nlmod.grid.get_extent(ds_single), nrows=1, ncols=2, figsize=(12, 6), sharey=True\n", + ")\n", + "vmin, vmax = -0.05, 0.0\n", + "\n", + "nlmod.plot.map_array(\n", + " head_single,\n", + " ds_single,\n", + " ax=ax0,\n", + " vmin=vmin,\n", + " vmax=vmax,\n", + " plot_grid=False,\n", + " colorbar=False,\n", + ")\n", + "ax0.set_title(\"Single Model Head\")\n", + "nlmod.plot.map_array(\n", + " head_parent,\n", + " ds_parent,\n", + " ax=ax1,\n", + " vmin=vmin,\n", + " vmax=vmax,\n", + " plot_grid=False,\n", + " colorbar_label=\"Head [m+ref]\",\n", + ")\n", + "nlmod.plot.map_array(\n", + " head_child, ds_child, ax=ax1, vmin=vmin, vmax=vmax, plot_grid=True, colorbar=False\n", + ")\n", + "ax1.set_title(\"Nested Model Head\")\n", + "ax1.set_ylabel(\"\");" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Plot the difference in heads" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if (~ds_parent[\"active_domain\"].isel(layer=0)).sum() == ds_child.sizes[\"icell2d\"]:\n", + " (icell2d,) = np.nonzero(~ds_parent[\"active_domain\"].values[0])\n", + " head_parent.values[:, :, icell2d] = head_child.values[0]\n", + " dh = head_single - head_parent\n", + " fig, ax = nlmod.plot.get_map(nlmod.grid.get_extent(ds_single))\n", + " nlmod.plot.map_array(\n", + " dh.isel(layer=0),\n", + " ds_single,\n", + " ax=ax,\n", + " cmap=\"seismic\",\n", + " norm=mpl.colors.CenteredNorm(vcenter=0),\n", + " colorbar_label=\"Head Difference (m)\",\n", + " )\n", + " ax.set_title(\"Head Difference (Single - Nested)\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# cleanup model files\n", + "shutil.rmtree(model_ws)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/generate_logo.py b/docs/generate_logo.py index a63930cf..b9c3ccfb 100644 --- a/docs/generate_logo.py +++ b/docs/generate_logo.py @@ -1,5 +1,7 @@ import os + import matplotlib.pyplot as plt + import nlmod # %% diff --git a/docs/generate_logo_text.py b/docs/generate_logo_text.py index fb6cb04e..3a1e7028 100644 --- a/docs/generate_logo_text.py +++ b/docs/generate_logo_text.py @@ -1,13 +1,14 @@ # %% import os + import geopandas as gpd -from shapely.geometry import Polygon, MultiPolygon -from matplotlib.textpath import TextPath -from matplotlib.font_manager import FontProperties import matplotlib.pyplot as plt +import numpy as np +from matplotlib.font_manager import FontProperties +from matplotlib.textpath import TextPath +from shapely.geometry import Polygon import nlmod -import numpy as np # %% @@ -103,7 +104,9 @@ def string_to_gdf(text, font, size=500, x_offset=0, **kwargs): ax.axis("equal") ax.axis(extent) color = "k" -nlmod.plot.modelgrid(ds, color=color, ax=ax, linewidth=0.5, clip_on=False, antialiased=False) +nlmod.plot.modelgrid( + ds, color=color, ax=ax, linewidth=0.5, clip_on=False, antialiased=False +) ax.set_xlabel("") ax.set_ylabel("") @@ -111,7 +114,7 @@ def string_to_gdf(text, font, size=500, x_offset=0, **kwargs): ax.axis("off") # %% save logo -fname = f"logo_text" +fname = "logo_text" dpi = 300 if figwidth != 5: fname = f"{fname}_{figwidth}" diff --git a/docs/getting_started.rst b/docs/getting_started.rst index b14209ca..cfbdae52 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -109,7 +109,6 @@ On top of that there are some optional dependecies: - rasterstats (used in nlmod.util.zonal_statistics) - contextily (nlmod.plot.add_background_map) - scikit-image (used in nlmod.read.rws.calculate_sea_coverage) -- py7zr (used in nlmod.read.bofek.download_bofek_gdf) - joblib (used in nlmod.cache) - tqdm (used for showing progress in long-running methods) - hydropandas (used in nlmod.read.knmi and nlmod.read.bro) diff --git a/docs/utilities/04_modifying_layermodels.ipynb b/docs/utilities/04_modifying_layermodels.ipynb index 7c9a9562..5c9f48ba 100644 --- a/docs/utilities/04_modifying_layermodels.ipynb +++ b/docs/utilities/04_modifying_layermodels.ipynb @@ -15,7 +15,6 @@ "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", - "import numpy as np\n", "from shapely.geometry import LineString\n", "\n", "import nlmod\n", @@ -56,14 +55,14 @@ " else:\n", " fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(14, 12), sharex=True)\n", " dcs1 = DatasetCrossSection(ds1, line=line, ax=ax1, zmin=zmin, zmax=zmax)\n", - " polys2 = dcs1.plot_layers(colors=colors, min_label_area=min_label_area)\n", + " dcs1.plot_layers(colors=colors, min_label_area=min_label_area)\n", " dcs1.plot_grid(linewidth=0.5, vertical=False)\n", " ax1.set_ylabel(ylabel)\n", "\n", " if ds2 is not None:\n", " ax1.set_title(title1)\n", " dcs2 = DatasetCrossSection(ds2, line=line, ax=ax2, zmin=zmin, zmax=zmax)\n", - " polys1 = dcs2.plot_layers(colors=colors, min_label_area=min_label_area)\n", + " dcs2.plot_layers(colors=colors, min_label_area=min_label_area)\n", " dcs2.plot_grid(linewidth=0.5, vertical=False)\n", " ax2.set_ylabel(ylabel)\n", " ax2.set_xlabel(xlabel)\n", @@ -289,10 +288,10 @@ "metadata": {}, "outputs": [], "source": [ - "combine_layers = [\n", - " tuple(np.argwhere(regis.layer.str.startswith(\"URz\").data).squeeze().tolist()),\n", - " tuple(np.argwhere(regis.layer.isin([\"PZWAz2\", \"PZWAz3\"]).data).squeeze().tolist()),\n", - "]\n", + "combine_layers = {\n", + " \"URz1\": [\"URz1\", \"URz2\", \"URz3\", \"URz4\", \"URz5\"],\n", + " \"PZWAz2\": [\"PZWAz2\", \"PZWAz3\"],\n", + "}\n", "combine_layers" ] }, diff --git a/docs/utilities/05_caching.ipynb b/docs/utilities/05_caching.ipynb index 410ec61f..29990f49 100644 --- a/docs/utilities/05_caching.ipynb +++ b/docs/utilities/05_caching.ipynb @@ -59,7 +59,7 @@ "metadata": {}, "outputs": [], "source": [ - "model_ws = \"model5\"\n", + "model_ws = \"05_caching\"\n", "\n", "# Model directories\n", "figdir, cachedir = nlmod.util.get_model_dirs(model_ws)\n", diff --git a/docs/utilities/06_gridding_vector_data.ipynb b/docs/utilities/06_gridding_vector_data.ipynb index 302d7923..315cb3e2 100644 --- a/docs/utilities/06_gridding_vector_data.ipynb +++ b/docs/utilities/06_gridding_vector_data.ipynb @@ -53,7 +53,9 @@ "ds = nlmod.get_ds([950, 1250, 20050, 20350], delr=100)\n", "# vertex grid\n", "dsv = nlmod.grid.refine(\n", - " ds, refinement_features=[([Point(1200, 20200)], \"point\", 1)], model_ws=\"model7\"\n", + " ds,\n", + " refinement_features=[([Point(1200, 20200)], \"point\", 1)],\n", + " model_ws=\"06_gridding_vector_data\",\n", ")" ] }, @@ -71,7 +73,10 @@ "outputs": [], "source": [ "point_geom = [\n", - " Point(x, y) for x, y in zip([1000, 1200, 1225, 1300], [20200, 20175, 20175, 20425])\n", + " Point(x, y)\n", + " for x, y in zip(\n", + " [1000, 1200, 1225, 1300], [20200, 20175, 20175, 20425], strict=False\n", + " )\n", "]\n", "point_gdf = gpd.GeoDataFrame({\"values\": [1, 52, 66, 24]}, geometry=point_geom)\n", "line_geom = [\n", @@ -110,7 +115,7 @@ "pol_gdf.plot(ax=ax, alpha=0.6)\n", "\n", "ax.set_xlim(ax.get_xlim()[0], 1400)\n", - "ax.set_ylim(ax.get_ylim()[0], 20500)" + "ax.set_ylim(ax.get_ylim()[0], 20500);" ] }, { @@ -157,7 +162,7 @@ "point_gdf.plot(\"values\", ax=axes[3], vmin=vmin, vmax=vmax, legend=True)\n", "nlmod.grid.modelgrid_from_ds(ds).plot(ax=axes[3])\n", "axes[3].set_title(\"points\")\n", - "axes[3].axis(\"scaled\")" + "axes[3].axis(\"scaled\");" ] }, { @@ -199,7 +204,7 @@ "point_gdf.plot(\"values\", ax=axes[2], vmin=vmin, vmax=vmax, legend=True)\n", "nlmod.grid.modelgrid_from_ds(ds).plot(ax=axes[2])\n", "axes[2].set_title(\"points\")\n", - "axes[2].axis(\"scaled\")" + "axes[2].axis(\"scaled\");" ] }, { @@ -239,7 +244,7 @@ "line_gdf.plot(\"values\", ax=axes[3], vmin=vmin, vmax=vmax, legend=True)\n", "nlmod.grid.modelgrid_from_ds(ds).plot(ax=axes[3])\n", "axes[3].set_title(\"lines\")\n", - "axes[3].axis(\"scaled\")" + "axes[3].axis(\"scaled\");" ] }, { @@ -279,7 +284,7 @@ "pol_gdf.plot(\"values\", ax=axes[3], vmin=vmin, vmax=vmax, legend=True)\n", "nlmod.grid.modelgrid_from_ds(ds).plot(ax=axes[3])\n", "axes[3].set_title(\"polygons\")\n", - "axes[3].axis(\"scaled\")" + "axes[3].axis(\"scaled\");" ] }, { @@ -316,7 +321,7 @@ "\n", "nlmod.grid.modelgrid_from_ds(ds).plot(ax=ax)\n", "ax.set_xlim(ax.get_xlim()[0], 1300)\n", - "ax.set_ylim(ax.get_ylim()[0], 20400)" + "ax.set_ylim(ax.get_ylim()[0], 20400);" ] }, { diff --git a/docs/utilities/07_resampling.ipynb b/docs/utilities/07_resampling.ipynb index 4058d535..6bc8badb 100644 --- a/docs/utilities/07_resampling.ipynb +++ b/docs/utilities/07_resampling.ipynb @@ -65,12 +65,13 @@ "outputs": [], "source": [ "ds = nlmod.get_ds([950, 1250, 20050, 20350], delr=100)\n", - "rng = np.random.default_rng(12345)\n", + "rng = np.random.default_rng(123)\n", "ds[\"data\"] = (\"y\", \"x\"), rng.random((len(ds.y), len(ds.x))) * 10\n", "\n", + "norm = Normalize(0, 10)\n", "fig, ax = plt.subplots()\n", "ax.set_aspect(\"equal\")\n", - "ds[\"data\"].plot(ax=ax, lw=0.1, edgecolor=\"k\");" + "ds[\"data\"].plot(ax=ax, lw=0.1, edgecolor=\"k\", norm=norm);" ] }, { @@ -91,7 +92,7 @@ "\n", "fig, ax = plt.subplots()\n", "ax.set_aspect(\"equal\")\n", - "ds[\"data_nan\"].plot(ax=ax, lw=0.1, edgecolor=\"k\");" + "ds[\"data_nan\"].plot(ax=ax, lw=0.1, edgecolor=\"k\", norm=norm);" ] }, { @@ -108,13 +109,16 @@ "outputs": [], "source": [ "dsv = nlmod.grid.refine(\n", - " ds, refinement_features=[([Point(1200, 20200)], \"point\", 1)], model_ws=\"model7\"\n", + " ds,\n", + " refinement_features=[([Point(1200, 20200)], \"point\", 1)],\n", + " model_ws=\"07_resampling\",\n", ")\n", - "dsv[\"data\"] = \"icell2d\", rng.random(len(dsv.data))\n", + "dsv[\"data\"] = \"icell2d\", rng.random(len(dsv.data)) * 10\n", "\n", "fig, ax = plt.subplots()\n", "ax.set_aspect(\"equal\")\n", - "nlmod.plot.data_array(dsv[\"data\"], ds=dsv, edgecolor=\"k\");" + "pc = nlmod.plot.data_array(dsv[\"data\"], ds=dsv, edgecolor=\"k\", norm=norm)\n", + "plt.colorbar(pc);" ] }, { @@ -135,7 +139,7 @@ "\n", "fig, ax = plt.subplots()\n", "ax.set_aspect(\"equal\")\n", - "nlmod.plot.data_array(dsv[\"data_nan\"], ds=dsv, edgecolor=\"k\");" + "nlmod.plot.data_array(dsv[\"data_nan\"], ds=dsv, edgecolor=\"k\", norm=norm);" ] }, { @@ -163,10 +167,10 @@ "source": [ "def compare_structured_data_arrays(da1, da2, method, edgecolor=\"k\"):\n", " fig, axes = plt.subplots(ncols=2, figsize=(12, 6))\n", - " da1.plot(ax=axes[0], edgecolor=edgecolor, vmin=0, vmax=9)\n", + " da1.plot(ax=axes[0], edgecolor=edgecolor, norm=norm)\n", " axes[0].set_aspect(\"equal\")\n", " axes[0].set_title(\"original grid\")\n", - " da2.plot(ax=axes[1], edgecolor=edgecolor, vmin=0, vmax=9)\n", + " da2.plot(ax=axes[1], edgecolor=edgecolor, norm=norm)\n", " axes[1].set_aspect(\"equal\")\n", " axes[1].set_title(f\"resampled grid, method {method}\")" ] @@ -282,7 +286,6 @@ "source": [ "def compare_struct_to_vertex(struc2d, res_vertex2d_n, dsv, method):\n", " fig, axes = plt.subplots(ncols=2, figsize=(12, 6))\n", - " norm = Normalize(0, 9)\n", " struc2d.plot(ax=axes[0], edgecolor=\"k\", norm=norm)\n", " axes[0].set_aspect(\"equal\")\n", " axes[0].set_title(\"structured grid\")\n", @@ -328,7 +331,6 @@ "source": [ "def compare_vertex_to_struct(vertex1, dsv, struc_out_n, method):\n", " fig, axes = plt.subplots(ncols=2, figsize=(12, 6))\n", - " norm = Normalize(0, 9)\n", " pc = nlmod.plot.data_array(vertex1, ds=dsv, ax=axes[0], edgecolor=\"k\", norm=norm)\n", " plt.colorbar(pc)\n", " axes[0].set_title(\"original\")\n", @@ -416,7 +418,6 @@ "source": [ "def compare_vertex_arrays(vertex1, vertex2, dsv, method):\n", " fig, axes = plt.subplots(ncols=2, figsize=(12, 6))\n", - " norm = Normalize(0, 9)\n", " pc = nlmod.plot.data_array(vertex1, ds=dsv, ax=axes[0], edgecolor=\"k\", norm=norm)\n", " plt.colorbar(pc)\n", " axes[0].set_title(\"original\")\n", @@ -514,7 +515,9 @@ "gdf = gpd.GeoDataFrame(\n", " geometry=[LineString([(extent[0], extent[2]), (extent[1], extent[3])]).buffer(10.0)]\n", ")\n", - "dsv = nlmod.grid.refine(ds_ahn, model_ws=\"model7\", refinement_features=[(gdf, 1)])" + "dsv = nlmod.grid.refine(\n", + " ds_ahn, model_ws=\"07_resampling\", refinement_features=[(gdf, 1)]\n", + ")" ] }, { @@ -523,7 +526,6 @@ "metadata": {}, "outputs": [], "source": [ - "norm = Normalize(ahn.min(), ahn.max())\n", "for method in [\"nearest\", \"average\", \"min\", \"max\"]:\n", " ahn_res = nlmod.resample.structured_da_to_ds(ahn, dsv, method=method)\n", "\n", diff --git a/docs/utilities/08_gis.ipynb b/docs/utilities/08_gis.ipynb index 48653996..82648896 100644 --- a/docs/utilities/08_gis.ipynb +++ b/docs/utilities/08_gis.ipynb @@ -15,6 +15,7 @@ "outputs": [], "source": [ "import os\n", + "\n", "import xarray as xr\n", "from IPython.display import FileLink\n", "\n", @@ -51,8 +52,10 @@ "metadata": {}, "outputs": [], "source": [ - "model_ws = os.path.join(\"..\", \"examples\", \"ijmuiden\")\n", - "model_name = \"IJmuiden\"" + "source_ws_struc = os.path.join(\"..\", \"examples\", \"01_basic_model\")\n", + "model_name = \"IJmuiden\"\n", + "output_ws = \"08_gis\"\n", + "os.makedirs(output_ws, exist_ok=True)" ] }, { @@ -62,7 +65,7 @@ "outputs": [], "source": [ "ds_struc = xr.load_dataset(\n", - " os.path.join(model_ws, f\"{model_name}.nc\"), mask_and_scale=False\n", + " os.path.join(source_ws_struc, f\"{model_name}.nc\"), mask_and_scale=False\n", ")\n", "ds_struc" ] @@ -74,9 +77,8 @@ "outputs": [], "source": [ "# create gisdir\n", - "gisdir_struc = os.path.join(model_ws, \"gis\")\n", - "if not os.path.exists(gisdir_struc):\n", - " os.mkdir(gisdir_struc)" + "gisdir_struc = os.path.join(output_ws, \"structured\")\n", + "os.makedirs(gisdir_struc, exist_ok=True)" ] }, { @@ -92,6 +94,7 @@ "metadata": {}, "outputs": [], "source": [ + "source_ws_vert = os.path.join(\"..\", \"examples\", \"03_local_grid_refinement\")\n", "model_name = \"IJm_planeten\"" ] }, @@ -102,7 +105,7 @@ "outputs": [], "source": [ "ds_vert = xr.load_dataset(\n", - " os.path.join(model_ws, f\"{model_name}.nc\"), mask_and_scale=False\n", + " os.path.join(source_ws_vert, f\"{model_name}.nc\"), mask_and_scale=False\n", ")\n", "ds_vert" ] @@ -114,9 +117,8 @@ "outputs": [], "source": [ "# create gisdir\n", - "gisdir_vert = os.path.join(model_ws, \"gis\")\n", - "if not os.path.exists(gisdir_vert):\n", - " os.mkdir(gisdir_vert)" + "gisdir_vert = os.path.join(output_ws, \"vertex\")\n", + "os.makedirs(gisdir_vert, exist_ok=True)" ] }, { diff --git a/docs/utilities/13_plot_methods.ipynb b/docs/utilities/13_plot_methods.ipynb index 7f2fdd5f..3872bd15 100644 --- a/docs/utilities/13_plot_methods.ipynb +++ b/docs/utilities/13_plot_methods.ipynb @@ -37,11 +37,12 @@ "outputs": [], "source": [ "import os\n", + "\n", "import flopy\n", - "import pandas as pd\n", "import geopandas as gpd\n", - "import numpy as np\n", "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", "import xarray as xr\n", "from shapely.geometry import LineString\n", "\n", @@ -76,7 +77,7 @@ "outputs": [], "source": [ "model_name = \"Schoonhoven\"\n", - "model_ws = os.path.join(\"..\", \"data_sources\", \"schoonhoven\")\n", + "model_ws = os.path.join(\"..\", \"examples\", \"09_schoonhoven\")\n", "fname_nc = os.path.join(model_ws, f\"{model_name}.nc\")\n", "if not os.path.isfile(fname_nc):\n", " raise (\n", @@ -85,10 +86,8 @@ " )\n", " )\n", "ds = xr.open_dataset(fname_nc)\n", - "\n", - "# add calculated heads\n", - "ds[\"head\"] = nlmod.gwf.get_heads_da(ds)\n", - "ds" + "ds.attrs[\"model_ws\"] = model_ws\n", + "ds[\"head\"] = nlmod.gwf.output.get_heads_da(ds)" ] }, { @@ -133,7 +132,7 @@ "\n", "# plot using flopy\n", "pmv = flopy.plot.PlotMapView(modelgrid=modelgrid, ax=ax[1])\n", - "pmv.plot_array(ds[\"top\"])" + "pmv.plot_array(ds[\"top\"]);" ] }, { @@ -221,12 +220,13 @@ "# create data frame with well data (random)\n", "nwells = 11\n", "max_dist = 1000\n", + "rng = np.random.default_rng()\n", "df = pd.DataFrame(\n", " {\n", - " \"screen_top\": np.random.randint(-80, -40, nwells),\n", - " \"screen_bottom\": np.random.randint(-125, -81, nwells),\n", - " \"x\": np.random.randint(ds.extent[0], ds.extent[1], nwells),\n", - " \"y\": np.random.randint(ds.extent[2], ds.extent[3], nwells),\n", + " \"screen_top\": rng.integers(-80, -40, nwells),\n", + " \"screen_bottom\": rng.integers(-125, -81, nwells),\n", + " \"x\": rng.integers(ds.extent[0], ds.extent[1], nwells),\n", + " \"y\": rng.integers(ds.extent[2], ds.extent[3], nwells),\n", " },\n", " index=[f\"well{i}\" for i in range(nwells)],\n", ")\n", @@ -235,7 +235,7 @@ " cmap(i)\n", " for i in np.linspace(0, int((len(df) - 1) * cmap.N / len(df)), len(df), dtype=int)\n", "]\n", - "df['tubecolor'] = 'white'\n", + "df[\"tubecolor\"] = \"white\"\n", "gdf = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df.x, df.y))\n", "\n", "# plot cross section\n", @@ -249,9 +249,9 @@ " df,\n", " legend=True,\n", " max_dist=max_dist,\n", - " filter_patch_kwargs={'alpha':0.5, 'zorder':10},\n", - " filter_rect_kwargs={'alpha':0.5, 'zorder':10},\n", - " tubeline_kwargs={'alpha':0.5},\n", + " filter_patch_kwargs={\"alpha\": 0.5, \"zorder\": 10},\n", + " filter_rect_kwargs={\"alpha\": 0.5, \"zorder\": 10},\n", + " tubeline_kwargs={\"alpha\": 0.5},\n", " legend_kwds={\"loc\": (0, 1), \"frameon\": False, \"ncol\": 7},\n", ")\n", "\n", @@ -326,7 +326,7 @@ "x = 118228\n", "y = 439870\n", "head_point = nlmod.gwf.get_head_at_point(ds[\"head\"], x=x, y=y, ds=ds)\n", - "head_point.plot.line(hue=\"layer\", size=10);" + "head_point.plot.line(hue=\"layer\", size=10, add_legend=True);" ] }, { @@ -363,27 +363,14 @@ "metadata": {}, "outputs": [], "source": [ - "df.plot(figsize=(10, 10));" + "ax = df.plot(figsize=(10, 3))\n", + "ax.legend(loc=(0, 1), frameon=False, ncol=10, fontsize=\"x-small\");" ] } ], "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.10" + "name": "python" } }, "nbformat": 4, diff --git a/docs/workflows/01_layer_generation.ipynb b/docs/workflows/01_layer_generation.ipynb index fb2537db..948512b5 100644 --- a/docs/workflows/01_layer_generation.ipynb +++ b/docs/workflows/01_layer_generation.ipynb @@ -114,7 +114,9 @@ "outputs": [], "source": [ "ds = nlmod.to_model_ds(regis, delr=delr, delc=delc)\n", - "ds = nlmod.grid.refine(ds, model_ws=\"model_12\", refinement_features=refinement_features)" + "ds = nlmod.grid.refine(\n", + " ds, model_ws=\"01_layer_generation\", refinement_features=refinement_features\n", + ")" ] }, { @@ -153,7 +155,9 @@ "outputs": [], "source": [ "ds = nlmod.get_ds(extent, delr=delr, delc=delc)\n", - "ds = nlmod.grid.refine(ds, model_ws=\"model_12\", refinement_features=refinement_features)\n", + "ds = nlmod.grid.refine(\n", + " ds, model_ws=\"01_layer_generation\", refinement_features=refinement_features\n", + ")\n", "ds = nlmod.grid.update_ds_from_layer_ds(ds, regis, method=\"average\")" ] }, diff --git a/docs/workflows/02_grid_rotation.ipynb b/docs/workflows/02_grid_rotation.ipynb index 53da31ec..57907641 100644 --- a/docs/workflows/02_grid_rotation.ipynb +++ b/docs/workflows/02_grid_rotation.ipynb @@ -70,7 +70,7 @@ " yorigin=500_000,\n", " delr=10.0,\n", " model_name=\"nlmod\",\n", - " model_ws=\"model11\",\n", + " model_ws=\"02_grid_rotation\",\n", ")\n", "\n", "ds = nlmod.time.set_ds_time(ds, time=\"2023-01-01\", start=\"2013-01-01\")" @@ -165,7 +165,7 @@ "metadata": {}, "outputs": [], "source": [ - "knmi_ds = nlmod.read.knmi.get_recharge(ds)\n", + "knmi_ds = nlmod.read.knmi.get_recharge(ds, hourly_precision=True)\n", "ds.update(knmi_ds)" ] }, @@ -276,7 +276,7 @@ "f, ax = nlmod.plot.get_map(ds.extent)\n", "pc = nlmod.plot.data_array(head.sel(layer=1).mean(\"time\"), ds=ds, edgecolor=\"k\")\n", "cbar = nlmod.plot.colorbar_inside(pc)\n", - "bgt.plot(ax=ax, edgecolor=\"k\", facecolor=\"none\")" + "bgt.plot(ax=ax, edgecolor=\"k\", facecolor=\"none\");" ] }, { @@ -303,7 +303,7 @@ "# to real-world coordinates before plotting\n", "affine = nlmod.grid.get_affine_mod_to_world(ds)\n", "bgt_rw = nlmod.grid.affine_transform_gdf(bgt, affine)\n", - "bgt_rw.plot(ax=ax, edgecolor=\"k\", facecolor=\"none\")" + "bgt_rw.plot(ax=ax, edgecolor=\"k\", facecolor=\"none\");" ] }, { diff --git a/docs/workflows/03_aggregating_surface_water.ipynb b/docs/workflows/03_aggregating_surface_water.ipynb index bedefdee..60d741e8 100644 --- a/docs/workflows/03_aggregating_surface_water.ipynb +++ b/docs/workflows/03_aggregating_surface_water.ipynb @@ -19,13 +19,26 @@ "outputs": [], "source": [ "import os\n", + "\n", + "import flopy\n", + "import geopandas as gpd\n", "import matplotlib.pyplot as plt\n", "import pandas as pd\n", - "import geopandas as gpd\n", - "import flopy\n", + "\n", "import nlmod" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "77b57aef", + "metadata": {}, + "outputs": [], + "source": [ + "nlmod.util.get_color_logger(\"INFO\")\n", + "nlmod.show_versions()" + ] + }, { "cell_type": "markdown", "id": "01d6ac40", @@ -43,7 +56,7 @@ "outputs": [], "source": [ "model_name = \"Schoonhoven\"\n", - "model_ws = os.path.join(\"..\", \"data_sources\", \"schoonhoven\")\n", + "model_ws = \"03_aggregating_surface_water\"\n", "figdir, cachedir = nlmod.util.get_model_dirs(model_ws)\n", "extent = [116_500, 120_000, 439_000, 442_000]\n", "time = pd.date_range(\"2020\", \"2023\", freq=\"MS\") # monthly timestep" @@ -65,7 +78,7 @@ "metadata": {}, "outputs": [], "source": [ - "fname_bgt = os.path.join(cachedir, \"bgt.gpkg\")\n", + "fname_bgt = os.path.join(\"..\", \"data_sources\", \"02_surface_water\", \"cache\", \"bgt.gpkg\")\n", "if not os.path.isfile(fname_bgt):\n", " raise (\n", " Exception(\n", @@ -91,7 +104,7 @@ "outputs": [], "source": [ "sfw = bgt\n", - "sfw[\"stage\"] = sfw[[\"winter_stage\", \"summer_stage\"]].mean(1)\n", + "sfw[\"stage\"] = sfw[[\"winter_stage\", \"summer_stage\"]].mean(axis=1)\n", "# use a water depth of 0.5 meter\n", "sfw[\"botm\"] = sfw[\"stage\"] - 0.5\n", "# set the stage of the Lek to 0.0 m NAP and the botm to -3 m NAP\n", @@ -284,7 +297,15 @@ "outputs": [], "source": [ "fig, ax = nlmod.plot.get_map(extent)\n", - "sfw_grid.plot(ax=ax, column=\"cellid\")\n", + "# compute node number for plotting since geopandas no longer supports\n", + "# casting a tuple to a str for plotting\n", + "sfw_grid[\"node\"] = sfw_grid.cellid.apply(lambda c: c[0] * ds.sizes[\"x\"] + c[1])\n", + "sfw_grid.plot(\n", + " ax=ax,\n", + " column=\"node\",\n", + " cmap=\"tab10\",\n", + " add_labels=False,\n", + ")\n", "nlmod.plot.modelgrid(ds, ax=ax, lw=0.2);" ] }, @@ -463,7 +484,7 @@ "outputs": [], "source": [ "new_celldata = celldata.loc[~celldata.rbot.isna()]\n", - "print(f\"removed {len(celldata)-len(new_celldata)} reaches because rbot is nan\")" + "print(f\"removed {len(celldata) - len(new_celldata)} reaches because rbot is nan\")" ] }, { @@ -635,22 +656,8 @@ } ], "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.10" + "name": "python" } }, "nbformat": 4, diff --git a/docs/workflows/10_modpath.ipynb b/docs/workflows/10_modpath.ipynb index c6baab83..c5e02f20 100644 --- a/docs/workflows/10_modpath.ipynb +++ b/docs/workflows/10_modpath.ipynb @@ -57,7 +57,7 @@ "outputs": [], "source": [ "# load lgr model dataset\n", - "model_ws = os.path.join(\"..\", \"examples\", \"ijmuiden\")\n", + "model_ws = os.path.join(\"..\", \"examples\", \"03_local_grid_refinement\")\n", "model_name = \"IJm_planeten\"\n", "\n", "ds = xr.open_dataset(os.path.join(model_ws, f\"{model_name}.nc\"))" @@ -386,7 +386,7 @@ "source": [ "f, ax = plt.subplots(nrows=1, ncols=1, figsize=(12, 4))\n", "\n", - "for i, pid in enumerate(np.unique(pdata[\"particleid\"])):\n", + "for _, pid in enumerate(np.unique(pdata[\"particleid\"])):\n", " pf = pdata[pdata[\"particleid\"] == pid]\n", " x0, y0, z0 = pf[[\"x\", \"y\", \"z\"]][0]\n", " distance = np.sqrt((pf[\"x\"] - x0) ** 2 + (pf[\"y\"] - y0) ** 2 + (pf[\"z\"] - z0) ** 2)\n", @@ -493,7 +493,7 @@ "source": [ "f, ax = plt.subplots(nrows=1, ncols=1, figsize=(12, 4))\n", "\n", - "for i, pid in enumerate(np.unique(pdata[\"particleid\"])):\n", + "for _, pid in enumerate(np.unique(pdata[\"particleid\"])):\n", " pf = pdata[pdata[\"particleid\"] == pid]\n", " x0, y0, z0 = pf[[\"x\", \"y\", \"z\"]][0]\n", " distance = np.sqrt((pf[\"x\"] - x0) ** 2 + (pf[\"y\"] - y0) ** 2 + (pf[\"z\"] - z0) ** 2)\n", diff --git a/docs/workflows/11_particle_tracking_prt.ipynb b/docs/workflows/11_particle_tracking_prt.ipynb index 3de69fc0..78487834 100644 --- a/docs/workflows/11_particle_tracking_prt.ipynb +++ b/docs/workflows/11_particle_tracking_prt.ipynb @@ -18,13 +18,14 @@ "outputs": [], "source": [ "from pathlib import Path\n", + "\n", "import flopy as fp\n", - "import numpy as np\n", "import geopandas as gpd\n", - "import xarray as xr\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", "import matplotlib as mpl\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import xarray as xr\n", "\n", "import nlmod" ] @@ -59,7 +60,7 @@ "metadata": {}, "outputs": [], "source": [ - "model_ws = Path(\"../examples/scratch_model\")\n", + "model_ws = Path(\"../examples/00_model_from_scratch\")\n", "model_name = \"from_scratch\"\n", "\n", "ds = xr.open_dataset(model_ws / f\"{model_name}.nc\")\n", @@ -392,8 +393,8 @@ " z = nodes_lay[\"z\"].values[active]\n", " pids = np.arange(pid, pid + len(k), dtype=int)\n", " pid += len(k)\n", - " cellid = tuple(zip(k, j))\n", - " release_ptsi = list(zip(pids, cellid, x, y, z))\n", + " cellid = tuple(zip(k, j, strict=False))\n", + " release_ptsi = list(zip(pids, cellid, x, y, z, strict=False))\n", " release_pts += release_ptsi\n", "\n", "print(f\"Number of release points: {len(release_pts)}\")" @@ -506,7 +507,7 @@ "outputs": [], "source": [ "f, ax = plt.subplots(figsize=(8, 8))\n", - "for pid, gr in pathlines_bww.groupby(\"pid\"):\n", + "for _, gr in pathlines_bww.groupby(\"pid\"):\n", " for ilay, gr_lay in gr.groupby(\"ilay\"):\n", " ax.plot(gr_lay[\"x\"], gr_lay[\"y\"], lw=1.0, color=f\"C{ilay}\", zorder=1)\n", "\n", @@ -530,6 +531,7 @@ "for pid, gr in pathlines_bw.groupby(\"pid\"):\n", " t0 = gr[\"t\"].idxmin()\n", " x0, y0, z0, ilay0 = gr.loc[t0, [\"x\", \"y\", \"z\", \"ilay\"]]\n", + " ilay0 = int(ilay0)\n", " distance = np.sqrt((gr[\"x\"] - x0) ** 2 + (gr[\"y\"] - y0) ** 2 + (gr[\"z\"] - z0) ** 2)\n", " if pid in well_pids:\n", " plot_kwargs = {\"color\": \"k\", \"alpha\": 1.0, \"zorder\": 10, \"label\": f\"well {pid}\"}\n", diff --git a/docs/workflows/18_observations.ipynb b/docs/workflows/18_observations.ipynb index ccedc043..ffecdcb9 100644 --- a/docs/workflows/18_observations.ipynb +++ b/docs/workflows/18_observations.ipynb @@ -42,7 +42,7 @@ "metadata": {}, "outputs": [], "source": [ - "model_ws = Path(\"../examples/ijmuiden\")\n", + "model_ws = Path(\"../examples/03_local_grid_refinement\")\n", "model_name = \"IJm_planeten\"\n", "\n", "# read model dataset\n", diff --git a/docs/workflows/20_voronoi.ipynb b/docs/workflows/20_voronoi.ipynb index 7ffc0ef5..2f1f567b 100644 --- a/docs/workflows/20_voronoi.ipynb +++ b/docs/workflows/20_voronoi.ipynb @@ -69,7 +69,7 @@ "gwf = nlmod.gwf.gwf(ds, sim)\n", "dis = nlmod.gwf.disv(ds, gwf)\n", "npf = nlmod.gwf.npf(ds, gwf)\n", - "ghb = nlmod.gwf.ghb(ds, gwf, bhead=10*edge_mask, cond=1e8 * edge_mask)\n", + "ghb = nlmod.gwf.ghb(ds, gwf, bhead=10 * edge_mask, cond=1e8 * edge_mask)\n", "ic = nlmod.gwf.ic(ds, gwf, starting_head=10.0)\n", "df = pd.DataFrame(index=[\"well1\"], columns=[\"x\", \"y\", \"top\", \"botm\", \"Q\"])\n", "df.loc[\"well1\"] = 1500, 500, 0.0, -10.0, -200.0\n", diff --git a/nlmod/cache.py b/nlmod/cache.py index afc531a5..ad9cd839 100644 --- a/nlmod/cache.py +++ b/nlmod/cache.py @@ -22,8 +22,9 @@ def clear_cache(cachedir, prompt=True): - """Clears the cache in a given cache directory by removing all .pklz and - corresponding .nc files. + """Clears the cache in a given cache directory. + + Removes all .pklz and corresponding .nc files. Parameters ---------- @@ -74,12 +75,14 @@ def cache_netcdf( attrs=None, nc_hash=True, ): - """Decorator to read/write the result of a function from/to a file to speed up - function calls with the same arguments. Should only be applied to functions that: + """Decorator to read/write the result of a function from/to a file. + + Speeds up function calls with the same arguments. Should only be applied to + functions that: - return an Xarray Dataset - have no more than one xarray dataset as function argument - have functions arguments of types that can be checked using the - _is_valid_cache functions + _is_valid_cache functions. 1. The directory and filename of the cache should be defined by the person calling a function with this decorator. If not defined no cache is @@ -381,8 +384,9 @@ def wrapper(*args, cachedir=None, cachename=None, **kwargs): def cache_pickle(func): - """Decorator to read/write the result of a function from/to a file to speed - up function calls with the same arguments. Should only be applied to + """Decorator to read/write the result of a function from/to a file. + + Speeds up function calls with the same arguments. Should only be applied to functions that: - return a picklable object @@ -467,7 +471,8 @@ def decorator(*args, cachedir=None, cachename=None, **kwargs): func_args_dic["_pklz_hash"] = joblib.hash(cached_pklz) except ImportError: logger.warning( - "joblib is not installed, cannot add dataframe hash to function arguments" + "joblib is not installed, cannot add dataframe hash to " + "function arguments" ) # check if cache was created with same function arguments as @@ -499,7 +504,8 @@ def decorator(*args, cachedir=None, cachename=None, **kwargs): func_args_dic["_pklz_hash"] = joblib.hash(temp) except ImportError: logger.warning( - "joblib is not installed, cannot add dataframe hash to function arguments" + "joblib is not installed, cannot add dataframe hash to " + "function arguments" ) # pickle function arguments @@ -566,7 +572,8 @@ def _same_function_arguments(func_args_dic, func_args_dic_cache): # check if cache and function call have same argument values if item is None: - # Value of None type is always None so the check happens in previous if statement + # Value of None type is always None so the check happens in + # previous if statement pass elif isinstance( item, (numbers.Number, bool, str, bytes, list, tuple, pathlib.PurePath) @@ -685,8 +692,7 @@ def _get_modification_time(func): def _update_docstring_and_signature(func): - """Add function arguments 'cachedir' and 'cachename' to the docstring and signature - of a function. + """Add function arguments 'cachedir' and 'cachename' to the docstring and signature. The function arguments are added before the "Returns" header in the docstring. If the function has no Returns header in the docstring, the function @@ -747,8 +753,9 @@ def _update_docstring_and_signature(func): def _check_for_data_array(ds): - """Check if the saved NetCDF-file represents a DataArray or a Dataset, and return - this data-variable. + """Check if the saved NetCDF-file represents a DataArray or a Dataset. + + Returns this data-variable. The file contains a DataArray when a variable called "__xarray_dataarray_variable__" is present in the Dataset. If so, return a DataArray, otherwise return the Dataset. @@ -846,7 +853,8 @@ def ds_contains( datavars.append("icvert") if "angrot" in ds.attrs: - # set by `nlmod.base.to_model_ds()` and `nlmod.dims.resample._set_angrot_attributes()` + # set by `nlmod.base.to_model_ds()` and + # `nlmod.dims.resample._set_angrot_attributes()` attrs_angrot_required = ["angrot", "xorigin", "yorigin"] attrs.extend(attrs_angrot_required) @@ -965,6 +973,18 @@ class NumpyEncoder(json.JSONEncoder): """Special json encoder for numpy types.""" def default(self, o): + """Encode numpy types to standard Python types. + + Parameters + ---------- + o : object + The object to encode. + + Returns + ------- + object + The encoded object. + """ if isinstance(o, np.integer): return int(o) elif isinstance(o, np.floating): diff --git a/nlmod/dims/attributes_encodings.py b/nlmod/dims/attributes_encodings.py index 5b4e30b0..1afe87af 100644 --- a/nlmod/dims/attributes_encodings.py +++ b/nlmod/dims/attributes_encodings.py @@ -127,9 +127,10 @@ def get_encodings( ds, set_encoding_inplace=True, allowed_to_read_data_vars_for_minmax=True ): - """Get the encoding for the data_vars. Based on the minimum values and maximum - values set in `dim_attrs` and the maximum allowed difference from - `encoding_requirements`. + """Get the encoding for the data_vars. + + Based on the minimum values and maximum values set in `dim_attrs` and the maximum + allowed difference from `encoding_requirements`. If a loss of data resolution is allowed floats can also be stored at int16, halfing the space required for storage. The maximum acceptabel loss in resolution @@ -167,9 +168,9 @@ def get_encodings( if np.issubdtype(da.dtype, np.character): continue - assert ( - "_FillValue" not in da.attrs - ), f"Custom fillvalues are not supported. {varname} has a fillvalue set." + assert "_FillValue" not in da.attrs, ( + f"Custom fillvalues are not supported. {varname} has a fillvalue set." + ) encoding = { "zlib": True, diff --git a/nlmod/dims/base.py b/nlmod/dims/base.py index b66863d3..9791f365 100644 --- a/nlmod/dims/base.py +++ b/nlmod/dims/base.py @@ -77,10 +77,30 @@ def set_ds_attrs( # add some directories if model_ws is not None: - figdir, cachedir = util.get_model_dirs(model_ws) - ds.attrs["model_ws"] = model_ws - ds.attrs["figdir"] = figdir - ds.attrs["cachedir"] = cachedir + ds = set_model_ws(ds, model_ws) + + return ds + + +def set_model_ws(ds, model_ws): + """Set the model workspace of a model dataset. + + Parameters + ---------- + ds : xarray dataset + An existing model dataset + model_ws : str + workspace of the model. This is where modeldata is saved to. + + Returns + ------- + ds : xarray dataset + model dataset with updated model workspace. + """ + figdir, cachedir = util.get_model_dirs(model_ws) + ds.attrs["model_ws"] = model_ws + ds.attrs["figdir"] = figdir + ds.attrs["cachedir"] = cachedir return ds @@ -688,7 +708,7 @@ def get_ds( coords = {"x": x, "y": y, "layer": layer} if angrot != 0.0: affine = grid.get_affine_mod_to_world(attrs) - xc, yc = affine * np.meshgrid(x, y) + xc, yc = affine @ np.meshgrid(x, y) coords["xc"] = (("y", "x"), xc) coords["yc"] = (("y", "x"), yc) @@ -723,12 +743,12 @@ def check_variable(var, shape): dims = ["layer", "y", "x"] ds = xr.Dataset( - data_vars=dict( - top=(dims[1:], top), - botm=(dims, botm), - kh=(dims, kh), - kv=(dims, kv), - ), + data_vars={ + "top": (dims[1:], top), + "botm": (dims, botm), + "kh": (dims, kh), + "kv": (dims, kv), + }, coords=coords, attrs=attrs, ) diff --git a/nlmod/dims/grid.py b/nlmod/dims/grid.py index d454ee55..20e6565f 100644 --- a/nlmod/dims/grid.py +++ b/nlmod/dims/grid.py @@ -52,8 +52,9 @@ def snap_extent(extent, delr, delc): - """Snap the extent in such a way that an integer number of columns and rows fit in - the extent. The new extent is always equal to, or bigger than the original extent. + """Snap the extent so that an integer number of columns and rows fit. + + The new extent is always equal to, or bigger than the original extent. Parameters ---------- @@ -230,7 +231,7 @@ def get_row_col_from_xy(x, y, ds, rotated=True, gi=None): raise (ValueError(f"Point ({x}, {y}) is outside of the model grid")) row, col = cellids[0] return row, col - if rotated and ("angrot" in ds.attrs) and (ds.attrs["angrot"] != 0.0): + if rotated and is_rotated(ds): # calculate the x and y in model coordinates affine = get_affine_world_to_mod(ds) x, y = affine * (x, y) @@ -359,7 +360,7 @@ def modelgrid_from_ds(ds, rotated=True, nlay=None, top=None, botm=None, **kwargs modelgrid : StructuredGrid, VertexGrid grid information. """ - if rotated and ("angrot" in ds.attrs) and (ds.attrs["angrot"] != 0.0): + if rotated and is_rotated(ds): xoff = ds.attrs["xorigin"] yoff = ds.attrs["yorigin"] angrot = ds.attrs["angrot"] @@ -414,6 +415,7 @@ def modelgrid_to_vertex_ds(mg, ds, nodata=-1): "'modelgrid_to_vertex_ds' is deprecated and will be removed in a" "future version, please use 'modelgrid_to_ds' instead", DeprecationWarning, + stacklevel=2, ) # add modelgrid to ds @@ -453,12 +455,19 @@ def modelgrid_to_ds(mg=None, grbfile=None): x, y = mg.xyedges from .base import _get_structured_grid_ds + # temporary fix for flopy 3.11.0.dev3: + # https://github.com/modflowpy/flopy/issues/2831 + if mg.top.ndim != mg.botm[0].ndim: + top = mg.top.reshape(mg.botm[0].shape) + else: + top = mg.top + ds = _get_structured_grid_ds( xedges=x, yedges=y, nlay=mg.nlay, botm=mg.botm, - top=mg.top, + top=top, xorigin=mg.xoffset, yorigin=mg.yoffset, angrot=mg.angrot, @@ -538,10 +547,12 @@ def get_dims_coords_from_modelgrid(mg): def gridprops_to_vertex_ds(gridprops, ds, nodata=-1): - """Gridprops is a dictionary containing keyword arguments needed to generate a flopy - modelgrid instance. + """Gridprops to vertex dataset. + + Gridprops is a dictionary containing keyword arguments needed to generate + a flopy modelgrid instance. """ - _, xv, yv = zip(*gridprops["vertices"]) + _, xv, yv = zip(*gridprops["vertices"], strict=False) ds["xv"] = ("iv", np.array(xv)) ds["yv"] = ("iv", np.array(yv)) @@ -560,7 +571,7 @@ def get_vertices_from_ds(ds): Flopy needs needs this list to build a disv-package """ - vertices = list(zip(ds["iv"].data, ds["xv"].data, ds["yv"].data)) + vertices = list(zip(ds["iv"].data, ds["xv"].data, ds["yv"].data, strict=False)) return vertices @@ -758,8 +769,7 @@ def refine( def ds_to_gridprops(ds_in, gridprops, method="nearest", icvert_nodata=-1): - """Resample a xarray dataset of a structured grid to a new dataset with a vertex - grid. + """Resample a xarray dataset of a structured grid to a vertex grid. Returns a dataset with resampled variables and the untouched variables. @@ -803,8 +813,9 @@ def ds_to_gridprops(ds_in, gridprops, method="nearest", icvert_nodata=-1): interp_vars.append(key) else: logger.info( - f"Data variable {key} has spatial coordinates but it cannot be refined " - "because of its non-numeric dtype. It is not available in the output Dataset." + f"Data variable {key} has spatial coords but cannot be refined " + "because of its non-numeric dtype. " + "It is not available in the output Dataset." ) else: not_interp_vars.append(key) @@ -821,7 +832,7 @@ def ds_to_gridprops(ds_in, gridprops, method="nearest", icvert_nodata=-1): ) if is_rotated(ds_out): affine = get_affine_mod_to_world(ds_out) - ds_out["xc"], ds_out["yc"] = affine * (ds_out.x, ds_out.y) + ds_out["xc"], ds_out["yc"] = affine @ (ds_out.x, ds_out.y) if "area" in gridprops: if "area" in ds_out: @@ -863,7 +874,7 @@ def get_xyi_icell2d(gridprops=None, ds=None): xyi = np.vstack((xc_gwf, yc_gwf)).T icell2d = np.array([c[0] for c in gridprops["cell2d"]]) elif ds is not None: - xyi = np.array(list(zip(ds.x.values, ds.y.values))) + xyi = np.array(list(zip(ds.x.values, ds.y.values, strict=False))) icell2d = ds.icell2d.values else: raise ValueError("either gridprops or ds should be specified") @@ -872,9 +883,10 @@ def get_xyi_icell2d(gridprops=None, ds=None): def update_ds_from_layer_ds(ds, layer_ds, method="nearest", **kwargs): - """Add variables from a layer Dataset to a model Dataset. Keep de grid- information - from the model Dataset (x and y or icell2d), but update the layer dimension when - neccesary. + """Add variables from a layer Dataset to a model Dataset. + + Keep the grid information from the model Dataset (x and y or icell2d), + but update the layer dimension when necessary. Parameters ---------- @@ -969,12 +981,15 @@ def col_to_list(col_in, ds, cellids): # 3d grid col_lst = [ col_in.data[lay, row, col] - for lay, row, col in zip(cellids[0], cellids[1], cellids[2]) + for lay, row, col in zip( + cellids[0], cellids[1], cellids[2], strict=False + ) ] elif len(cellids) == 2: # 2d grid or vertex 3d grid col_lst = [ - col_in.data[row, col] for row, col in zip(cellids[0], cellids[1]) + col_in.data[row, col] + for row, col in zip(cellids[0], cellids[1], strict=False) ] elif len(cellids) == 1: # 2d vertex grid @@ -1075,7 +1090,7 @@ def lrc_to_reclist( else: cols.append(col_to_list(i_aux, ds, cellids)) - reclist = list(zip(zip(layers, rows, columns), *cols)) + reclist = list(zip(zip(layers, rows, columns, strict=False), *cols, strict=False)) return reclist @@ -1168,7 +1183,7 @@ def lcid_to_reclist( else: cols.append(col_to_list(i_aux, ds, cellids)) - reclist = list(zip(zip(layers, cellids[-1]), *cols)) + reclist = list(zip(zip(layers, cellids[-1], strict=False), *cols, strict=False)) return reclist @@ -1192,8 +1207,8 @@ def cols_to_reclist(ds, cellids, *args, cellid_column=0): """ cols = [col_to_list(col, ds, cellids) for col in args] if cellid_column is not None: - cols.insert(cellid_column, list(zip(*cellids))) - return list(zip(*cols)) + cols.insert(cellid_column, list(zip(*cellids, strict=False))) + return list(zip(*cols, strict=False)) def da_to_reclist( @@ -1297,7 +1312,8 @@ def da_to_reclist( ignore_cells = int(np.sum((mask) & (idomain[layer] != 1))) if ignore_cells > 0: logger.info( - f"ignore {ignore_cells} out of {np.sum(mask.values)} cells because idomain is inactive" + f"ignore {ignore_cells} out of {np.sum(mask.values)} cells " + "because idomain is inactive" ) layers = col_to_list(layer, ds, cellids) else: @@ -1336,7 +1352,8 @@ def polygon_to_area(modelgrid, polygon, da, gridtype="structured"): pass elif polygon.geom_type == "MultiPolygon": warnings.warn( - "function not tested for MultiPolygon type, can have unexpected results" + "function not tested for MultiPolygon type, can have unexpected results", + stacklevel=2, ) else: raise TypeError( @@ -1348,7 +1365,7 @@ def polygon_to_area(modelgrid, polygon, da, gridtype="structured"): if gridtype == "structured": area_array = util.get_da_from_da_ds(da, dims=("y", "x"), data=0.0) - for row, col, area in zip(df["row"], df["col"], df["areas"]): + for row, col, area in zip(df["row"], df["col"], df["areas"], strict=False): area_array[row, col] = area elif gridtype == "vertex": area_array = util.get_da_from_da_ds(da, dims=("icell2d",), data=0.0) @@ -1360,8 +1377,9 @@ def polygon_to_area(modelgrid, polygon, da, gridtype="structured"): def gdf_to_data_array_struc( gdf, gwf, field="VALUE", agg_method=None, interp_method=None ): - """Project vector data on a structured grid. Aggregate data if multiple geometries - are in a single cell. + """Project vector data on a structured grid. + + Aggregate data if multiple geometries are in a single cell. Parameters ---------- @@ -1388,8 +1406,10 @@ def gdf_to_data_array_struc( The DataArray with the projected vector data. """ warnings.warn( - "The method gdf_to_data_array_struc is deprecated. Please use gdf_to_da instead.", + "The method gdf_to_data_array_struc is deprecated. " + "Please use gdf_to_da instead.", DeprecationWarning, + stacklevel=2, ) x = gwf.modelgrid.get_xcellcenters_for_layer(0)[0] @@ -1415,9 +1435,7 @@ def gdf_to_data_array_struc( else: # aggregation not neccesary gdf_agg = gdf_cellid[[field]] - gdf_agg.set_index( - pd.MultiIndex.from_tuples(gdf_cellid.cellid.values), inplace=True - ) + gdf_agg = gdf_agg.set_index(pd.MultiIndex.from_tuples(gdf_cellid.cellid.values)) for ind, row in gdf_agg.iterrows(): da.values[ind[0], ind[1]] = row[field] @@ -1428,8 +1446,11 @@ def gdf_to_data_array_struc( def gdf_to_da( gdf, ds, column, agg_method=None, fill_value=np.nan, min_total_overlap=0.0, ix=None ): - """Project vector data on a grid. Aggregate data if multiple geometries are in a - single cell. Supports structured and vertex grids. This method replaces + """Project vector data on a grid. + + Aggregate data if multiple geometries are in a single cell. + + Supports structured and vertex grids. This method replaces gdf_to_data_array_struc. Parameters @@ -1490,14 +1511,14 @@ def gdf_to_da( # aggregation not neccesary gdf_agg = gdf_cellid[[column]] if isinstance(gdf_cellid.cellid.iloc[0], tuple): - gdf_agg.set_index( - pd.MultiIndex.from_tuples(gdf_cellid.cellid.values), inplace=True + gdf_agg = gdf_agg.set_index( + pd.MultiIndex.from_tuples(gdf_cellid.cellid.values) ) else: - gdf_agg.set_index(gdf_cellid.cellid.values, inplace=True) + gdf_agg = gdf_agg.set_index(gdf_cellid.cellid.values) if ds.gridtype == "structured": - ixs, iys = zip(*gdf_agg.index.values) + ixs, iys = zip(*gdf_agg.index.values, strict=False) da.values[ixs, iys] = gdf_agg[column] elif ds.gridtype == "vertex": da[gdf_agg.index] = gdf_agg[column] @@ -1639,8 +1660,8 @@ def aggregate_vector_per_cell(gdf, fields_methods, modelgrid=None): geom_types = gdf.geometry.type.unique() if len(geom_types) > 1: if len(geom_types) == 2 and ( - set(geom_types) == set(["LineString", "MultiLineString"]) - or set(geom_types) == set(["Polygon", "MultiPolygon"]) + set(geom_types) == {"LineString", "MultiLineString"} + or set(geom_types) == {"Polygon", "MultiPolygon"} ): pass else: @@ -2021,13 +2042,14 @@ def gdf_to_grid( shpn = shp.copy() shpn["cellid"] = row["cellids"] shpn[geometry] = row["geometry"] - if shp[geometry].geom_type == ["LineString", "MultiLineString"]: + if shp[geometry].geom_type in ["LineString", "MultiLineString"]: shpn["length"] = row["lengths"] elif shp[geometry].geom_type in ["Polygon", "MultiPolygon"]: shpn["area"] = row["areas"] shps.append(shpn) if len(shps) == 0: - # Unable to determine the column names, because no geometries intersect with the grid + # Unable to determine the column names, + # because no geometries intersect with the grid logger.info("No geometries intersect with the grid") columns = gdf.columns.to_list() + ["area", "length", "cellid"] else: @@ -2090,8 +2112,10 @@ def gdf_area_to_da( Notes ----- - Uses `flopy.utils.GridIntersect` under the hood for spatial intersection. - - For rotated grids, geometries are transformed into model space using an affine transformation. - - If the `sparse` package is installed and `sparse=True`, a `sparse.COO` array is returned. + - For rotated grids, geometries are transformed into model space using an + affine transformation. + - If the `sparse` package is installed and `sparse=True`, + a `sparse.COO` array is returned. - Suitable for use in spatial weighting or disaggregation tasks. Raises @@ -2129,7 +2153,7 @@ def gdf_area_to_da( ): df = ix.intersect(gdf.at[index, geometry], geo_dataframe=True, **kwargs) if structured: - for row, col, area in zip(df["row"], df["col"], df["areas"]): + for row, col, area in zip(df["row"], df["col"], df["areas"], strict=False): data[(row, col, irow)] += area else: data[list(df["cellid"]), irow] = df["areas"] @@ -2163,8 +2187,10 @@ def get_thickness_from_topbot(top, bot): or (layer, icell2d). """ warnings.warn( - "The method get_thickness_from_topbot is deprecated. Please use nlmod.layers.calculate_thickness instead.", + "The method get_thickness_from_topbot is deprecated. " + "Please use nlmod.layers.calculate_thickness instead.", DeprecationWarning, + stacklevel=2, ) if np.ndim(top) > 2: @@ -2188,9 +2214,10 @@ def get_thickness_from_topbot(top, bot): def get_vertices_arr(ds, modelgrid=None, vert_per_cid=4, epsilon=0, rotated=False): - """Get vertices of a vertex modelgrid from a ds or the modelgrid. Only return the 4 - corners of each cell and not the corners of adjacent cells thus limiting the - vertices per cell to 4 points. + """Get vertices of a vertex modelgrid from a ds or the modelgrid. + + Only return the 4 corners of each cell and not the corners of adjacent cells, + thus limiting the vertices per cell to 4 points. This method uses the xvertices and yvertices attributes of the modelgrid. When no modelgrid is supplied, a modelgrid-object is created from ds. @@ -2225,6 +2252,7 @@ def get_vertices_arr(ds, modelgrid=None, vert_per_cid=4, epsilon=0, rotated=Fals "this function is deprecated and will eventually be removed, " "please use 'modelgrid_from_ds' and 'modelgrid.map_polygons' in the future.", DeprecationWarning, + stacklevel=2, ) if modelgrid is None: modelgrid = modelgrid_from_ds(ds, rotated=rotated) @@ -2232,21 +2260,23 @@ def get_vertices_arr(ds, modelgrid=None, vert_per_cid=4, epsilon=0, rotated=Fals yvert = modelgrid.yvertices if vert_per_cid == 4: coord_list = [] - for xv, yv in zip(xvert, yvert): - coords = rdp(list(zip(xv, yv)), epsilon=epsilon)[:-1] + for xv, yv in zip(xvert, yvert, strict=False): + coords = rdp(list(zip(xv, yv, strict=False)), epsilon=epsilon)[:-1] if len(coords) > 4: raise RuntimeError( - "unexpected number of coördinates, you probably want to change epsilon" + "unexpected number of coördinates, " + "you probably want to change epsilon" ) coord_list.append(coords) vertices_arr = np.array(coord_list) elif vert_per_cid == 5: coord_list = [] - for xv, yv in zip(xvert, yvert): - coords = rdp(list(zip(xv, yv)), epsilon=epsilon) + for xv, yv in zip(xvert, yvert, strict=False): + coords = rdp(list(zip(xv, yv, strict=False)), epsilon=epsilon) if len(coords) > 5: raise RuntimeError( - "unexpected number of coördinates, you probably want to change epsilon" + "unexpected number of coördinates, " + "you probably want to change epsilon" ) coord_list.append(coords) vertices_arr = np.array(coord_list) @@ -2257,9 +2287,10 @@ def get_vertices_arr(ds, modelgrid=None, vert_per_cid=4, epsilon=0, rotated=Fals def get_vertices(ds, vert_per_cid=4, epsilon=0, rotated=False): - """Get vertices of a vertex modelgrid from a ds or the modelgrid. Only return the 4 - corners of each cell and not the corners of adjacent cells thus limiting the - vertices per cell to 4 points. + """Get vertices of a vertex modelgrid from a ds or the modelgrid. + + Only return the 4 corners of each cell and not the corners of adjacent cells, + thus limiting the vertices per cell to 4 points. This method uses the xvertices and yvertices attributes of the modelgrid. When no modelgrid is supplied, a modelgrid-object is created from ds. @@ -2294,6 +2325,7 @@ def get_vertices(ds, vert_per_cid=4, epsilon=0, rotated=False): "get_vertices is deprecated and will eventually be removed, " "please use 'modelgrid_from_ds' and 'modelgrid.map_polygons'.", DeprecationWarning, + stacklevel=2, ) vertices_arr = get_vertices_arr( @@ -2314,8 +2346,10 @@ def get_vertices(ds, vert_per_cid=4, epsilon=0, rotated=False): @cache.cache_netcdf(coords_2d=True) def mask_model_edge(ds, idomain=None): - """Get data array which is 1 for every active cell (defined by idomain) at the - boundaries of the model (xmin, xmax, ymin, ymax). Other cells are 0. + """Get a mask of the model edge. + + This method calculates a data array which is 1 for every active cell (defined by + idomain) at the boundaries of the model (xmin, xmax, ymin, ymax). Other cells are 0. Parameters ---------- @@ -2357,7 +2391,7 @@ def mask_model_edge(ds, idomain=None): ) elif ds.gridtype == "vertex": - polygons_grid = polygons_from_ds(ds) + polygons_grid = polygons_from_ds(ds, rotated=False) gdf_grid = gpd.GeoDataFrame(geometry=polygons_grid) extent_edge = gdf_grid.union_all().exterior cids_edge = gdf_grid.loc[gdf_grid.touches(extent_edge)].index @@ -2371,13 +2405,16 @@ def mask_model_edge(ds, idomain=None): return ds_out -def polygons_from_ds(ds): +def polygons_from_ds(ds, rotated=True): """Create polygons of each cell in a model dataset. Parameters ---------- ds : xr.Dataset Dataset with model data. + rotated : bool, optional + if True, grid-rotation is applied to the polygons. If False, the polygons are + returned in model coordinates. The default is True. Raises ------ @@ -2411,13 +2448,13 @@ def polygons_from_ds(ds): ] elif ds.gridtype == "vertex": - modelgrid = modelgrid_from_ds(ds) + modelgrid = modelgrid_from_ds(ds, rotated=False) polygons = [Polygon(v.vertices) for v in modelgrid.map_polygons] else: raise ValueError( f"gridtype must be 'structured' or 'vertex', not {ds.gridtype}" ) - if is_rotated(ds): + if rotated and is_rotated(ds): # rotate the model coordinates to real coordinates affine = get_affine_mod_to_world(ds).to_shapely() polygons = [affine_transform(polygon, affine) for polygon in polygons] @@ -2436,7 +2473,7 @@ def get_extent_polygon(ds, rotated=True): """Get the model extent, as a shapely Polygon.""" attrs = _get_attrs(ds) polygon = util.extent_to_polygon(attrs["extent"]) - if rotated and "angrot" in ds.attrs and attrs["angrot"] != 0.0: + if rotated and is_rotated(ds): affine = get_affine_mod_to_world(ds) polygon = affine_transform(polygon, affine.to_shapely()) return polygon @@ -2450,7 +2487,7 @@ def get_extent_gdf(ds, rotated=True, crs="EPSG:28992"): ds : xr.Dataset model dataset. rotated : bool, optional - if True, the extent is corrected for angrot. The default is True. + if True, the extent is corrected for angrot (rotation). The default is True. crs : str, optional Coordinate reference system. The default is "EPSG:28992". @@ -2473,14 +2510,14 @@ def affine_transform_gdf(gdf, affine): def get_extent(ds, rotated=True, xmargin=0.0, ymargin=0.0): - """Get the model extent, corrected for angrot if necessary. + """Get the model extent, corrected for angrot (rotation) if necessary. Parameters ---------- ds : xr.Dataset model dataset. rotated : bool, optional - if True, the extent is corrected for angrot. The default is True. + if True, the extent is corrected for angrot (rotation). The default is True. xmargin : float, optional margin to add to the x-extent. The default is 0.0. ymargin : float, optional @@ -2499,7 +2536,7 @@ def get_extent(ds, rotated=True, xmargin=0.0, ymargin=0.0): extent[2] - ymargin, extent[3] + ymargin, ] - if rotated and "angrot" in attrs and attrs["angrot"] != 0.0: + if rotated and is_rotated(ds): affine = get_affine_mod_to_world(ds) xc = np.array([extent[0], extent[1], extent[1], extent[0]]) yc = np.array([extent[2], extent[2], extent[3], extent[3]]) @@ -2514,7 +2551,7 @@ def get_affine_mod_to_world(ds): xorigin = attrs["xorigin"] yorigin = attrs["yorigin"] angrot = attrs["angrot"] - return Affine.translation(xorigin, yorigin) * Affine.rotation(angrot) + return Affine.translation(xorigin, yorigin) @ Affine.rotation(angrot) def get_affine_world_to_mod(ds): @@ -2523,7 +2560,7 @@ def get_affine_world_to_mod(ds): xorigin = attrs["xorigin"] yorigin = attrs["yorigin"] angrot = attrs["angrot"] - return Affine.rotation(-angrot) * Affine.translation(-xorigin, -yorigin) + return Affine.rotation(-angrot) @ Affine.translation(-xorigin, -yorigin) def get_affine(ds, sx=None, sy=None): @@ -2542,17 +2579,17 @@ def get_affine(ds, sx=None, sy=None): xorigin = attrs["xorigin"] yorigin = attrs["yorigin"] angrot = -attrs["angrot"] - # xorigin and yorigin represent the lower left corner, while for the transform we - # need the upper left + # xorigin and yorigin represent the lower left corner, + # while for the transform we need the upper left dy = attrs["extent"][3] - attrs["extent"][2] xoff = xorigin + dy * np.sin(angrot * np.pi / 180) yoff = yorigin + dy * np.cos(angrot * np.pi / 180) return ( Affine.translation(xoff, yoff) - * Affine.scale(sx, sy) - * Affine.rotation(angrot) + @ Affine.scale(sx, sy) + @ Affine.rotation(angrot) ) else: xoff = attrs["extent"][0] yoff = attrs["extent"][3] - return Affine.translation(xoff, yoff) * Affine.scale(sx, sy) + return Affine.translation(xoff, yoff) @ Affine.scale(sx, sy) diff --git a/nlmod/dims/layers.py b/nlmod/dims/layers.py index 7a43df6a..e683c848 100644 --- a/nlmod/dims/layers.py +++ b/nlmod/dims/layers.py @@ -3,6 +3,14 @@ import flopy import numpy as np + +try: + import numba + + _NUMBA_AVAILABLE = True +except ImportError: + numba = None + _NUMBA_AVAILABLE = False import xarray as xr from geopandas import GeoSeries, points_from_xy @@ -51,7 +59,8 @@ def calculate_thickness(ds, top="top", bot="botm"): else: raise ValueError("2d top should have same last dimension as bot") - # subtracting floats can result in rounding errors. Mainly anoying for zero thickness layers. + # subtracting floats can result in rounding errors. Mainly anoying for zero + # thickness layers. thickness = thickness.where(~np.isclose(thickness, 0.0), 0.0) if isinstance(ds[bot], xr.DataArray): @@ -72,8 +81,7 @@ def calculate_thickness(ds, top="top", bot="botm"): def calculate_transmissivity( ds, kh="kh", thickness="thickness", top="top", botm="botm" ): - """Calculate the transmissivity (T) as the product of the horizontal conductance - (kh) and the thickness (D). + """Calculate the transmissivity (T) as the product of the kh and thickness. Parameters ---------- @@ -83,14 +91,14 @@ def calculate_transmissivity( kh : str, optional name of data variable containing horizontal conductivity, by default 'kh' thickness : str, optional - name of data variable containing thickness, if this data variable does not exists - thickness is calculated using top and botm. By default 'thickness' + name of data variable containing thickness, if this data variable does not + exists thickness is calculated using top and botm. By default 'thickness' top : str, optional name of data variable containing tops, only used to calculate thickness if not available in dataset. By default "top" botm : str, optional - name of data variable containing bottoms, only used to calculate thickness if not - available in dataset. By default "botm" + name of data variable containing bottoms, only used to calculate thickness if + not available in dataset. By default "botm" Returns ------- @@ -127,8 +135,7 @@ def calculate_transmissivity( def calculate_resistance( ds, kv="kv", thickness="thickness", top="top", botm="botm", between_layers=None ): - """Calculate vertical resistance (c) of model layers from the vertical conductivity - (kv) and the thickness. + """Calculate resistance (c) of model layers from kv and layer thickness. Parameters ---------- @@ -138,14 +145,14 @@ def calculate_resistance( kv : str, optional name of data variable containing vertical conductivity, by default 'kv' thickness : str, optional - name of data variable containing thickness, if this data variable does not exists - thickness is calculated using top and botm. By default 'thickness' + name of data variable containing thickness, if this data variable does not + exists thickness is calculated using top and botm. By default 'thickness' top : str, optional name of data variable containing tops, only used to calculate thickness if not available in dataset. By default "top" botm : str, optional - name of data variable containing bottoms, only used to calculate thickness if not - available in dataset. By default "botm" + name of data variable containing bottoms, only used to calculate thickness if + not available in dataset. By default "botm" between_layers : bool, optional If True, calculate the resistance between the layers, which MODFLOW uses to calculate the flow. The resistance between two layers is then assigned to the @@ -273,9 +280,9 @@ def split_layers_ds( split_dict[lay0] = [1 / split_dict[lay0]] * split_dict[lay0] elif hasattr(split_dict[lay0], "__iter__"): # make sure the fractions add up to 1 - assert np.isclose( - np.sum(split_dict[lay0]), 1 - ), f"Fractions for splitting layer '{lay0}' do not add up to 1." + assert np.isclose(np.sum(split_dict[lay0]), 1), ( + f"Fractions for splitting layer '{lay0}' do not add up to 1." + ) split_dict[lay0] = split_dict[lay0] / np.sum(split_dict[lay0]) else: raise ValueError( @@ -285,7 +292,10 @@ def split_layers_ds( logger.info(f"Splitting layers {list(split_dict)}") if "layer" in ds["top"].dims: - msg = "Top in ds has a layer dimension. split_layers_ds will remove the layer dimension from top in ds." + msg = ( + "Top in ds has a layer dimension. split_layers_ds will remove the " + "layer dimension from top in ds." + ) logger.warning(msg) else: ds = ds.copy() @@ -325,7 +335,7 @@ def split_layers_ds( if return_reindexer: # determine reindexer - reindexer = dict(zip(layers, layers_org)) + reindexer = dict(zip(layers, layers_org, strict=False)) for lay0 in split_dict: reindexer.pop(lay0) return ds, reindexer @@ -545,10 +555,14 @@ def kveq_combined_layers(kv, thickness, reindexer): da_kv = _get_empty_layered_da(kv, nlay=list(reindexer.keys())[-1] + 1) for k, v in reindexer.items(): if isinstance(v, tuple): - kveq = np.nansum(thickness.data[v, ...], axis=0) / np.nansum( - thickness.data[v, ...] / kv.data[v, ...], axis=0 + numerator = np.nansum(thickness.data[v, ...], axis=0) + denominator = np.nansum(thickness.data[v, ...] / kv.data[v, ...], axis=0) + kveq = np.divide( + numerator, + denominator, + out=np.full_like(numerator, np.nan), + where=denominator != 0, ) - kveq[np.isinf(kveq)] = np.nan else: kveq = kv.data[v] da_kv.data[k] = kveq @@ -663,7 +677,27 @@ def combine_layers_ds( if isinstance(combine_layers, dict): # remove single layer entries if they exist: + combine_layers = dict(combine_layers.items()) + rename_layers = {v[0]: k for k, v in combine_layers.items() if len(v) == 1} + # remove missing layers + for k, v in combine_layers.items(): + missing = [] + for i in v: + if i not in ds.layer: + missing.append(i) + if len(missing) > 0: + logger.warning( + "Layer(s) %s not found in dataset, will be " + "removed from combine_layers.", + missing, + ) + # Preserve original layer order while removing missing entries. + combine_layers[k] = [i for i in v if i not in missing] + + # Groups can become empty or single-layer after removing missing layers. + # Keep only true merge groups to avoid reindex/name mismatches downstream. combine_layers = {k: v for k, v in combine_layers.items() if len(v) > 1} + new_layer_names = combine_layers.keys() combine_layers_integer = [ tuple(np.where(ds.layer.isin(x))[0]) if isinstance(x[0], str) else x @@ -672,6 +706,7 @@ def combine_layers_ds( else: # remove single layer entries if they exist: combine_layers = [x for x in combine_layers if len(x) > 1] + rename_layers = None combine_layers_integer = [ tuple(np.where(ds.layer.isin(x))[0]) if isinstance(x[0], str) else x for x in combine_layers @@ -697,7 +732,7 @@ def combine_layers_ds( f"Only consecutive layers can be combined. Check input: {msg}" ) # set new layer name dictionary - new_layer_names = dict(zip(combine_layers_integer, new_layer_names)) + new_layer_names = dict(zip(combine_layers_integer, new_layer_names, strict=False)) # collection for data arrays da_dict = {} @@ -749,6 +784,11 @@ def combine_layers_ds( # remove layer dimension from top ds_combine = remove_layer_dim_from_top(ds_combine, inconsistency_threshold=0.001) + if rename_layers is not None: + ds_combine = ds_combine.assign_coords( + layer=[rename_layers.get(lay, lay) for lay in ds_combine["layer"].data] + ) + if return_reindexer: return ds_combine, reindexer return ds_combine @@ -787,8 +827,10 @@ def add_kh_kv_from_ml_layer_to_ds( are ignored at the moment """ warnings.warn( - "add_kh_kv_from_ml_layer_to_ds is deprecated. Please use nlmod.grid.update_ds_from_layer_ds instead.", + "add_kh_kv_from_ml_layer_to_ds is deprecated. " + "Please use nlmod.grid.update_ds_from_layer_ds instead.", DeprecationWarning, + stacklevel=2, ) ds.attrs["anisotropy"] = anisotropy @@ -923,9 +965,33 @@ def set_layer_thickness(ds, layer, thickness, change="botm", copy=True): return ds +def get_zcellcenters(ds): + """Calculate the z-coordinates of cell centers. + + Equivalent of modelgrid.zcellcenters in flopy. + + Parameters + ---------- + ds : xarray.Dataset + dataset containing information about layers, including top and botm + + Returns + ------- + zcellcenters : xarray.DataArray + data array containing the z-coordinates of cell centers + """ + if "layer" in ds["top"].dims: + top = ds["top"] + else: + top = ds["botm"] + calculate_thickness(ds) + return (top + ds["botm"]) / 2 + + def set_minimum_layer_thickness(ds, layer, min_thickness, change="botm", copy=True): - """Make sure layer has a minimum thickness by lowering the botm of the layer where - neccesary. + """Make sure layer has a minimum thickness. + + By lowering the botm of the layer where neccesary. + """ assert layer in ds.layer assert change == "botm", "Only change=botm allowed for now" @@ -948,17 +1014,17 @@ def set_minimum_layer_thickness(ds, layer, min_thickness, change="botm", copy=Tr def remove_thin_layers( ds, min_thickness=0.1, update_thickness_every_layer=False, copy=True ): - """Remove cells with a thickness less than min_thickness (setting the thickness to - 0) + """Remove cells with a thickness less than min_thickness. - The thickness of the removed cells is added to the first active layer below + Sets the thickness to 0. The thickness of the removed cells is added to the first + active layer below Parameters ---------- ds : xr,Dataset Dataset containing information about layers. min_thickness : float, optional - THe minimum thickness of a layer. The default is 0.1. + The minimum thickness of a layer. The default is 0.1. update_thickness_every_layer : bool, optional If True, loop over the layers, from the top down, and remove thin layers, adding the thickness to the first active layer below. If the thickness of this layer is @@ -1018,8 +1084,10 @@ def remove_thin_layers( def get_kh_kv(kh, kv, anisotropy, fill_value_kh=1.0, fill_value_kv=0.1, idomain=None): - """Create kh and kv grid data for flopy from existing kh, kv and anistropy grids with - nan values (typically from REGIS). + """Create kh and kv grid data. + + From from existing kh, kv and anistropy grids with nan values (typically from + REGIS). fill nans in kh grid in these steps: 1. take kv and multiply by anisotropy, if this is nan: @@ -1608,6 +1676,7 @@ def update_idomain_from_thickness(idomain, thickness, mask): warnings.warn( "update_idomain_from_thickness is deprecated. Please use get_idomain instead.", DeprecationWarning, + stacklevel=2, ) for ilay, thick in enumerate(thickness): if ilay == 0: @@ -1628,7 +1697,9 @@ def update_idomain_from_thickness(idomain, thickness, mask): return idomain -def aggregate_by_weighted_mean_to_ds(ds, source_ds, var_name): +def aggregate_by_weighted_mean_to_ds( + ds, source_ds, var_name, source_top_name="top", source_botm_name="botm" +): """Aggregate source data to a model dataset using the weighted mean. The weighted average per model layer is calculated for the variable in the @@ -1640,6 +1711,10 @@ def aggregate_by_weighted_mean_to_ds(ds, source_ds, var_name): model dataset containing layer information (x, y, top, botm) source_ds : xr.Dataset dataset containing x, y, top, botm and a data variable to aggregate. + source_top_name : str + name of the top variable in source_ds + source_botm_name : str + name of the botm variable in source_ds var_name : str name of the data array to aggregate @@ -1658,7 +1733,11 @@ def aggregate_by_weighted_mean_to_ds(ds, source_ds, var_name): nlmod.read.geotop.aggregate_to_ds """ msg = "x and/or y coordinates do not match between 'ds' and 'source_ds'" - assert (ds.x == source_ds.x).all() and (ds.y == source_ds.y).all(), msg + assert (ds.x == source_ds.x).all(), msg + assert (ds.y == source_ds.y).all(), msg + + assert "top" in ds, "'ds' must contain 'top' variable" + assert "botm" in ds, "'ds' must contain 'botm' variable" if "layer" in ds["top"].dims: # make sure there is no layer dimension in top @@ -1669,6 +1748,17 @@ def aggregate_by_weighted_mean_to_ds(ds, source_ds, var_name): agg_ar = [] + n_src = len(source_ds.layer) + if "layer" in source_ds[source_top_name].dims: + source_ds_tops = source_ds[source_top_name] + else: + source_ds_tops = [ + source_ds[source_top_name] + .expand_dims(dim="layer") + .assign_coords({"layer": [-1]}) + ] + [source_ds[source_botm_name].isel(layer=i) for i in range(n_src - 1)] + source_ds_tops = xr.concat(source_ds_tops, dim="layer") + for ilay in range(len(ds.layer)): if ilay == 0: top = ds["top"] @@ -1676,8 +1766,8 @@ def aggregate_by_weighted_mean_to_ds(ds, source_ds, var_name): top = ds["botm"][ilay - 1].drop_vars("layer") bot = ds["botm"][ilay].drop_vars("layer") - s_top = source_ds.top - s_bot = source_ds.bottom + s_top = source_ds_tops + s_bot = source_ds[source_botm_name] s_top = s_top.where(s_top < top, top) s_top = s_top.where(s_top > bot, bot) s_bot = s_bot.where(s_bot < top, top) @@ -1693,30 +1783,39 @@ def aggregate_by_weighted_mean_to_ds(ds, source_ds, var_name): def check_elevations_consistency(ds): + """Check if the top and bottom elevations of layers are consistent. + + Parameters + ---------- + ds : xr.Dataset + The model Dataset. + """ if "layer" in ds["top"].dims: tops = ds["top"].data top_ref = np.full(tops.shape[1:], np.nan) - for lay, layer in zip(range(tops.shape[0]), ds.layer.data): + for lay, layer in zip(range(tops.shape[0]), ds.layer.data, strict=False): top = tops[lay] mask = ~np.isnan(top) higher = top[mask] > top_ref[mask] if np.any(higher): n = int(higher.sum()) logger.warning( - f"The top of layer {layer} is higher than the top of a previous layer in {n} cells" + f"The top of layer {layer} is higher than the top of a " + f"previous layer in {n} cells" ) top_ref[mask] = top[mask] bots = ds["botm"].data bot_ref = np.full(bots.shape[1:], np.nan) - for lay, layer in zip(range(bots.shape[0]), ds.layer.data): + for lay, layer in zip(range(bots.shape[0]), ds.layer.data, strict=False): bot = bots[lay] mask = ~np.isnan(bot) higher = bot[mask] > bot_ref[mask] if np.any(higher): n = int(higher.sum()) logger.warning( - f"The bottom of layer {layer} is higher the bottom of a previous layer in {n} cells" + f"The bottom of layer {layer} is higher the bottom of a " + f"previous layer in {n} cells" ) bot_ref[mask] = bot[mask] @@ -1795,7 +1894,10 @@ def insert_layer(ds, name, top, bot, kh=None, kv=None, copy=True): # make a copy, so we are sure we do not alter the original DataSet ds = ds.copy(deep=True) if "layer" in ds["top"].dims: - msg = "Top in ds has a layer dimension. insert_layer will remove the layer dimension from top in ds." + msg = ( + "Top in ds has a layer dimension. insert_layer will remove the layer " + "dimension from top in ds." + ) logger.warning(msg) else: ds["top"] = ds["botm"] + calculate_thickness(ds) @@ -1962,12 +2064,17 @@ def remove_layer(ds, layer): return ds -def get_isosurface_1d(da, z, value): - """Linear interpolation to get the elevation corresponding to value. +def get_isosurface_1d(da, z, value, left=np.nan, right=np.nan): + """Linear interpolation to get the first elevation corresponding to value. This function interpolates linearly along z, if da crosses the given interpolation value at multiple depths, the first elevation is returned. + Note + ---- + This function is no longer used in nlmod, but is kept for backward compatibility, + and as a reference for the implementations in get_isosurface. + Parameters ---------- da : 1d-array @@ -1976,17 +2083,257 @@ def get_isosurface_1d(da, z, value): array of elevations value : float value for which to compute the elevations corresponding to value + left : float, optional + value to return when value is below the minimum of da. The default is np.nan. + right : float, optional + value to return when value is above the maximum of da. The default is np.nan. Returns ------- float first elevation at which data crosses value + + See Also + -------- + get_isosurface : generalization of this function to 3D and 4D DataArrays, with + support for numba and numpy implementations. + _get_isosurface_1d_numpy : vectorized numpy implementation of this function + _get_isosurface_1d_numba : numba implementation of this function """ - mask = np.invert(np.isnan(da)) - return np.interp(value, da[mask].squeeze(), z[mask].squeeze()) + mask_valid = np.isfinite(da) + z, da = z[mask_valid], da[mask_valid] + f = da - value + if len(z) < 2: + return np.nan + + # exact first hit + idx0 = np.flatnonzero(f == 0) + if idx0.size: + return z[idx0[0]] + + # first sign change interval + s = f[:-1] * f[1:] + idx = np.flatnonzero(s < 0) + # no crossing + if not idx.size: + if value < da.min(): + return left + elif value > da.max(): + return right + else: + return np.nan + i = idx[0] + return z[i] + (value - da[i]) * (z[i + 1] - z[i]) / (da[i + 1] - da[i]) -def get_isosurface(da, z, value, input_core_dims=None, exclude_dims=None, **kwargs): +def _get_isosurface_1d_numpy(da_arr, z_arr, value, left=np.nan, right=np.nan): + """ + Vectorized numpy implementation of get_isosurface_1d. + + da_arr, z_arr: ndarrays with layer as the LAST axis. When called via + xr.apply_ufunc with input_core_dims=[["layer"], ["layer"], []], xarray + automatically moves the layer dimension to the last axis before calling + this function. + + Returns array of shape da_arr.shape[:-1]. + """ + # apply_ufunc may pass z without broadcast-only dims present in da (e.g. time). + # take_along_axis requires equal rank, so align z to da explicitly. + if z_arr.ndim != da_arr.ndim: + z_arr = np.broadcast_to(z_arr, da_arr.shape) + + valid = np.isfinite(da_arr) + f = np.where(valid, da_arr - value, np.nan) + + # --- exact hits --- + exact = f == 0.0 + has_exact = exact.any(axis=-1) + idx_exact = np.argmax(exact, axis=-1) + z_exact = np.take_along_axis(z_arr, idx_exact[..., np.newaxis], axis=-1)[..., 0] + + # --- first sign change between adjacent valid pairs --- + sc = (f[..., :-1] * f[..., 1:] < 0) & valid[..., :-1] & valid[..., 1:] + has_sc = sc.any(axis=-1) + idx_sc = np.argmax(sc, axis=-1) + + z_i = np.take_along_axis(z_arr, idx_sc[..., np.newaxis], axis=-1)[..., 0] + z_i1 = np.take_along_axis(z_arr, (idx_sc + 1)[..., np.newaxis], axis=-1)[..., 0] + da_i = np.take_along_axis(da_arr, idx_sc[..., np.newaxis], axis=-1)[..., 0] + da_i1 = np.take_along_axis(da_arr, (idx_sc + 1)[..., np.newaxis], axis=-1)[..., 0] + interp = z_i + (value - da_i) * (z_i1 - z_i) / (da_i1 - da_i) + + # --- out-of-bounds --- + da_min = np.nanmin(da_arr, axis=-1) + da_max = np.nanmax(da_arr, axis=-1) + + out = np.full(da_arr.shape[:-1], np.nan) + out = np.where(has_exact, z_exact, out) + out = np.where(~has_exact & has_sc, interp, out) + out = np.where(~has_exact & ~has_sc & (value < da_min), left, out) + out = np.where(~has_exact & ~has_sc & (value > da_max), right, out) + return out + + +if _NUMBA_AVAILABLE: + + @numba.guvectorize( + ["(float64[:], float64[:], float64, float64, float64, float64[:])"], + "(n),(n),(),(),()->()", + nopython=True, + target="parallel", # or "cpu" for single-threaded + cache=True, + ) + def _get_isosurface_1d_gufunc_numba(da, z, value, left, right, out): # numba impl + """Numba implementation of get_isosurface_1d. + + This is some wizardry that automatically returns an out variable without + having to specify that in the call. The signature of the gufunc is specified + in the decorator. + + Parameters + ---------- + da : 1d-array + array of values, e.g. concentration, pressure, etc. + z : 1d-array + array of elevations + value : float + value for which to compute the elevations corresponding to value + left : float + value to return when value is below the minimum of da. + right : float + value to return when value is above the maximum of da. + + Returns + ------- + out : float + first elevation at which data crosses value + """ + # collect valid entries + n_valid = 0 + for i in range(len(da)): + if np.isfinite(da[i]): + n_valid += 1 + if n_valid < 2: + out[0] = np.nan + return + + z_v = np.empty(n_valid) + da_v = np.empty(n_valid) + j = 0 + for i in range(len(da)): + if np.isfinite(da[i]): + z_v[j] = z[i] + da_v[j] = da[i] + j += 1 + + f0 = da_v[0] - value + da_min = da_v[0] + da_max = da_v[0] + + # exact first hit + if f0 == 0.0: + out[0] = z_v[0] + return + + for i in range(1, n_valid): + fi = da_v[i] - value + if fi == 0.0: + out[0] = z_v[i] + return + if da_v[i] < da_min: + da_min = da_v[i] + if da_v[i] > da_max: + da_max = da_v[i] + + # first sign change + fp = da_v[0] - value + for i in range(1, n_valid): + fc = da_v[i] - value + if fp * fc < 0.0: + out[0] = z_v[i - 1] + (value - da_v[i - 1]) * (z_v[i] - z_v[i - 1]) / ( + da_v[i] - da_v[i - 1] + ) + return + fp = fc + + # no crossing + if value < da_min: + out[0] = left + elif value > da_max: + out[0] = right + else: + out[0] = np.nan + + def _get_isosurface_1d_numba( + da: np.ndarray, + z: np.ndarray, + value: float, + left: float, + right: float, + ) -> np.ndarray: + """Typed wrapper so linters see the correct signature.""" + return _get_isosurface_1d_gufunc_numba(da, z, value, left, right) # pylint: disable=no-value-for-parameter + + def _get_isosurface_numba(da, z, value, left=np.nan, right=np.nan, **kwargs): + """Wrapper for numba implementation of get_isosurface_1d. + + This wrapper is needed to move the layer dimension to the last position, as + required by the gufunc, and to move the result back to an xarray DataArray with + the correct dimensions and coordinates. + + Parameters + ---------- + da : xr.DataArray + 3D or 4D DataArray with values, e.g. concentration, pressure + z : xr.DataArray + 3D DataArray with elevations + value : float + value at which to compute the elevations of the isosurface + left : float, optional + value to return when value is above the maximum of da. The default is + np.nan. + right : float, optional + value to return when value is below the minimum of da. The default is + np.nan. + kwargs : dict + additional arguments passed to xarray.apply_ufunc, not used in this + function but included for consistency with get_isosurface. + + Returns + ------- + xr.DataArray + 2D/3D DataArray with elevations of the isosurface + """ + # move layer axis to last position + layer_dim = next(d for d in da.dims if d not in {"time", "x", "y", "icell2d"}) + da_t = da.transpose(..., layer_dim) + z_t = z.transpose(..., layer_dim) + result_np = _get_isosurface_1d_numba( + da_t.values, + z_t.values, + np.float64(value), + np.float64(left), + np.float64(right), + ) + dims = [d for d in da.dims if d != layer_dim] + return xr.DataArray( + result_np, + dims=dims, + coords={d: da.coords[d] for d in dims if d in da.coords}, + ) + + +def get_isosurface( + da, + z, + value, + left=np.nan, + right=np.nan, + method="numba", + input_core_dims=None, + exclude_dims=None, + **kwargs, +): """Linear interpolation to compute the elevation of an isosurface. Currently only supports linear interpolation. @@ -1999,6 +2346,15 @@ def get_isosurface(da, z, value, input_core_dims=None, exclude_dims=None, **kwar 3D DataArray with elevations value : float value at which to compute the elevations of the isosurface + left : float, optional + value to return when value is above the maximum of da. The default is np.nan. + right : float, optional + value to return when value is below the minimum of da. The default is np.nan. + method : str, optional + method to compute the isosurface. The default is "numba". + Other option is "numpy". The numba method is usually faster than the + numpy method, but the numpy method can be faster for small datasets, and does + not require numba to be installed. input_core_dims : list of lists, optional list of core dimensions for each input, if not provided assumes core dimensions are any dimensions that are not x, y or icell2d. Example input_core_dims for @@ -2018,24 +2374,42 @@ def get_isosurface(da, z, value, input_core_dims=None, exclude_dims=None, **kwar xr.DataArray 2D/3D DataArray with elevations of the isosurface """ - if input_core_dims is None: - dims_da = set(da.dims) - {"time", "x", "y", "icell2d"} - dims_z = set(z.dims) - {"x", "y", "icell2d"} - input_core_dims = [list(dims_da), list(dims_z), []] - if exclude_dims is None: - exclude_dims = {"layer"} - - return xr.apply_ufunc( - get_isosurface_1d, - da, - z, - value, - vectorize=True, # loop over time dimension - input_core_dims=input_core_dims, - exclude_dims=exclude_dims, - dask="forbidden", - **kwargs, - ) + if method == "numba": + if not _NUMBA_AVAILABLE: + logger.warning( + "numba is not installed, falling back to numpy method for " + "get_isosurface." + ) + method = "numpy" + else: + return _get_isosurface_numba( + da, + z, + value, + left=left, + right=right, + **kwargs, + ) + if method == "numpy": + if input_core_dims is None: + dims_da = set(da.dims) - {"time", "x", "y", "icell2d"} + dims_z = set(z.dims) - {"x", "y", "icell2d"} + input_core_dims = [list(dims_da), list(dims_z), []] + if exclude_dims is None: + exclude_dims = {"layer"} + + return xr.apply_ufunc( + _get_isosurface_1d_numpy, + da, + z, + value, + input_core_dims=input_core_dims, + exclude_dims=exclude_dims, + dask="parallelized", + output_dtypes=[float], + kwargs={"right": right, "left": left}, + **kwargs, + ) def add_bathymetry_to_layer_model( @@ -2405,6 +2779,8 @@ def get_modellayers_indexer( obs_ds["modellayer"].astype(int) ].values elif drop_nan_layers: + nan_mask = obs_ds["modellayer"].isnull() + pts = pts.loc[~nan_mask.values] obs_ds = obs_ds.dropna(dim, subset=["modellayer"]) obs_ds["modellayer"].values = ds["layer"][ obs_ds["modellayer"].astype(int) @@ -2443,9 +2819,7 @@ def get_modellayers_indexer( # add local x, y coords of observation points if structured grid is rotated if full_output and grid.is_rotated(ds) and grid.is_structured(ds): affine = grid.get_affine_world_to_mod(ds) - pts_local = pts.loc[obs_ds["name"].values.tolist()].affine_transform( - affine.to_shapely() - ) + pts_local = pts.affine_transform(affine.to_shapely()) obs_ds["x_obs_local"] = (dim,), pts_local.x.values obs_ds["y_obs_local"] = (dim,), pts_local.y.values diff --git a/nlmod/dims/rdp.py b/nlmod/dims/rdp.py index 260c29f6..aebb77f1 100644 --- a/nlmod/dims/rdp.py +++ b/nlmod/dims/rdp.py @@ -1,5 +1,5 @@ """ -rdp +rdp. This module is only used in the deprecated get_vertices_arr function in the grid module. As soon as that function is removed this module can be removed as well. @@ -19,15 +19,21 @@ def pldist(point, start, end): - """Calculates the distance from ``point`` to the line given by the points ``start`` - and ``end``. - - :param point: a point - :type point: numpy array - :param start: a point of the line - :type start: numpy array - :param end: another point of the line - :type end: numpy array + """Calculates the distance from point to the line given by start and end. + + Parameters + ---------- + point : numpy array + A point. + start : numpy array + A point of the line. + end : numpy array + Another point of the line. + + Returns + ------- + float + The distance from point to the line. """ if np.all(np.equal(start, end)): return np.linalg.norm(point - start) @@ -41,12 +47,20 @@ def pldist(point, start, end): def rdp_rec(M, epsilon, dist=pldist): """Simplifies a given array of points. Recursive version. - :param M: an array - :type M: numpy array - :param epsilon: epsilon in the rdp algorithm - :type epsilon: float - :param dist: distance function - :type dist: function with signature ``f(point, start, end)`` -- see :func:`rdp.pldist` + Parameters + ---------- + M : numpy array + An array of points. + epsilon : float + Epsilon in the rdp algorithm. + dist : function, optional + Distance function with signature f(point, start, end). + See :func:`rdp.pldist`. The default is pldist. + + Returns + ------- + numpy array + Simplified array of points. """ dmax = 0.0 index = -1 @@ -99,14 +113,22 @@ def _rdp_iter(M, start_index, last_index, epsilon, dist=pldist): def rdp_iter(M, epsilon, dist=pldist, return_mask=False): """Simplifies a given array of points. Iterative version. - :param M: an array - :type M: numpy array - :param epsilon: epsilon in the rdp algorithm - :type epsilon: float - :param dist: distance function - :type dist: function with signature ``f(point, start, end)`` -- see :func:`rdp.pldist` - :param return_mask: return the mask of points to keep instead - :type return_mask: bool + Parameters + ---------- + M : numpy array + An array of points. + epsilon : float + Epsilon in the rdp algorithm. + dist : function, optional + Distance function with signature f(point, start, end). + See :func:`rdp.pldist`. The default is pldist. + return_mask : bool, optional + Return the mask of points to keep instead. The default is False. + + Returns + ------- + numpy array or bool array + Simplified array of points or mask of points to keep. """ mask = _rdp_iter(M, 0, len(M) - 1, epsilon, dist) @@ -119,42 +141,47 @@ def rdp_iter(M, epsilon, dist=pldist, return_mask=False): def rdp(M, epsilon=0, dist=pldist, algo="iter", return_mask=False): """Simplifies a given array of points using the Ramer-Douglas-Peucker algorithm. - Example: - >>> from rdp import rdp - >>> rdp([[1, 1], [2, 2], [3, 3], [4, 4]]) - [[1, 1], [4, 4]] This is a convenience wrapper around both :func:`rdp.rdp_iter` and :func:`rdp.rdp_rec` that detects if the input is a numpy array in order to adapt the output accordingly. This means that when it is called using a Python list as argument, a Python list is returned, and in case of an invocation using a numpy array, a NumPy array is returned. + The parameter ``return_mask=True`` can be used in conjunction - with ``algo="iter"`` to return only the mask of points to keep. Example: + with ``algo="iter"`` to return only the mask of points to keep. + + Parameters + ---------- + M : numpy array + A series of points with shape (n,d) where n is the number of points + and d is their dimension. + epsilon : float, optional + Epsilon in the rdp algorithm. The default is 0. + dist : function, optional + Distance function with signature f(point, start, end). + See :func:`rdp.pldist`. The default is pldist. + algo : str, optional + Either "iter" for an iterative algorithm or "rec" for a recursive + algorithm. The default is "iter". + return_mask : bool, optional + Return mask instead of simplified array. The default is False. + + Returns + ------- + numpy array or list + Simplified array of points or mask of points to keep. + + Examples + -------- >>> from rdp import rdp + >>> rdp([[1, 1], [2, 2], [3, 3], [4, 4]]) + [[1, 1], [4, 4]] >>> import numpy as np >>> arr = np.array([1, 1, 2, 2, 3, 3, 4, 4]).reshape(4, 2) - >>> arr - array([[1, 1], - [2, 2], - [3, 3], - [4, 4]]) >>> mask = rdp(arr, algo="iter", return_mask=True) - >>> mask - array([ True, False, False, True], dtype=bool) >>> arr[mask] - array([[1, 1], - [4, 4]]) - :param M: a series of points - :type M: numpy array with shape ``(n,d)`` where ``n`` is the number of points and ``d`` their dimension - :param epsilon: epsilon in the rdp algorithm - :type epsilon: float - :param dist: distance function - :type dist: function with signature ``f(point, start, end)`` -- see :func:`rdp.pldist` - :param algo: either ``iter`` for an iterative algorithm or ``rec`` for a recursive algorithm - :type algo: string - :param return_mask: return mask instead of simplified array - :type return_mask: bool + array([[1, 1], [4, 4]]) """ if algo == "iter": algo = partial(rdp_iter, return_mask=return_mask) diff --git a/nlmod/dims/shared.py b/nlmod/dims/shared.py index acb1be33..bc46a7aa 100644 --- a/nlmod/dims/shared.py +++ b/nlmod/dims/shared.py @@ -1,5 +1,12 @@ from enum import Enum +try: + import numba + + _NUMBA_AVAILABLE = True +except ImportError: + numba = None + _NUMBA_AVAILABLE = False import numpy as np import xarray as xr @@ -7,10 +14,10 @@ class GridTypeDims(Enum): """Enum for grid dimensions.""" - STRUCTURED_LAYERED = ("layer", "y", "x") VERTEX_LAYERED = ("layer", "icell2d") - STRUCTURED = ("y", "x") VERTEX = ("icell2d",) + STRUCTURED_LAYERED = ("layer", "y", "x") + STRUCTURED = ("y", "x") @classmethod def parse_dims(cls, ds): @@ -31,6 +38,14 @@ def parse_dims(cls, ds): ValueError If no partially matching gridtype is found. """ + layer_is_dim = "layer" in ds.dims + if "x" in ds and "y" in ds: + x_dims = set(ds["x"].dims) + y_dims = set(ds["y"].dims) + if "icell2d" in x_dims or "icell2d" in y_dims: + return cls.VERTEX_LAYERED if layer_is_dim else cls.VERTEX + if x_dims == {"x"} and y_dims == {"y"}: + return cls.STRUCTURED_LAYERED if layer_is_dim else cls.STRUCTURED for gridtype in GridTypeDims: if set(gridtype.value).issubset(ds.dims): return gridtype @@ -134,9 +149,7 @@ def get_delr(ds): def get_delc(ds): - """ - Get the distance along columns (delc) from the y-coordinate of a structured model - dataset. + """Get the distance along columns (delc) from y of structured model dataset. Parameters ---------- @@ -147,7 +160,6 @@ def get_delc(ds): ------- delc : np.ndarray The cell-size along columns (of length nrow). - """ assert is_structured(ds) if "extent" in ds.attrs: @@ -160,7 +172,7 @@ def get_delc(ds): def _get_delta_along_axis(x): - """Internal method to determine delr or delc from x or y relative to xmin or ymax""" + """Internal method to determine delr or delc from x or y relative to bounds.""" delr = [x[0] * 2] for xi in x[1:]: delr.append((xi - np.sum(delr)) * 2) @@ -187,6 +199,40 @@ def _shoelace_formula(x, y): return 0.5 * np.abs(np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1))) +if _NUMBA_AVAILABLE: + + @numba.njit(parallel=True) + def _compute_vertex_areas_numba(xv, yv, icvert, fill_value=-1): + ncells, max_vert = icvert.shape + areas = np.empty(ncells, dtype=np.float64) + + # parallel=True parallelizes this outer loop across your CPU cores + for i in numba.prange(ncells): + # Determine actual number of vertices for this cell (ignoring padding) + nv = 0 + for j in range(max_vert): + if icvert[i, j] == fill_value: + break + nv += 1 + + if nv < 3: + areas[i] = 0.0 + continue + + # Inline Shoelace formula (avoids array allocations like np.roll) + area_sum = 0.0 + for j in range(nv): + # Current vertex index and next vertex index (wrapped around) + v1 = icvert[i, j] + v2 = icvert[i, (j + 1) % nv] + + area_sum += xv[v1] * yv[v2] - xv[v2] * yv[v1] + + areas[i] = 0.5 * abs(area_sum) + + return areas + + def get_area(ds): """Calculate the area of each cell in the model grid. @@ -197,8 +243,8 @@ def get_area(ds): Returns ------- - ds : xr.Dataset - model dataset with an area variable + area : xr.DataArray + area of each cell """ if ds.gridtype == "structured": area = xr.DataArray( @@ -207,16 +253,22 @@ def get_area(ds): coords={"y": ds["y"], "x": ds["x"]}, ) elif ds.gridtype == "vertex": - area = np.zeros(ds["icell2d"].size) - for icell2d in ds["icell2d"]: - area[icell2d] = _shoelace_formula( - ds["xv"][ds["icvert"].isel(icell2d=icell2d)], - ds["yv"][ds["icvert"].isel(icell2d=icell2d)], - ) - area = xr.DataArray( - area, - dims=("icell2d"), - coords={"icell2d": ds["icell2d"]}, + if _NUMBA_AVAILABLE: + xv = ds["xv"].values + yv = ds["yv"].values + icvert = ds["icvert"].values + fill_val = ds["icvert"].attrs.get("nodata", -1) + area_np = _compute_vertex_areas_numba(xv, yv, icvert, fill_value=fill_val) + else: + area_np = np.zeros(ds["icell2d"].size) + for icell2d in ds["icell2d"]: + area_np[icell2d] = _shoelace_formula( + ds["xv"][ds["icvert"].isel(icell2d=icell2d)], + ds["yv"][ds["icvert"].isel(icell2d=icell2d)], + ) + + return xr.DataArray( + area_np, dims=("icell2d"), coords={"icell2d": ds["icell2d"]} ) else: raise ValueError("function only support structured or vertex gridtypes") diff --git a/nlmod/dims/time.py b/nlmod/dims/time.py index a9c6da47..61b10c35 100644 --- a/nlmod/dims/time.py +++ b/nlmod/dims/time.py @@ -81,6 +81,7 @@ def set_ds_time_deprecated( "this function is deprecated and will eventually be removed, " "please use nlmod.time.set_ds_time() in the future.", DeprecationWarning, + stacklevel=2, ) # checks @@ -155,7 +156,7 @@ def set_ds_time_deprecated( def _pd_timestamp_to_cftime(time_pd): - """Convert a pandas timestamp into a cftime stamp + """Convert a pandas timestamp into a cftime stamp. Parameters ---------- @@ -259,14 +260,19 @@ def set_ds_time( else: raise TypeError("Cannot parse start datetime.") - # parse time make sure 'time' and 'start' are same type (pd.Timestamps or cftime.datetime) + # parse time make sure 'time' and 'start' are same type + # (pd.Timestamps or cftime.datetime) if isinstance(time[0], (int, np.integer, float)): if isinstance(start, cftime.datetime): time = [start + dt.timedelta(days=int(td)) for td in time] else: time = start + pd.to_timedelta(time, time_units) elif isinstance(time[0], str): - time = pd.to_datetime(time) + try: + time = pd.to_datetime(time, format="mixed") + except TypeError: + # Fallback for older pandas versions without format="mixed". + time = pd.to_datetime(time) if isinstance(start, cftime.datetime): time = _pd_timestamp_to_cftime(time) elif isinstance(time[0], (pd.Timestamp)): @@ -274,12 +280,16 @@ def set_ds_time( time = _pd_timestamp_to_cftime(time) elif isinstance(time[0], (np.datetime64, xr.core.variable.Variable)): logger.info( - "time arguments with types np.datetime64, xr.core.variable.Variable not tested!" + "time arguments with types np.datetime64, xr.core.variable.Variable " + "not tested!" ) elif isinstance(time[0], cftime.datetime): start = _pd_timestamp_to_cftime(start) else: - msg = f"Cannot process 'time' argument. Datatype -> {type(time)} not understood." + msg = ( + f"Cannot process 'time' argument. Datatype -> {type(time)} " + "not understood." + ) raise TypeError(msg) except (OutOfBoundsDatetime, OutOfBoundsTimedelta) as e: msg = ( @@ -570,7 +580,9 @@ def estimate_nstp( ) dt_lists = [ [dt0i * tsmulti**nstpii for nstpii in range(nstpi)] - for dt0i, nstpi, tsmulti in zip(dt0_ceiled, nstp_ceiled, tsmult) + for dt0i, nstpi, tsmulti in zip( + dt0_ceiled, nstp_ceiled, tsmult, strict=False + ) ] dt_arr = np.concatenate(dt_lists) @@ -639,10 +651,23 @@ def get_time_step_length(perlen, nstp, tsmult): def ds_time_from_model(gwf): + """Deprecated: use ds_time_idx_from_model instead. + + Parameters + ---------- + gwf : flopy MFModel object + groundwater flow or groundwater transport model + + Returns + ------- + IndexVariable + time coordinate for xarray data-array or dataset + """ warnings.warn( "this function was renamed to `ds_time_idx_from_model`. " "Please use the new function name.", DeprecationWarning, + stacklevel=2, ) return ds_time_idx_from_model(gwf) @@ -664,10 +689,23 @@ def ds_time_idx_from_model(gwf): def ds_time_from_modeltime(modeltime): + """Deprecated: use ds_time_idx_from_modeltime instead. + + Parameters + ---------- + modeltime : flopy MFModel.modeltime object + model time object + + Returns + ------- + IndexVariable + time coordinate for xarray data-array or dataset + """ warnings.warn( "this function was renamed to `ds_time_idx_from_model`. " "Please use the new function name.", DeprecationWarning, + stacklevel=2, ) return ds_time_idx_from_modeltime(modeltime) @@ -786,7 +824,6 @@ def dataframe_to_flopy_timeseries( AssertionError If DataFrame contains NaNs or if `ds.time` is not datetime64. """ - msg = "No nan values are allowed in dataframe_to_flopy_timeseries" assert not df.isna().any(axis=None), msg msg = "get recharge requires a datetime64[ns] time index" @@ -801,7 +838,7 @@ def dataframe_to_flopy_timeseries( df = df.copy() df.index = (df.index - pd.to_datetime(ds.time.start)) / pd.Timedelta(1, "D") # generate a list of tuples with time as the first record, followed by the columns - timeseries = [(i,) + tuple(v) for i, v in zip(df.index, df.values)] + timeseries = [(i,) + tuple(v) for i, v in zip(df.index, df.values, strict=False)] if package is None: return timeseries if filename is None: diff --git a/nlmod/gis.py b/nlmod/gis.py index 0f04f10c..87cd6d6a 100644 --- a/nlmod/gis.py +++ b/nlmod/gis.py @@ -47,9 +47,9 @@ def vertex_da_to_gdf( gdf : geopandas.GeoDataframe geodataframe of one or more DataArrays. """ - assert ( - model_ds.gridtype == "vertex" - ), f"expected model dataset with gridtype vertex, got {model_ds.gridtype}" + assert model_ds.gridtype == "vertex", ( + f"expected model dataset with gridtype vertex, got {model_ds.gridtype}" + ) if isinstance(data_variables, str): data_variables = [data_variables] @@ -126,9 +126,9 @@ def struc_da_to_gdf( gdf : geopandas.GeoDataframe geodataframe of one or more DataArrays. """ - assert ( - model_ds.gridtype == "structured" - ), f"expected model dataset with gridtype vertex, got {model_ds.gridtype}" + assert model_ds.gridtype == "structured", ( + f"expected model dataset with gridtype vertex, got {model_ds.gridtype}" + ) if isinstance(data_variables, str): data_variables = [data_variables] @@ -150,7 +150,8 @@ def struc_da_to_gdf( dv_dic[f"{da_name}_mean"] = da_mean.values.flatten("F") else: raise NotImplementedError( - "Can only use the mean of a DataArray with dimension time, use dealing_with_time='mean'" + "Can only use the mean of a DataArray with dimension time, " + "use dealing_with_time='mean'" ) else: raise ValueError( @@ -158,7 +159,8 @@ def struc_da_to_gdf( ) else: raise NotImplementedError( - f"expected two or three dimensions got {no_dims} for data variable {da_name}" + f"expected two or three dimensions got {no_dims} for " + f"data variable {da_name}" ) # create geometries @@ -293,7 +295,8 @@ def ds_to_vector_file( da_names -= item else: logger.info( - f"could not add {item} into to geopackage because 1 or more of the data variables do not exist" + f"could not add {item} into to geopackage because 1 or more of " + "the data variables do not exist" ) # create unique shapefiles for the other data variables @@ -328,8 +331,9 @@ def ds_to_ugrid_nc_file( split_time_dimension=False, for_imod_qgis_plugin=False, ): - """Save a model dataset to a UGRID NetCDF file, so it can be opened as a Mesh Layer - in qgis. + """Save a model dataset to a UGRID NetCDF file. + + So it can be opened as a Mesh Layer in qgis. Parameters ---------- diff --git a/nlmod/gwf/gwf.py b/nlmod/gwf/gwf.py index b7ee309d..574f4340 100644 --- a/nlmod/gwf/gwf.py +++ b/nlmod/gwf/gwf.py @@ -113,7 +113,7 @@ def _dis(ds, model, length_units="METERS", pname="dis", **kwargs): yorigin = ds.extent[2] angrot = 0.0 - idomain = get_idomain(ds).data + idomain = kwargs.pop("idomain", get_idomain(ds).data) if model.model_type == "gwf6": filename = f"{ds.model_name}.dis" klass = flopy.mf6.ModflowGwfdis @@ -257,8 +257,8 @@ def npf( vertical hydraulic conductivity, when passed as string, the array is obtained from ds. By default assumes data is stored as "kv". save_flows : bool, optional - value is passed to flopy.mf6.ModflowGwfnpf() to determine if cell by - cell flows should be saved to the cbb file. Default is False + Value is passed to flopy.mf6.ModflowGwfnpf() to determine if cell by + cell flows should be saved to the budget file. The default is False. pname : str, optional package name @@ -301,6 +301,7 @@ def ghb( pname="ghb", auxiliary=None, layer=None, + save_flows=True, **kwargs, ): """Create general head boundary from model dataset. @@ -324,6 +325,9 @@ def ghb( layer : int or None The layer in which the boundary is added. It is added to the first active layer when layer is None. The default is None. + save_flows : bool, optional + Value is passed to flopy.mf6.ModflowGwfghb() to determine if flows should be + saved to the budget file. The default is True. Raises ------ @@ -358,7 +362,7 @@ def ghb( auxiliary="CONCENTRATION" if auxiliary is not None else None, maxbound=len(ghb_rec), stress_period_data=ghb_rec, - save_flows=True, + save_flows=save_flows, pname=pname, **kwargs, ) @@ -382,6 +386,7 @@ def drn( cond="drn_cond", pname="drn", layer=None, + save_flows=True, **kwargs, ): """Create drain from model dataset. @@ -407,6 +412,9 @@ def drn( layer : int or None The layer in which the boundary is added. It is added to the first active layer when layer is None. The default is None. + save_flows : bool, optional + Value is passed to flopy.mf6.ModflowGwfdrn() to determine if flows should be + saved to the budget file. The default is True. Returns ------- @@ -434,7 +442,7 @@ def drn( gwf, maxbound=len(drn_rec), stress_period_data=drn_rec, - save_flows=True, + save_flows=save_flows, pname=pname, **kwargs, ) @@ -455,6 +463,7 @@ def riv( pname="riv", auxiliary=None, layer=None, + save_flows=True, **kwargs, ): """Create river package from model dataset. @@ -484,6 +493,9 @@ def riv( layer : int or None The layer in which the boundary is added. It is added to the first active layer when layer is None. The default is None. + save_flows : bool, optional + Value is passed to flopy.mf6.ModflowGwfriv() to determine if flows should be + saved to the budget file. The default is True. Returns ------- @@ -514,7 +526,7 @@ def riv( maxbound=len(riv_rec), stress_period_data=riv_rec, auxiliary="CONCENTRATION" if auxiliary is not None else None, - save_flows=True, + save_flows=save_flows, pname=pname, **kwargs, ) @@ -538,6 +550,7 @@ def chd( pname="chd", auxiliary=None, layer=0, + save_flows=True, **kwargs, ): """Create constant head package from model dataset. @@ -561,6 +574,9 @@ def chd( layer : int or None The layer in which the boundary is added. It is added to the first active layer when layer is None. The default is 0. + save_flows : bool, optional + Value is passed to flopy.mf6.ModflowGwfchd() to determine if flows should be + saved to the budget file. The default is True. chd : str, optional deprecated, the new argument is 'mask' @@ -575,6 +591,7 @@ def chd( warnings.warn( "the 'chd' kwarg has been renamed to 'mask'!", DeprecationWarning, + stacklevel=2, ) mask = kwargs.pop("chd") @@ -599,7 +616,7 @@ def chd( pname=pname, maxbound=len(chd_rec), stress_period_data=chd_rec, - save_flows=True, + save_flows=save_flows, **kwargs, ) if (auxiliary is not None) and (ds.transport == 1): @@ -674,8 +691,8 @@ def sto( iconvert : int, optional See description in ModflowGwfsto. The default is 1 (differs from FloPY). save_flows : bool, optional - value is passed to flopy.mf6.ModflowGwfsto() to determine if flows - should be saved to the cbb file. Default is False + Value is passed to flopy.mf6.ModflowGwfsto() to determine if flows + should be saved to the budget file. The default is False. pname : str, optional package name @@ -710,7 +727,9 @@ def sto( return sto -def surface_drain_from_ds(ds, gwf, resistance, elev="ahn", pname="drn", **kwargs): +def surface_drain_from_ds( + ds, gwf, resistance, elev="ahn", pname="drn", save_flows=True, **kwargs +): """Create surface level drain (maaivelddrainage in Dutch) from the model dataset. Parameters @@ -728,6 +747,9 @@ def surface_drain_from_ds(ds, gwf, resistance, elev="ahn", pname="drn", **kwargs the elevation data is stored under "ahn". pname : str, optional package name + save_flows : bool, optional + Value is passed to flopy.mf6.ModflowGwfdrn() to determine if flows + should be saved to the budget file. The default is True. Returns ------- @@ -755,7 +777,7 @@ def surface_drain_from_ds(ds, gwf, resistance, elev="ahn", pname="drn", **kwargs pname=pname, maxbound=len(drn_rec), stress_period_data={0: drn_rec}, - save_flows=True, + save_flows=save_flows, **kwargs, ) @@ -941,8 +963,8 @@ def oc( default is True. print_head : bool or str Prints the head to the list-file. If print_head is a string, it needs to be - "all", "first" or "last". If print_head is True, it is set to "last". The default - is False. + "all", "first" or "last". If print_head is True, it is set to "last". + The default is False. print_budget : bool or str Prints the budgets to the list-file. If print_budget is a string, it needs to be "all", "first" or "last". If print_budget is True, it is set to "last". The @@ -962,8 +984,12 @@ def oc( head_filerecord = [headfile] budgetfile = f"{ds.model_name}.cbc" budget_filerecord = [budgetfile] - saverecord = _set_record(save_head, save_budget, output="head") - printrecord = _set_record(print_head, print_budget, output="head") + saverecord = kwargs.pop("saverecord", None) + if saverecord is None: + saverecord = _set_record(save_head, save_budget, output="head") + printrecord = kwargs.pop("printrecord", None) + if printrecord is None: + printrecord = _set_record(print_head, print_budget, output="head") oc = flopy.mf6.ModflowGwfoc( gwf, diff --git a/nlmod/gwf/hfb.py b/nlmod/gwf/hfb.py index 6f166bef..1ad422c8 100644 --- a/nlmod/gwf/hfb.py +++ b/nlmod/gwf/hfb.py @@ -6,12 +6,15 @@ import numpy as np import pandas as pd import xarray as xr -from geopandas import GeoSeries -from shapely.geometry import Point, Polygon +from flopy.utils import make_hfb_array +from geopandas import GeoDataFrame, GeoSeries +from shapely.geometry import MultiLineString, Point, Polygon from ..dims.grid import ( gdf_to_da, gdf_to_grid, + get_delc, + get_delr, get_node_structured, is_structured, is_vertex, @@ -22,17 +25,184 @@ logger = logging.getLogger(__name__) +LINE_GEOM_TYPES = {"LineString", "MultiLineString"} + + +def hfb_from_df( + df, + gwf, + ds, + *, + hydchr="hydchr", + depth="depth", + elevation="elevation", + pname="hfb", + **kwargs, +): + """Add a Horizontal Flow Barrier (HFB) package from line features. + + Parameters + ---------- + df : geopandas.GeoDataFrame + GeoDataFrame with LineString or MultiLineString features. + gwf : flopy.mf6.ModflowGwf + Groundwater flow model to add the HFB package to. + ds : xr.Dataset + Dataset with model data. Used for grid intersection, active cells, and layer + placement. + hydchr : str or float, optional + Column in ``df`` with the hydraulic characteristic, or a single value applied + to every feature. The default is "hydchr". + depth : str or float, optional + Column in ``df`` with the barrier depth below model top, or a single value + applied to every feature. Exactly one of ``depth`` or ``elevation`` must be set + per feature. The default is "depth". + elevation : str or float, optional + Column in ``df`` with the barrier bottom elevation, or a single value applied + to every feature. Exactly one of ``depth`` or ``elevation`` must be set per + feature. The default is "elevation". + pname : str, optional + Package name. The default is "hfb". + **kwargs : dict + Keyword arguments are passed to ``flopy.mf6.ModflowGwfhfb``. + + Returns + ------- + hfb : flopy.mf6.ModflowGwfhfb or None + HFB package. Returns None when no stress-period data are generated. + """ + logger.info("creating mf6 HFB from dataframe") + + if not isinstance(df, GeoDataFrame): + raise TypeError("df must be a geopandas GeoDataFrame") + + if df.empty: + logger.warning("no hfb pkg added") + return None + + unsupported = set(df.geometry.geom_type.dropna().unique()) - LINE_GEOM_TYPES + if unsupported: + raise ValueError( + "hfb_from_df supports only LineString and MultiLineString geometries; " + f"found {sorted(unsupported)}", + ) + + spd = [] + for row_number, (index, row) in enumerate(df.iterrows()): + hydchr_value = _get_optional_hfb_value(row, hydchr, "hydchr", index) + if hydchr_value is None: + raise ValueError(f"hydchr must be set for HFB feature at index {index!r}") + if hydchr_value <= 0: + raise ValueError( + f"hydchr must be positive for HFB feature at index {index!r}", + ) + + depth_value = _get_optional_hfb_value(row, depth, "depth", index) + elevation_value = _get_optional_hfb_value(row, elevation, "elevation", index) + if (depth_value is None) == (elevation_value is None): + raise ValueError( + "Exactly one of depth or elevation must be set for HFB feature " + f"at index {index!r}", + ) + + feature = df.iloc[[row_number]] + if depth_value is not None: + spd += get_hfb_spd(ds, feature, hydchr=hydchr_value, depth=depth_value) + else: + spd += get_hfb_spd( + ds, + feature, + hydchr=hydchr_value, + elevation=elevation_value, + ) + + spd = _clean_hfb_spd(spd) + if len(spd) == 0: + logger.warning("no hfb pkg added") + return None + + maxhfb = kwargs.pop("maxhfb", len(spd)) + return flopy.mf6.ModflowGwfhfb( + gwf, + stress_period_data={0: spd}, + maxhfb=maxhfb, + pname=pname, + **kwargs, + ) + + +def _get_optional_hfb_value(row, value_or_column, name, index): + if value_or_column is None: + return None + if isinstance(value_or_column, str): + if value_or_column not in row.index: + raise KeyError( + f"Column {value_or_column!r} not found for {name} of HFB feature " + f"at index {index!r}", + ) + value = row[value_or_column] + else: + value = value_or_column + if pd.isna(value): + return None + return float(value) + + +def _clean_hfb_spd(spd): + return [ + [cellid1, cellid2, hydchr_float] + for cellid1, cellid2, hydchr in spd + if (hydchr_float := float(hydchr)) != 0 + ] + + +def _iter_linestring_geometries(linestrings): + if isinstance(linestrings, GeoDataFrame): + geometries = linestrings.geometry + elif isinstance(linestrings, GeoSeries): + geometries = linestrings + else: + geometries = [linestrings] + + for geometry in geometries: + if geometry is None: + continue + if isinstance(geometry, MultiLineString): + yield from geometry.geoms + else: + yield geometry + + +def _get_hfb_cells_from_linestrings(ds, linestrings, idomain): + modelgrid = modelgrid_from_ds(ds, idomain=idomain.values) + cells = [] + for geometry in _iter_linestring_geometries(linestrings): + cells.extend( + [ + (tuple(int(i) for i in row.cellid1), tuple(int(i) for i in row.cellid2)) + for row in make_hfb_array(modelgrid, geometry) + ] + ) + return cells + def get_hfb_spd(ds, linestrings, hydchr, depth=None, elevation=None): - """Generate a stress period data for horizontal flow barrier between two cell nodes, - with several limitations. The stress period data can be used directly in the HFB - package of flopy. The hfb is placed at the cell interface; it follows the sides of - the cells. + """Generate HFB stress period data between two cell nodes. + + The function has several limitations. The stress period data can be used directly + in the HFB package of flopy. The hfb is placed at the cell interface; it follows + the sides of the cells. The estimation of the cross-sectional area at the interface is pretty crude, as the thickness at the cell interface is just the average of the thicknesses of the two cells. + For the layer that contains the bottom of the barrier, a negative HYDCHR is used. + MODFLOW 6 then multiplies its own conductance between the two cells by that + factor. The factor combines the open part of the face (below the barrier) with the + walled part, so a barrier that barely penetrates a layer leaves the face nearly + open. For an impermeable barrier the factor is simply the open fraction. + Parameters ---------- ds : xr.Dataset @@ -54,88 +224,141 @@ def get_hfb_spd(ds, linestrings, hydchr, depth=None, elevation=None): spd : List of Tuple Stress period data used to configure the hfb package of Flopy. """ - assert ( - sum([depth is None, elevation is None]) == 1 - ), "Use either depth or elevation argument" + assert sum([depth is None, elevation is None]) == 1, ( + "Use either depth or elevation argument" + ) if not isinstance(ds, xr.Dataset): raise TypeError("Please pass a model dataset!") - thick = calculate_thickness(ds) + thick = calculate_thickness(ds).values idomain = get_idomain(ds) tops = np.concatenate((ds["top"].values[np.newaxis], ds["botm"].values)) - cells = line_to_hfb(linestrings, ds) + cells = _get_hfb_cells_from_linestrings(ds, linestrings, idomain) - # drop cells on the edge of the model - cells = [x for x in cells if len(x) > 1] + kh = ds["kh"].values spd = [] - for icell2d1, icell2d2 in cells: - # TODO: Improve assumption of the thickness between the cells. - if isinstance(icell2d1, (int, np.integer)): - thicki = (thick[:, icell2d1] + thick[:, icell2d2]) / 2 - topi = (tops[:, icell2d1] + tops[:, icell2d2]) / 2 - else: - thicki = (thick[:, *icell2d1] + thick[:, *icell2d2]) / 2 - topi = (tops[*icell2d1] + tops[*icell2d2]) / 2 + for cellid1, cellid2 in cells: + if idomain.values[cellid1] <= 0: + continue - for ilay in range(ds.sizes["layer"]): - cellid1 = (ilay,) + ( - icell2d1 if isinstance(icell2d1, tuple) else (icell2d1,) - ) - cellid2 = (ilay,) + ( - icell2d2 if isinstance(icell2d2, tuple) else (icell2d2,) - ) + if idomain.values[cellid2] <= 0: + continue - if idomain.values[cellid1] <= 0: - continue + ilay = cellid1[0] + # TODO: Improve assumption of the thickness between the cells. + cell_index1 = (slice(None), *cellid1[1:]) + cell_index2 = (slice(None), *cellid2[1:]) + thicki = (thick[cell_index1] + thick[cell_index2]) / 2 + topi = (tops[cell_index1] + tops[cell_index2]) / 2 + + if depth is not None: + layer_top_depth = sum(thicki[:ilay]) + layer_bottom_depth = sum(thicki[: ilay + 1]) + if layer_bottom_depth <= depth: + # hfb spans the entire cell + spd.append([cellid1, cellid2, hydchr]) + + elif layer_top_depth <= depth: + # hfb spans the cell partially + hydchr_frac = (depth - layer_top_depth) / thicki[ilay] + if not 0 <= hydchr_frac <= 1: + raise RuntimeError("HFB depth fraction is outside [0, 1]") + + _append_partial_hydchr( + spd, ds, cellid1, cellid2, hydchr, hydchr_frac, kh + ) + + elif topi[ilay + 1] >= elevation: + # hfb spans the entire cell + spd.append([cellid1, cellid2, hydchr]) + + elif topi[ilay] >= elevation: + # hfb spans the cell partially + hydchr_frac = (topi[ilay] - elevation) / thicki[ilay] + if not 0 <= hydchr_frac <= 1: + raise RuntimeError("HFB elevation fraction is outside [0, 1]") + + _append_partial_hydchr(spd, ds, cellid1, cellid2, hydchr, hydchr_frac, kh) - if idomain.values[cellid2] <= 0: - continue + return spd - if depth is not None: - if sum(thicki[: ilay + 1]) <= depth: - # hfb spans the entire cell - spd.append([cellid1, cellid2, hydchr]) - elif sum(thicki[:ilay]) <= depth: - # hfb spans the cell partially - hydchr_frac = (depth - sum(thicki[:ilay])) / thicki[ilay] - assert 0 <= hydchr_frac <= 1, "Something is wrong" +def _append_partial_hydchr(spd, ds, cellid1, cellid2, hydchr, frac, kh): + """Append the HYDCHR for a layer that the barrier penetrates over ``frac``. - spd.append([cellid1, cellid2, hydchr * hydchr_frac]) - break # go to next cell + The face consists of an open strip over ``1 - frac`` in parallel with a walled + strip over ``frac``, where the barrier ``b = hydchr`` is in series with the + aquifer. MF6 multiplies its own cell-to-cell conductance by ``-HYDCHR`` when + HYDCHR is negative; the factor is ``(1 - frac) + frac * b / (c + b)``, which is + ``1 - frac`` for an impermeable barrier. + ``c = 1 / (d1 / k1 + d2 / k2)`` is the aquifer conductance per unit face area, + with ``d`` the distance from each cell center to the shared face. When kh is not + positive, only the open-strip factor ``1 - frac`` is used. + """ + if frac == 0: + # the layer is not penetrated; leave the face open + return + kh1 = kh[cellid1] + kh2 = kh[cellid2] + if np.isfinite(kh1) and np.isfinite(kh2) and kh1 > 0 and kh2 > 0: + d1, d2 = _center_to_face_distances(ds, cellid1, cellid2) + c = 1.0 / (d1 / kh1 + d2 / kh2) + walled = hydchr / (c + hydchr) + else: + walled = 0.0 + spd.append([cellid1, cellid2, -float((1.0 - frac) + frac * walled)]) - else: - if topi[ilay + 1] >= elevation: - # hfb spans the entire cell - spd.append([cellid1, cellid2, hydchr]) - else: - # hfb spans the cell partially - hydchr_frac = (topi[ilay] - elevation) / thicki[ilay] - assert 0 <= hydchr_frac <= 1, "Something is wrong" +def _center_to_face_distances(ds, cellid1, cellid2): + """Return the distances from both cell centers to their shared face.""" + if is_structured(ds): + if cellid1[1] == cellid2[1]: + delr = get_delr(ds) + return delr[cellid1[2]] / 2, delr[cellid2[2]] / 2 + delc = get_delc(ds) + return delc[cellid1[1]] / 2, delc[cellid2[1]] / 2 - spd.append([cellid1, cellid2, hydchr * hydchr_frac]) - break # go to next cell + icvert = ds["icvert"].values + nodata = ds["icvert"].attrs.get("nodata", -1) - return spd + def vertices(icell2d): + return {int(iv) for iv in icvert[icell2d] if np.isfinite(iv) and iv != nodata} + + shared = sorted(vertices(cellid1[1]) & vertices(cellid2[1])) + xa, ya = ds["xv"].values[shared[0]], ds["yv"].values[shared[0]] + xb, yb = ds["xv"].values[shared[-1]], ds["yv"].values[shared[-1]] + length = np.hypot(xb - xa, yb - ya) + distances = [] + for icell2d in (cellid1[1], cellid2[1]): + x, y = ds["x"].values[icell2d], ds["y"].values[icell2d] + distances.append(abs((xb - xa) * (ya - y) - (xa - x) * (yb - ya)) / length) + return tuple(distances) def line2hfb(gdf, ds=None, gwf=None, prevent_rings=True, plot=False): + """Snap line to grid and return HFB cell pairs.""" warnings.warn( - "The function 'line2hfb' is deprecated and will be removed in a future version. " - "Please use 'line_to_hfb' instead.", + "The function 'line2hfb' is deprecated and will be removed in a future " + "version. Please use 'flopy.utils.make_hfb_array' for cell-pair routing, " + "or 'get_hfb_spd'/'hfb_from_df' for nlmod HFB stress-period data.", DeprecationWarning, stacklevel=2, ) - return line_to_hfb( - gdf=gdf, - ds=ds, - gwf=gwf, - prevent_rings=prevent_rings, - plot=plot, - ) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="The function 'line_to_hfb' is deprecated.*", + category=DeprecationWarning, + ) + return line_to_hfb( + gdf=gdf, + ds=ds, + gwf=gwf, + prevent_rings=prevent_rings, + plot=plot, + ) def line_to_hfb(gdf, ds=None, gwf=None, prevent_rings=True, plot=False): @@ -165,6 +388,14 @@ def line_to_hfb(gdf, ds=None, gwf=None, prevent_rings=True, plot=False): cellids : 2d list of ints a list with pairs of cells that have a hfb between them. """ + warnings.warn( + "The function 'line_to_hfb' is deprecated and will be removed in a future " + "version. Please use 'flopy.utils.make_hfb_array' for cell-pair routing, " + "or 'get_hfb_spd'/'hfb_from_df' for nlmod HFB stress-period data.", + DeprecationWarning, + stacklevel=2, + ) + # for the idea, see: # https://gis.stackexchange.com/questions/188755/how-to-snap-a-road-network-to-a-hexagonal-grid-in-qgis @@ -176,13 +407,13 @@ def line_to_hfb(gdf, ds=None, gwf=None, prevent_rings=True, plot=False): else: raise TypeError( "Please pass either a flopy.discretization.grid.Grid or " - "flopy.mf6.ModflowGwf object as gwf." + "flopy.mf6.ModflowGwf object as gwf.", ) elif ds is not None: mgrid = modelgrid_from_ds(ds) else: raise ValueError( - "Please pass either a dataset or a flopy.mf6.ModflowGwf object." + "Please pass either a dataset or a flopy.mf6.ModflowGwf object.", ) gdfg = gdf_to_grid(gdf, gwf if gwf is not None else ds, desc="Intersecting HFB(s)") @@ -202,22 +433,24 @@ def line_to_hfb(gdf, ds=None, gwf=None, prevent_rings=True, plot=False): cell2d.index.name = "icell2d" icvert = np.array( [ - mgrid._build_structured_iverts(*mgrid.get_lrc(icpl)[0][1:]) + mgrid._build_structured_iverts( # pylint: disable=protected-access + *mgrid.get_lrc(icpl)[0][1:], + ) for icpl in range(mgrid.ncpl) - ] + ], ) # add first vertex to the end of the list icvert = np.hstack([icvert, icvert[:, :1]]) gdfg["cellid_structured"] = gdfg["cellid"] gdfg["cellid"] = gdfg["cellid"].map( - lambda cid: get_node_structured(0, *cid, shape=mgrid.shape) + lambda cid: get_node_structured(0, *cid, shape=mgrid.shape), ) elif mgrid.grid_type == "vertex": cell2d = pd.DataFrame(mgrid.cell2d) cell2d.columns = ["icell2d", "xc", "yc", "ncvert"] + [ f"icvert_{i}" for i in range(cell2d.columns.size - 4) ] - cell2d.set_index("icell2d", inplace=True) + cell2d = cell2d.set_index("icell2d") vertices = pd.DataFrame( index=np.arange(mgrid.nvert), data=mgrid.verts, @@ -228,7 +461,7 @@ def line_to_hfb(gdf, ds=None, gwf=None, prevent_rings=True, plot=False): else: raise ValueError( f"gridtype {mgrid.grid_type} not supported. Only 'structured' " - "and 'vertex' are supported." + "and 'vertex' are supported.", ) # for every cell determine which cell-edge could form the line @@ -273,15 +506,6 @@ def line_to_hfb(gdf, ds=None, gwf=None, prevent_rings=True, plot=False): # get unique segments hfb_seg_unique = np.unique(hfb_seg[:, 1:], axis=0) - # NOTE: see next note describing the idea behind this code. Leaving here for future - # review. - # hfb_seg_unique, uidx = np.unique(hfb_seg[:, 1:], return_index=True, axis=0) - # ucids = hfb_seg[uidx, 0] - # try: - # endpoint_cellid = mgrid.intersect(get_coordinates(gdf.geometry.union_all()[-1])) - # except Exception: # e.g. point is on our outside model boundary - # endpoint_cellid = ucids[-1] # pick last cellid (this might not be the endpoint) - # n_segments_endpoint_cell = 0 # Get rid of disconnected (or 'open') segments # Let's remove disconnected/open segments @@ -294,18 +518,6 @@ def line_to_hfb(gdf, ds=None, gwf=None, prevent_rings=True, plot=False): segments_per_iv[segment[1]] == 1 and segments_per_iv[segment[0]] >= 3 ): mask[i] = False - # NOTE: the code below can be used to avoid dropping all segments in the - # final cell, leaving it here for review in the future. - # if ensure_endpoint_segment: - # # keep at least one segment in final cell - # if ucids[i] == endpoint_cellid and n_segments_endpoint_cell == 0: - # continue - # else: - # mask[i] = False - # else: - # mask[i] = False - # elif ensure_endpoint_segment and ucids[i] == endpoint_cellid: - # n_segments_endpoint_cell += 1 hfb_seg_unique = hfb_seg_unique[mask] if plot: @@ -368,10 +580,18 @@ def line_to_hfb_buffer(gdf, ds, buffer_distance=None, gi=None): hfb_seg : list of tuples List of tuples with cellids that share a face. """ + warnings.warn( + "The function 'line_to_hfb_buffer' is deprecated and will be removed in a " + "future version. Please use 'flopy.utils.make_hfb_array' for cell-pair " + "routing, or 'get_hfb_spd'/'hfb_from_df' for nlmod HFB stress-period data.", + DeprecationWarning, + stacklevel=2, + ) + if buffer_distance is None: # buffer distance ~ 1.5 * max cell size buffer_distance = 1.5 * np.max( - [np.max(np.abs(np.diff(ds["x"]))), np.max(np.abs(np.diff(ds["y"])))] + [np.max(np.abs(np.diff(ds["x"]))), np.max(np.abs(np.diff(ds["y"])))], ) zone_left = gdf.buffer(-buffer_distance, single_sided=True) @@ -386,7 +606,7 @@ def line_to_hfb_buffer(gdf, ds, buffer_distance=None, gi=None): logger.warning( "The left and right buffer zones intersect. Please check your input." "Perhaps there are multiple linestrings, or the linestring geometry is " - "causing potential issues. You can also try modifying the buffer_distance." + "causing potential issues. You can also try modifying the buffer_distance.", ) zone_gdf = pd.concat([zone_left, zone_right], axis=0).reset_index(drop=True) @@ -395,7 +615,7 @@ def line_to_hfb_buffer(gdf, ds, buffer_distance=None, gi=None): mgrid = modelgrid_from_ds(ds) hfb_seg = [] if is_structured(ds): - for cid in zip(*cellids): + for cid in zip(*cellids, strict=False): neighbors = mgrid.neighbors(0, *cid) for n in neighbors: if da.values[n[1:]] == -1: @@ -409,13 +629,20 @@ def line_to_hfb_buffer(gdf, ds, buffer_distance=None, gi=None): else: raise ValueError( f"gridtype {mgrid.grid_type} not supported. Only 'structured' " - "and 'vertex' are supported." + "and 'vertex' are supported.", ) return hfb_seg -def polygon_to_hfb( - gdf, ds, hydchr, column=None, gwf=None, lay=0, add_data=False, include_nan=True +def polygon_to_hfb( # pylint: disable=too-many-positional-arguments + gdf, + ds, + hydchr, + column=None, + gwf=None, + lay=0, + add_data=False, + include_nan=True, ): """Snap polygon exterior to grid to form a horizontal flow barrier. @@ -504,8 +731,7 @@ def polygon_to_hfb( spd[-1].extend([data[icell2d1], data[icell2d2]]) if gwf is None: return spd - else: - return flopy.mf6.ModflowGwfhfb(gwf, stress_period_data={0: spd}) + return flopy.mf6.ModflowGwfhfb(gwf, stress_period_data={0: spd}) def plot_hfb(cellids, modelgrid, ax=None, color="red", **kwargs): @@ -545,8 +771,12 @@ def plot_hfb(cellids, modelgrid, ax=None, color="red", **kwargs): spd = cellids.stress_period_data.data[0] cellids = [[row[0][1:], row[1][1:]] for row in spd] for line in cellids: - pc1 = Polygon(modelgrid.get_cell_vertices(*line[0])) - pc2 = Polygon(modelgrid.get_cell_vertices(*line[1])) + if len(line) == 2: + pc1 = Polygon(modelgrid.get_cell_vertices(*line[0])) + pc2 = Polygon(modelgrid.get_cell_vertices(*line[1])) + elif len(line) == 4: + pc1 = Polygon(modelgrid.get_cell_vertices(*line[:2])) + pc2 = Polygon(modelgrid.get_cell_vertices(*line[2:])) x, y = pc1.intersection(pc2).xy ax.plot(x, y, color=color, **kwargs) diff --git a/nlmod/gwf/lake.py b/nlmod/gwf/lake.py index c0258819..88fbc4d2 100644 --- a/nlmod/gwf/lake.py +++ b/nlmod/gwf/lake.py @@ -24,6 +24,15 @@ "ROUGH", ] +LAKE_GWT_KWDS = [ + "GWT_STATUS", + "GWT_CONCENTRATION", + "GWT_RAINFALL", + "GWT_EVAPORATION", + "GWT_RUNOFF", + "GWT_EXT-INFLOW", +] + # order of dictionary matters! OUTLET_DEFAULT = { "couttype": "WEIR", @@ -62,16 +71,21 @@ def lake_from_gdf( lakeno : with the number of the lake strt : with the starting head of the lake clake : with the bed resistance of the lake + A single lake may have multiple elements (polygon pieces) in one cell, e.g. + straight from nlmod.grid.gdf_to_grid. Pieces are combined into one lake-GWF + connection per cell with an area-weighted bed resistance (see + _aggregate_connections_per_cell), like nlmod.gwf.surface_water.aggregate + already does for RIV/DRN celldata. optional columns are 'STATUS', 'STAGE', 'RAINFALL', 'EVAPORATION', 'RUNOFF', 'INFLOW', 'WITHDRAWAL', 'AUXILIARY', 'RATE', 'INVERT', 'WIDTH', 'SLOPE', 'ROUGH'. These columns should contain the name of a dataarray in ds with the dimension time. - if the lake has any outlets they should be specified in the column - lakeout : the lake number of the outlet, if this is -1 the water - is removed from the model. - optinal columns are 'couttype', 'outlet_invert', 'outlet_width', - 'outlet_rough' and 'outlet_slope'. These columns should contain a - unique value for each outlet. + If the lake has any outlets they should be specified in the column 'lakeout'. + This is the lake number or boundname of the lake to which the outlet flows. If + lakeout is -1 the water is removed from the model. + Optional columns are 'couttype', 'outlet_invert', 'outlet_width', + 'outlet_rough' and 'outlet_slope'. These columns should contain a + unique value for each outlet. ds : xr.Dataset dataset containing relevant model grid and time information rainfall : int, float, str, np.array or pd.DataFrame, optional @@ -146,6 +160,12 @@ def lake_from_gdf( if gwt is not None: packagedata_gwt = [] perioddata_gwt = {0: []} + for iper in range(ds.sizes["time"]): + perioddata_gwt[iper] = [] + + lake_gwt_settings = [ + setting for setting in LAKE_GWT_KWDS if setting in gdf.columns + ] lake_settings = [setting for setting in LAKE_KWDS if setting in gdf.columns] @@ -158,6 +178,8 @@ def lake_from_gdf( gdf = add_lakeno_to_gdf(gdf, boundname_column) for lakeno, lake_gdf in gdf.groupby("lakeno"): + if lake_gdf.index.duplicated().any(): + lake_gdf = _aggregate_connections_per_cell(lake_gdf, ds) nlakeconn = lake_gdf.shape[0] if "strt" in lake_gdf: strt = _get_and_check_single_value(lake_gdf, "strt") @@ -207,6 +229,13 @@ def lake_from_gdf( and not lake_gdf["lakeout"].eq("").all() ): lakeout = _get_and_check_single_value(lake_gdf, "lakeout") + if isinstance(lakeout, str): + try: + # an integer lakeout could have been converted to a string + # after saving gdf to disk + lakeout = int(lakeout) + except ValueError: + pass if isinstance(lakeout, str): # when lakeout is a string, it represents the boundname # we need to find the lakeno that belongs to this boundname @@ -219,7 +248,8 @@ def lake_from_gdf( lakeout = gdf.loc[mask, "lakeno"].iloc[0] if not (gdf.loc[mask, "lakeno"] == lakeout).all(): raise ValueError( - f"expected single value of lakeno for lakeout {boundnameout}, got {gdf.loc[mask, 'lakeno']}" + f"expected single value of lakeno for lakeout {boundnameout}, " + f"got {gdf.loc[mask, 'lakeno']}" ) assert lakeno != lakeout, "lakein and lakeout cannot be the same" @@ -227,7 +257,8 @@ def lake_from_gdf( for outset, default_value in OUTLET_DEFAULT.items(): if outset not in lake_gdf.columns: logger.debug( - f"no value specified for {outset} and lake no {lakeno}, using default value {default_value}" + f"no value specified for {outset} and lake no {lakeno}, " + f"using default value {default_value}" ) setval = default_value else: @@ -235,12 +266,14 @@ def lake_from_gdf( if pd.notna(setval): if not (lake_gdf[outset] == setval).all(): raise ValueError( - f"expected single data variable for {outset} and lake number {lakeno}, got {lake_gdf[outset]}" + f"expected single data variable for {outset} and lake " + f"number {lakeno}, got {lake_gdf[outset]}" ) else: # setval is nan or None setval = default_value logger.debug( - f"no value specified for {outset} and lake no {lakeno}, using default value {default_value}" + f"no value specified for {outset} and lake no {lakeno}, " + f"using default value {default_value}" ) if outset == "outlet_invert" and isinstance(setval, str): # setval can be the name of a timeseries @@ -271,9 +304,18 @@ def lake_from_gdf( if pd.isna(datavar) or datavar == "": # None or nan or "" logger.debug(f"no {lake_setting} given for lake no {lakeno}") continue - perioddata[iper].append( - [lakeno, lake_setting, ds[datavar].values[iper]] - ) + if datavar in ds: + da = ds[datavar] + if len(da.dims) > 1: + if boundname_column is not None: + da = da.loc[:, boundname] + else: + da = da.loc[:, lakeno] + value = da.values[iper] + else: + value = datavar + + perioddata[iper].append([lakeno, lake_setting, value]) if gwt is not None: if "strt_concentration" in lake_gdf.columns: strt = _get_and_check_single_value(lake_gdf, "strt_concentration") @@ -286,14 +328,43 @@ def lake_from_gdf( packagedata_gwt.append([lakeno, strt, boundname]) else: packagedata_gwt.append([lakeno, strt]) - if rainfall is not None: - perioddata_gwt[0].append([lakeno, "rainfall", rainfall_concentration]) - if evaporation is not None: - perioddata_gwt[0].append( - [lakeno, "evaporation", evaporation_concentration] - ) - - if boundname_column is not None: + for iper in range(ds.sizes["time"]): + if rainfall is not None and ( + "GWT_RAINFALL" not in lake_gdf.columns + or lake_gdf["GWT_RAINFALL"].isna().all() + or lake_gdf["GWT_RAINFALL"].eq("").all() + ): + perioddata_gwt[iper].append( + [lakeno, "rainfall", rainfall_concentration] + ) + if evaporation is not None and ( + "GWT_EVAPORATION" not in lake_gdf.columns + or lake_gdf["GWT_EVAPORATION"].isna().all() + or lake_gdf["GWT_EVAPORATION"].eq("").all() + ): + perioddata_gwt[iper].append( + [lakeno, "evaporation", evaporation_concentration] + ) + for lake_setting in lake_gwt_settings: + datavar = _get_and_check_single_value(lake_gdf, lake_setting) + if pd.isna(datavar) or datavar == "": # None or nan or "" + logger.debug(f"no {lake_setting} given for lake no {lakeno}") + continue + if datavar in ds: + da = ds[datavar] + if len(da.dims) > 1: + if boundname_column is not None: + da = da.loc[:, boundname] + else: + da = da.loc[:, lakeno] + value = da.values[iper] + else: + value = datavar + perioddata_gwt[iper].append([lakeno, lake_setting[4:], value]) + + if "observations" in kwargs: + observations = kwargs.pop("observations") + elif boundname_column is not None: observations = {} if isinstance(obs_type, str): obs_type = [obs_type] @@ -349,10 +420,39 @@ def lake_from_gdf( return lak +def _aggregate_connections_per_cell(lake_gdf, ds): + """Combine multiple pieces of one lake within a grid cell into one connection. + + A lake polygon commonly intersects a grid cell in more than one piece (e.g. + after nlmod.grid.gdf_to_grid). MODFLOW 6 applies each VERTICAL lake-GWF + connection over the full cell area, so per-piece rows would multiply-count the + exchange. Each piece therefore contributes conductance for its own area: the + combined bed resistance is clake = cell_area / sum(piece_area / clake), so + that the connection conductance cell_area / clake equals the summed piece + conductance — the same area-weighted aggregation nlmod.gwf.surface_water + .aggregate applies to RIV/DRN celldata. All other columns take the value of + the first piece per cell. + """ + if "geometry" not in lake_gdf.columns or lake_gdf.geometry.isna().any(): + raise ValueError( + "found multiple elements of one lake in a single cell; provide polygon " + "geometries of the lake to aggregate these elements" + ) + cond = (lake_gdf.geometry.area / lake_gdf["clake"]).groupby(level=0).sum() + agg = lake_gdf.groupby(level=0).first() + agg["clake"] = ds["area"].data[agg.index] / cond + return agg + + def _get_and_check_single_value(lake_gdf, column): value = lake_gdf[column].iloc[0] if lake_gdf[column].isna().all() or lake_gdf[column].eq("").all(): return value + if pd.api.types.is_numeric_dtype(lake_gdf[column]): + # aggregated inputs (e.g. area-weighted stages) carry float-level noise + if not np.allclose(lake_gdf[column], value, rtol=1e-8, atol=0.0): + raise AssertionError(f"A single lake should have a single {column}") + return value if not (lake_gdf[column] == value).all(): raise (AssertionError(f"A single lake should have a single {column}")) return value @@ -362,10 +462,12 @@ def _parse_laksetting_value(value, ds, key, iper): if isinstance(value, (float, int, str)): return value elif isinstance(value, pd.Series): - assert len(value.index) == len(ds.time) and (value.index == ds.time).all() + assert len(value.index) == len(ds.time) + assert (value.index == ds.time).all() return value.iloc[iper] elif isinstance(value, pd.DataFrame): - assert len(value.index) == len(ds.time) and (value.index == ds.time).all() + assert len(value.index) == len(ds.time) + assert (value.index == ds.time).all() return value[key].iloc[iper] else: assert len(value) == len(ds.time) @@ -373,6 +475,20 @@ def _parse_laksetting_value(value, ds, key, iper): def add_lakeno_to_gdf(gdf, boundname_column): + """Add a lakeno column to a GeoDataFrame based on unique boundnames. + + Parameters + ---------- + gdf : gpd.GeoDataFrame + GeoDataFrame to add lakeno column to. + boundname_column : str + Name of the column containing boundnames. + + Returns + ------- + gdf : gpd.GeoDataFrame + GeoDataFrame with added lakeno column. + """ if boundname_column not in gdf.columns: raise (KeyError(f"Cannot find column {boundname_column} in gdf")) names = gdf[boundname_column].unique() @@ -414,8 +530,15 @@ def _copy_da_from_ds(gdf, ds, variable, boundname_column=None, set_to_0_in_ds=Fa def copy_meteorological_data_from_ds( gdf, ds, boundname_column=None, set_to_0_in_ds=False ): - """ - Copy meteorlogical data from the model dataset, and return rainfall and evaporation. + """Copy meteorological data from the model dataset. + + Returns rainfall and evaporation DataFrames. + + This method retrieves the values of rainfall and evaporation from a model Dataset. + It uses the 'recharge'variable, and optionally the 'evaporation'-variable, and + returns a rainfall- and evaporation-DataFrame. These dataframes contain input for + each of the lakes. The columns of this DataFrame are either the boundnames (when + boundname_column is specified) or the lake-number (lakeno). This method retrieves the values of rainfall and evaporation from a model Dataset. It uses the 'recharge'variable, and optionally the 'evaporation'-variable, and returns a rainfall- and evaporation-DataFrame. These dataframes contain input for @@ -479,8 +602,10 @@ def clip_meteorological_data_from_ds( ds, boundname_column=None, ): - """ - Clip meteorlogical data from the model dataset, and return rainfall and evaporation. + """Clip meteorological data from the model dataset. + + Returns rainfall and evaporation DataFrames. + This method retrieves the values of rainfall and evaporation from a model Dataset. It uses the 'recharge'variable, and optionally the 'evaporation'-variable, and returns a rainfall- and evaporation-DataFrame. These dataframes contain input for diff --git a/nlmod/gwf/output.py b/nlmod/gwf/output.py index 9c30eba1..1817f041 100644 --- a/nlmod/gwf/output.py +++ b/nlmod/gwf/output.py @@ -137,6 +137,7 @@ def get_budget_da( fname=None, grb_file=None, column="q", + pname=None, delayed=False, chunked=False, precision="auto", @@ -161,6 +162,10 @@ def get_budget_da( column : str name of column in rec-array to read, default is 'q' which contains the fluxes for most budget datasets. + pname : str, optional + mf6 package name to get data for, useful when multiple packages of + same type exist in the model. If None, then all packages of the same type will + be read and added together. delayed : bool, optional if delayed is True, do not load output data into memory, default is False. chunked : bool, optional @@ -178,7 +183,7 @@ def get_budget_da( cbcobj = get_cellbudgetfile( ds=ds, gwf=gwf, fname=fname, grb_file=grb_file, precision=precision ) - da = _get_budget_da(cbcobj, text, column=column, **kwargs) + da = _get_budget_da(cbcobj, text, column=column, pname=pname, **kwargs) da.attrs["units"] = "m3/d" # set time index if ds/gwt are provided @@ -199,8 +204,9 @@ def get_budget_da( def get_gwl_from_wet_cells(head, layer="layer", botm=None): - """Get the groundwater level from a multi-dimensional head array where dry cells are - NaN. This methods finds the most upper non-nan-value of each cell or timestep. + """Get groundwater level from multi-dimensional head array with dry cells as NaN. + + This method finds the most upper non-nan-value of each cell or timestep. Parameters ---------- @@ -260,8 +266,8 @@ def get_flow_residuals(ds, gwf=None, fname=None, grb_file=None, kstpkper=None): The location of the grb-file. grb_file is determied from ds when None. The default is None. kstpkper : tuple of 2 ints, optional - The index of the timestep and the stress period to include in the result. Include - all data in the budget-file when None. The default is None. + The index of the timestep and the stress period to include in the result. + Include all data in the budget-file when None. The default is None. Returns ------- @@ -298,7 +304,13 @@ def get_flow_residuals(ds, gwf=None, fname=None, grb_file=None, kstpkper=None): def get_flow_lower_face( - ds, gwf=None, fname=None, grb_file=None, kstpkper=None, lays=None + ds, + gwf=None, + fname=None, + grb_file=None, + kstpkper=None, + lays=None, + precision="auto", ): """Get the flow over the lower face of all model cells. @@ -318,11 +330,14 @@ def get_flow_lower_face( The location of the grb-file. grb_file is determied from ds when None. The default is None. kstpkper : tuple of 2 ints, optional - The index of the timestep and the stress period to include in the result. Include - all data in the budget-file when None. The default is None. + The index of the timestep and the stress period to include in the result. + Include all data in the budget-file when None. The default is None. lays : int or list of ints, optional The layers to include in the result. When lays is None, all layers are included. The default is None. + precision : str, optional + precision of floating point data in the budget-file. Accepted values are 'auto', + 'single' or 'double'. Returns ------- @@ -331,7 +346,9 @@ def get_flow_lower_face( """ if grb_file is None: grb_file = _get_grb_file(ds) - cbf = get_cellbudgetfile(ds=ds, gwf=gwf, fname=fname, grb_file=grb_file) + cbf = get_cellbudgetfile( + ds=ds, gwf=gwf, fname=fname, grb_file=grb_file, precision=precision + ) flowja = cbf.get_data(text="FLOW-JA-FACE", kstpkper=kstpkper) if ds.gridtype == "vertex": @@ -497,7 +514,8 @@ def highest3_mean(da: xr.DataArray): gxg_years = gxg_data.count("bimonth") == 24 gxg_data = gxg_data.where(gxg_years) - # First compute LG3 and HG3 per hydrological year, then compute the mean over the total. + # First compute LG3 and HG3 per hydrological year, then compute the mean + # over the total. if gxg_data.chunks is not None: # If data is lazily loaded/chunked, process data of one year at a time. gxg_data = gxg_data.chunk({"hydroyear": 1}) @@ -529,7 +547,7 @@ def highest3_mean(da: xr.DataArray): def calculate_gxg( head: xr.DataArray, below_surfacelevel: bool = False, - tolerance: pd.Timedelta = pd.Timedelta(days=7), + tolerance: pd.Timedelta = None, ) -> xr.DataArray: """Calculate GxG groundwater characteristics from head time series. @@ -556,7 +574,7 @@ def calculate_gxg( below_surfacelevel : boolean, optional, default: False. False (default) if heads are relative to a datum (e.g. sea level). If True, heads are taken as m below surface level. - tolerance: pd.Timedelta, default: 7 days. + tolerance: pd.Timedelta, optional, default: 7 days. Maximum time window allowed when searching for dates around the 14th and 28th of every month. @@ -576,6 +594,8 @@ def calculate_gxg( >>> head = nlmod.gwf.get_heads_da(ds) >>> gxg = nlmod.gwf.output.calculate_gxg(head) """ + if tolerance is None: + tolerance = pd.Timedelta(days=7) # if not head.dims == ("time", "y", "x"): # raise ValueError('Dimensions must be ("time", "y", "x")') if not np.issubdtype(head["time"].dtype, np.datetime64): diff --git a/nlmod/gwf/recharge.py b/nlmod/gwf/recharge.py index c181280a..67843c49 100644 --- a/nlmod/gwf/recharge.py +++ b/nlmod/gwf/recharge.py @@ -67,7 +67,7 @@ def ds_to_rch( gwf, filename=f"{gwf.name}.rch", pname=pname, - fixed_cell=False, + fixed_cell=kwargs.pop("fixed_cell", False), auxiliary="CONCENTRATION" if auxiliary is not None else None, maxbound=len(spd), stress_period_data={0: spd}, @@ -174,7 +174,7 @@ def ds_to_evt( gwf, filename=f"{gwf.name}.evt", pname=pname, - fixed_cell=False, + fixed_cell=kwargs.pop("fixed_cell", False), auxiliary="CONCENTRATION" if auxiliary is not None else None, maxbound=len(spd), nseg=nseg, @@ -378,7 +378,8 @@ def ds_to_uzf( # then use bfill to accont for inactive cells in the layer below, and set nans to -1 ivertcon = ivertcon.where(ivertcon >= 0).bfill("layer").fillna(-1).astype(int) - # packagedata : [iuzno, cellid, landflag, ivertcon, surfdep, vks, thtr, thts, thti, eps, boundname] + # packagedata : [iuzno, cellid, landflag, ivertcon, surfdep, vks, + # thtr, thts, thti, eps, boundname] packagedata = cols_to_reclist( ds, cellids, @@ -425,7 +426,8 @@ def ds_to_uzf( extwc = thtr if simulate_et and unsat_etwc: logger.info( - f"Setting evapotranspiration extinction water content (extwc) to {extwc}" + f"Setting evapotranspiration extinction water content (extwc) " + f"to {extwc}" ) if ha is None: ha = 0.0 @@ -470,7 +472,7 @@ def ds_to_uzf( iuzno_obs_vals = np.unique(iuzno_obs[~np.isnan(iuzno_obs)]).astype(int) # get cell ids of observations - cellids_obs = list(zip(*np.where(mask_obs))) + cellids_obs = list(zip(*np.where(mask_obs), strict=False)) cellid_str = ["_".join(map(str, x)) for x in cellids_obs] # account for surfdep, as this decreases the height of the top of the upper cell @@ -617,7 +619,9 @@ def _get_meteo_da_from_input(recharge, ds, pname, stn_var): use_ts = True ts_name = f"{pname}_0" - rch_unique_df = pd.DataFrame(recharge, columns=[ts_name]) + rch_unique_df = recharge.to_dataframe().rename( + columns={recharge.name: ts_name} + ) dims = ds["top"].dims coords = ds["top"].coords shape = [ds.sizes[dim] for dim in dims] @@ -658,7 +662,7 @@ def get_ts_name(stn): mask_recharge = fal != fal.attrs["nodata"] use_ts = False else: - raise NotImplementedError("Type {type(recharge)} not supported for recharge") + raise NotImplementedError(f"Type {type(recharge)} not supported for recharge") return recharge, mask_recharge, rch_unique_df diff --git a/nlmod/gwf/surface_water.py b/nlmod/gwf/surface_water.py index a15c74ca..18d357af 100644 --- a/nlmod/gwf/surface_water.py +++ b/nlmod/gwf/surface_water.py @@ -16,7 +16,7 @@ ) from ..dims.layers import get_idomain from ..read import bgt, waterboard -from ..util import extent_to_polygon, gdf_intersection_join, zonal_statistics, tqdm +from ..util import extent_to_polygon, gdf_intersection_join, tqdm, zonal_statistics logger = logging.getLogger(__name__) @@ -65,6 +65,31 @@ def aggregate(gdf, method, ds=None): def get_surfacewater_params(group, method, cid=None, ds=None, delange_params=None): + """Get surface water parameters based on the specified method. + + Parameters + ---------- + group : pandas.GroupBy + GroupBy object containing surface water data. + method : str + Method to use for parameter calculation. Options are 'area_weighted', + 'max_area', or 'de_lange'. + cid : tuple, optional + Cell id (row, col) for de_lange method. Default is None. + ds : xarray.Dataset, optional + Model dataset for de_lange method. Default is None. + delange_params : dict, optional + Additional parameters for de_lange method. Default is None. + + Returns + ------- + stage : float + Stage value. + cond : float + Conductance value. + rbot : float + River bottom value. + """ if method == "area_weighted": # stage stage = agg_area_weighted(group, "stage") @@ -111,16 +136,68 @@ def get_surfacewater_params(group, method, cid=None, ds=None, delange_params=Non def agg_max_area(gdf, col): + """Aggregate by taking the value from the row with maximum area. + + Parameters + ---------- + gdf : gpd.GeoDataFrame + GeoDataFrame to aggregate. + col : str + Column name to get value from. + + Returns + ------- + value + The value from the row with maximum area. + """ return gdf.loc[gdf.area.idxmax(), col] def agg_area_weighted(gdf, col): + """Aggregate by calculating area-weighted average. + + Parameters + ---------- + gdf : gpd.GeoDataFrame + GeoDataFrame to aggregate. + col : str + Column name to calculate weighted average from. + + Returns + ------- + float + The area-weighted average value. + """ nanmask = gdf[col].isna() aw = (gdf.area * gdf[col]).sum(skipna=True) / gdf.loc[~nanmask].area.sum() return aw def agg_de_lange(group, cid, ds, c1=0.0, c0=1.0, N=1e-3, crad_positive=True): + """Calculate conductance using the de Lange method. + + Parameters + ---------- + group : pandas.GroupBy + GroupBy object containing surface water data. + cid : tuple + Cell id (row, col). + ds : xarray.Dataset + Model dataset. + c1 : float, optional + Parameter for de Lange method. Default is 0.0. + c0 : float, optional + Parameter for de Lange method. Default is 1.0. + N : float, optional + Parameter for de Lange method. Default is 1e-3. + crad_positive : bool, optional + Whether to use positive crest radius. Default is True. + + Returns + ------- + tuple + Tuple containing (A, laytop, laybot, kh, kv, thickness, cond). + """ (A, laytop, laybot, kh, kv) = get_subsurface_params_by_cellid(ds, cid) rbot = group["botm"].min() @@ -174,6 +251,28 @@ def agg_de_lange(group, cid, ds, c1=0.0, c0=1.0, N=1e-3, crad_positive=True): def get_subsurface_params_by_cellid(ds, cid): + """Get subsurface parameters for a specific cell. + + Parameters + ---------- + ds : xarray.Dataset + Model dataset. + cid : tuple + Cell id (row, col). + + Returns + ------- + A : float + Area of the cell. + laytop : float + Top elevation of the cell. + laybot : array + Bottom elevations of each layer. + kh : array + Horizontal conductivity of each layer. + kv : array + Vertical conductivity of each layer. + """ r, c = cid A = ds.area.isel(x=c, y=r).data laytop = ds["top"].isel(x=c, y=r).data @@ -267,6 +366,26 @@ def de_lange_eqns(A, H0, kv, kh, c1, li, Bin, c0, p, N, crad_positive=True): def radial_resistance(L, B, H, kh, kv): + """Calculate radial resistance. + + Parameters + ---------- + L : float + Length. + B : float + Width. + H : float + Height. + kh : float + Horizontal conductivity. + kv : float + Vertical conductivity. + + Returns + ------- + float + Radial resistance value. + """ return ( L / (np.pi * np.sqrt(kh * kv)) @@ -275,10 +394,34 @@ def radial_resistance(L, B, H, kh, kv): def coth(x): + """Calculate hyperbolic cotangent. + + Parameters + ---------- + x : float + Input value. + + Returns + ------- + float + Hyperbolic cotangent of x. + """ return 1.0 / np.tanh(x) def estimate_polygon_length(gdf): + """Estimate polygon length based on shape factor. + + Parameters + ---------- + gdf : gpd.GeoDataFrame + GeoDataFrame with polygon geometries. + + Returns + ------- + gpd.GeoDataFrame + GeoDataFrame with estimated length column. + """ # estimate length from polygon (for shapefactor > 4) shape_factor = gdf.length / np.sqrt(gdf.area) @@ -305,9 +448,10 @@ def estimate_polygon_length(gdf): def distribute_cond_over_lays( cond, cellid, rivbot, laytop, laybot, idomain=None, kh=None, stage=None ): - """Distribute the conductance in a cell over the layers in that cell, based on the - the river-bottom and the layer bottoms, and optionally based on the stage and the - hydraulic conductivity. + """Distribute the conductance in a cell over the layers in that cell. + + Based on the river-bottom and the layer bottoms, and optionally based on the + stage and the hydraulic conductivity. """ if isinstance(rivbot, (np.ndarray, xr.DataArray)): rivbot = float(rivbot[cellid]) @@ -351,7 +495,9 @@ def distribute_cond_over_lays( try: first_active = np.where(idomain > 0)[0][0] except IndexError: - warnings.warn(f"No active layers in {cellid}, returning NaNs.") + warnings.warn( + f"No active layers in {cellid}, returning NaNs.", stacklevel=2 + ) return np.nan, np.nan else: first_active = 0 @@ -457,13 +603,13 @@ def build_spd( if np.isnan(cond): raise ValueError( f"Conductance is NaN in cell {cellid}. Info: area={row.area:.2f} " - f"len={row.len_estimate:.2f}, BL={row['rbot']}" + f", BL={row['rbot']}" ) if cond < 0: raise ValueError( f"Conductance is negative in cell {cellid}. Info: area={row.area:.2f} " - f"len={row.len_estimate:.2f}, BL={row['rbot']}" + f", BL={row['rbot']}" ) if layer_method == "distribute_cond_over_lays": @@ -502,7 +648,7 @@ def build_spd( cellid = (cellid,) # write SPD - for lay, cond in zip(lays, conds): + for lay, cond in zip(lays, conds, strict=False): cid = (lay,) + cellid if pkg == "RIV": spd.append([cid, stage, cond, rbot] + auxlist) @@ -513,6 +659,7 @@ def build_spd( def add_info_to_gdf(*args, **kwargs): + """Deprecated function, use nlmod.util.gdf_intersection_join instead.""" logger.warning( "nlmod.gwf.surface_water.add_info_to_gdf is deprecated. " "Use nlmod.util.gdf_intersection_join instead." @@ -605,7 +752,8 @@ def download_level_areas( mask = ~la[wb].is_valid if mask.any(): logger.warning( - f"{mask.sum()} geometries of level areas of {wb} are invalid. Thet are made valid by adding a buffer of 0.0." + f"{mask.sum()} geometries of level areas of {wb} are invalid. " + "They are made valid by adding a buffer of 0.0." ) # first copy to prevent ValueError: assignment destination is read-only la[wb] = la[wb].copy() @@ -912,12 +1060,13 @@ def gdf_to_seasonal_pkg( season_filename="season.ts", **kwargs, ): - """Add a surface water package to a groundwater-model, based on input from a - GeoDataFrame. This method adds two boundary conditions for each record in the - GeoDataFrame: one for the winter_stage and one for the summer_stage. - The conductance of each record is a time-series called 'winter' or 'summer' with - values of either 0 or 1. These conductance values are multiplied by an auxiliary - variable that contains the actual conductance. + """Add a surface water package to a groundwater-model based on GeoDataFrame input. + + This method adds two boundary conditions for each record in the GeoDataFrame: + one for the winter_stage and one for the summer_stage. The conductance of each + record is a time-series called 'winter' or 'summer' with values of either 0 or 1. + These conductance values are multiplied by an auxiliary variable that contains + the actual conductance. Parameters ---------- @@ -989,7 +1138,8 @@ def gdf_to_seasonal_pkg( mask = gdf["rbot"].isna() if mask.any(): logger.info( - f"Filling {mask.sum()} NaN's in rbot using a water depth of {default_water_depth} meter." + f"Filling {mask.sum()} NaN's in rbot using a water depth of " + f"{default_water_depth} meter." ) min_stage = pd.concat(stages, axis=1).min(axis=1) gdf.loc[mask, "rbot"] = min_stage[mask] - default_water_depth @@ -1122,7 +1272,9 @@ def add_season_timeseries( else: ts_data = [(0.0, 1.0, 0.0)] tmax = pd.to_datetime(ds["time"].data[-1]) - years = range(tmin.year, tmax.year + 1) + # add extra year to make sure we have a record for the start of the + # last season in the model + years = range(tmin.year, tmax.year + 2) for year in years: # add a record for the start of summer time = pd.Timestamp(f"{year}-{start_summer}") @@ -1227,12 +1379,36 @@ def get_seaonal_timeseries( def rivdata_from_xylist(gwf, xylist, layer, stage, cond, rbot, aux=None): + """Create river data from a list of xy coordinates. + + Parameters + ---------- + gwf : flopy.mf6.mfmodel.MFModel + Groundwater flow model. + xylist : list + List of (x, y) coordinates. + layer : int + Layer number. + stage : float + Stage value. + cond : float + Conductance value. + rbot : float + River bottom value. + aux : dict, optional + Auxiliary data. Default is None. + + Returns + ------- + list + List of river data. + """ gi = flopy.utils.GridIntersect(gwf.modelgrid) df = gi.intersect(xylist, shapetype="linestring", geo_dataframe=True) riv_data = [] if "row" in df.columns: # structured grid - for row, col in zip(df["row"], df["col"]): + for row, col in zip(df["row"], df["col"], strict=False): idata = [(layer, row, col), stage, cond, rbot] if aux is not None: idata.append(aux) diff --git a/nlmod/gwt/gwt.py b/nlmod/gwt/gwt.py index e633b807..7cc9ea23 100644 --- a/nlmod/gwt/gwt.py +++ b/nlmod/gwt/gwt.py @@ -279,8 +279,16 @@ def oc( conc_filerecord = [concfile] budgetfile = f"{gwt.name}.cbc" budget_filerecord = [budgetfile] - saverecord = _set_record(save_concentration, save_budget, output="concentration") - printrecord = _set_record(print_concentration, print_budget, output="concentration") + saverecord = kwargs.pop("saverecord", None) + if saverecord is None: + saverecord = _set_record( + save_concentration, save_budget, output="concentration" + ) + printrecord = kwargs.pop("printrecord", None) + if printrecord is None: + printrecord = _set_record( + print_concentration, print_budget, output="concentration" + ) oc = flopy.mf6.ModflowGwtoc( gwt, diff --git a/nlmod/gwt/output.py b/nlmod/gwt/output.py index 69b79964..5315640f 100644 --- a/nlmod/gwt/output.py +++ b/nlmod/gwt/output.py @@ -91,7 +91,8 @@ def get_concentration_da( def get_concentration_at_gw_surface(conc, layer="layer"): - """Get the concentration level from a multi-dimensional concentration array where + """Get the concentration level from a multi-dimensional concentration array. + dry or inactive cells are NaN. This methods finds the most upper non- nan-value of each cell or timestep. @@ -137,8 +138,9 @@ def get_concentration_at_gw_surface(conc, layer="layer"): def freshwater_head(ds, hp, conc, denseref=None, drhodc=None): - """Calculate equivalent freshwater head from point water heads. Heads file produced - by mf6 contains point water heads. + """Calculate equivalent freshwater head from point water heads. + + Heads file produced by mf6 contains point water heads. Parameters ---------- @@ -180,8 +182,9 @@ def freshwater_head(ds, hp, conc, denseref=None, drhodc=None): def pointwater_head(ds, hf, conc, denseref=None, drhodc=None): - """Calculate point water head from freshwater heads. Heads file produced by mf6 - contains point water heads. + """Calculate point water head from freshwater heads. + + Heads file produced by mf6 contains point water heads. Parameters ---------- diff --git a/nlmod/gwt/prepare.py b/nlmod/gwt/prepare.py index 32ff89f0..f23c98d8 100644 --- a/nlmod/gwt/prepare.py +++ b/nlmod/gwt/prepare.py @@ -56,5 +56,5 @@ def set_default_transport_parameters(ds, transport_type): else: raise ValueError("Only 'chloride' and 'tracer' transport types are defined.") - + ds.attrs["transport"] = 1 # set transport flag to 1 return ds diff --git a/nlmod/mfoutput/binaryfile.py b/nlmod/mfoutput/binaryfile.py index a08dea2b..e6a05495 100644 --- a/nlmod/mfoutput/binaryfile.py +++ b/nlmod/mfoutput/binaryfile.py @@ -55,14 +55,14 @@ def __create3D(data, fobj, column="q", node="node"): """ out = np.ma.zeros(fobj.nnodes, dtype=np.float32) out.mask = True - for n, q in zip(data[node], data[column]): + for n, q in zip(data[node], data[column], strict=False): idx = n - 1 out.data[idx] += q out.mask[idx] = False return np.ma.reshape(out, fobj.shape) -def _select_data_indices_budget(fobj, text, kstpkper): +def _select_data_indices_budget(fobj, text, kstpkper, paknam2=None): """Select data indices for budgetfile. Parameters @@ -73,6 +73,10 @@ def _select_data_indices_budget(fobj, text, kstpkper): text indicating which dataset to load kstpkper : tuple(int, int) step and period index + paknam2 : str + mf6 package name to get data for, useful when multiple packages of + same type exist in the model. If None, then all packages of the same type will + be Returns ------- @@ -84,20 +88,19 @@ def _select_data_indices_budget(fobj, text, kstpkper): if text is not None: text16 = fobj._find_text(text) + if paknam2 is not None: + paknam2 = fobj._find_paknam(paknam2, to=True) + # get kstpkper indices kstp1 = kstpkper[0] + 1 kper1 = kstpkper[1] + 1 - if text is None: - select_indices = np.where( - (fobj.recordarray["kstp"] == kstp1) & (fobj.recordarray["kper"] == kper1) - ) - else: - if text is not None: - select_indices = np.where( - (fobj.recordarray["kstp"] == kstp1) - & (fobj.recordarray["kper"] == kper1) - & (fobj.recordarray["text"] == text16) - ) + + mask = (fobj.recordarray["kstp"] == kstp1) & (fobj.recordarray["kper"] == kper1) + if text is not None: + mask = mask & (fobj.recordarray["text"] == text16) + if paknam2 is not None: + mask = mask & (fobj.recordarray["paknam2"] == paknam2) + select_indices = np.where(mask) # build and return the record list if isinstance(select_indices, tuple): @@ -105,7 +108,9 @@ def _select_data_indices_budget(fobj, text, kstpkper): return select_indices -def _get_binary_budget_data(kstpkper, fobj, text, column="q", node="node"): +def _get_binary_budget_data( + kstpkper, fobj, text, column="q", node="node", paknam2=None +): """Get budget data from binary CellBudgetFile. Code copied from flopy.utils.binaryfile.CellBudgetFile and adapted to @@ -124,6 +129,10 @@ def _get_binary_budget_data(kstpkper, fobj, text, column="q", node="node"): column : str name of column in rec-array to read, default is 'q' which contains the fluxes for most budget datasets. + paknam2 : str + mf6 package name to get data for, useful when multiple packages of + same type exist in the model. If None, then all packages of the same type will + be read and added together. Returns ------- @@ -131,7 +140,7 @@ def _get_binary_budget_data(kstpkper, fobj, text, column="q", node="node"): array containing data for a specific timestep """ # select indices to read - idx = _select_data_indices_budget(fobj, text, kstpkper) + idx = _select_data_indices_budget(fobj, text, kstpkper, paknam2=paknam2) # idx must be an ndarray, so if it comes in as an integer then convert if np.isscalar(idx): @@ -147,6 +156,7 @@ def _get_binary_budget_data(kstpkper, fobj, text, column="q", node="node"): for ipos in fobj.iposarray[idx]: data.append(_get_binary_budget_record(fobj, ipos, header, column, node)) + # check if multiple instances of the package exist, if so add them together if len(data) == 1: return data[0] else: diff --git a/nlmod/mfoutput/mfoutput.py b/nlmod/mfoutput/mfoutput.py index 3049d67a..e6a506de 100644 --- a/nlmod/mfoutput/mfoutput.py +++ b/nlmod/mfoutput/mfoutput.py @@ -203,6 +203,7 @@ def _get_budget_da( text, modelgrid=None, column="q", + pname=None, **kwargs, ): """Get budget data array based on CellBudgetFile and text string. @@ -222,6 +223,10 @@ def _get_budget_da( column : str name of column in rec-array to read, default is 'q' which contains the fluxes for most budget datasets. + pname : str, optional + mf6 package name to get data for, useful when multiple packages of + same type exist in the model. If None, then all packages of the same type will + be read and added together. Returns ------- @@ -245,6 +250,7 @@ def _get_budget_da( fobj=cbcobj, text=text, column=column, + paknam2=pname, **kwargs, ) @@ -296,7 +302,8 @@ def _get_flopy_data_object( if gwml is None: msg = f"Load the {var}s using either ds, {ml_name} or fname" raise (ValueError(msg)) - # return gwf.output.head(), gwf.output.budget() or gwt.output.concentration() + # return gwf.output.head(), gwf.output.budget() or + # gwt.output.concentration() return getattr(gwml.output, var)() fname = os.path.join(ds.model_ws, ds.model_name + extension) if grb_file is None and ds is not None: @@ -318,7 +325,7 @@ def _get_flopy_data_object( else: if modelgrid is None: logger.warning(msg) - warnings.warn(msg) + warnings.warn(msg, stacklevel=2) return flopy.utils.HeadFile(fname, text=var, modelgrid=modelgrid, **kwargs) diff --git a/nlmod/modpath/modpath.py b/nlmod/modpath/modpath.py index 6bfdc287..9659d563 100644 --- a/nlmod/modpath/modpath.py +++ b/nlmod/modpath/modpath.py @@ -22,8 +22,10 @@ def write_and_run(mpf, remove_prev_output=True, script_path=None, silent=False): - """Write modpath files and run the model. Extra options include removing previous - output and copying the modelscript to the model workspace. + """Write modpath files and run the model. + + Extra options include removing previous output and copying the modelscript + to the model workspace. Parameters ---------- @@ -131,6 +133,7 @@ def package_to_nodes(gwf, package_name, mpf=None, ibound=None): "The 'mpf' parameter is deprecated and will be removed in a future version." " Please pass 'ibound' directly.", DeprecationWarning, + stacklevel=2, ) ibound = mpf.ib gwf_package = gwf.get_package(package_name) @@ -290,8 +293,10 @@ def bas(mpf, porosity=0.3, **kwargs): def remove_output(mpf): - """Remove the output of a previous modpath run. Commonly used before starting a new - modpath run to avoid loading the wrong data when a modpath run has failed. + """Remove the output of a previous modpath run. + + Commonly used before starting a new modpath run to avoid loading the wrong + data when a modpath run has failed. Parameters ---------- @@ -354,7 +359,8 @@ def load_pathline_data( if mpf is None: if modelname is None: raise ValueError( - "if no mpf model is provided a modelname should be provided to load pathline data" + "if no mpf model is provided a modelname should be provided " + "to load pathline data" ) fpth = os.path.join(model_ws, f"{modelname}.mppth") else: @@ -395,7 +401,8 @@ def pg_from_fdt(nodes, divisions=3): Particle group. """ logger.info( - f"particle group with {divisions**2} particle per cell face, {6 * divisions**2} particles per cell" + f"particle group with {divisions**2} particle per cell face, " + f"{6 * divisions**2} particles per cell" ) sd = flopy.modpath.FaceDataType( drape=0, diff --git a/nlmod/observations.py b/nlmod/observations.py index 9e08e1e3..f30bab44 100644 --- a/nlmod/observations.py +++ b/nlmod/observations.py @@ -1,9 +1,18 @@ +import logging + +import flopy as fp import numpy as np +import pandas as pd import xarray as xr +from geopandas import GeoDataFrame, points_from_xy from scipy.spatial import Delaunay from nlmod.dims import grid +from .util import tqdm + +logger = logging.getLogger(__name__) + def _compute_interpolation_weights(xy, uv, d=2): """Calculate interpolation weights for barycentric interpolation [1]_. @@ -233,3 +242,144 @@ def interpolate_to_points( return interp_da # also returns vertices and weights else: return interp_da["interpolated"] + + +def build_obs( + ds, + gwf_or_gwt, + df, + obs_type="head", + fname=None, + x="x", + y="y", + z="z", + screen_top="screen_top", + screen_bottom="screen_bottom", +): + """Get observations from dataframe for a MODFLOW 6 model. + + Parameters + ---------- + ds : xr.Dataset + Dataset containing model grid and other model information + gwf_or_gwt : flopy.mf6.ModflowGwf or flopy.mf6.ModflowGwt + MODFLOW 6 groundwater flow or transport model object + df : pd.DataFrame or geopandas.GeoDataFrame + DataFrame containing observation locations and optionally the + layer of the observation. If a GeoDataFrame is provided, the + geometry column must contain Point geometries. + obs_type : str, optional + Type of observation, default is "head" + fname : str, optional + Name of the observation file, default is None, which will create + a file named "{obs_type}_obs.csv" + x : str, optional + Name of the column in df containing x-coordinates, default is "x" + y : str, optional + Name of the column in df containing y-coordinates, default is "y" + z : str, optional + Name of the column in df containing z-coordinates, default is "z". If + z is not provided, the mean of screen_top and screen_bottom will be used. + screen_top : str, optional + Name of the column in df containing the top of the screen, default is + "screen_top", only used if z is not provided + screen_bottom : str, optional + Name of the column in df containing the bottom of the screen, default is + "screen_bottom", only used if z is not provided + pname : str, optional + Name of the observation package, default is None, which will create a package + named "obs_{obs_type}" with a maximum length of 16 characters. If pname is + provided, it will be truncated to 16 characters if necessary. + **kwargs : additional keyword arguments + Additional keyword arguments to pass to the flopy.mf6.ModflowUtlobs constructor. + + Returns + ------- + obs : flopy.mf6.ModflowUtlobs + MODFLOW 6 observation package object + """ + # store observations at locations of measurements + if z not in df.columns: + df[z] = df.loc[:, [screen_top, screen_bottom]].mean(axis=1) + if isinstance(df, pd.DataFrame): + pts = points_from_xy(*df.loc[:, [x, y, z]].values.T) + elif isinstance(df, GeoDataFrame): + pts = df.geometry + else: + raise ValueError("df must be a pandas DataFrame or a GeoDataFrame") + ix = fp.utils.GridIntersect(gwf_or_gwt.modelgrid) + r = ix.points_to_cellids(np.asarray(pts), handle_z=True) + continuous = [] + for i, name in tqdm( + enumerate(df.index), + desc="Adding head observations", + total=df.index.size, + ): + if pd.isna(r["layer"].iloc[i]): + continue + if grid.is_structured(ds): + icellid = ( + int(r["layer"].iloc[i]), + int(r["row"].iloc[i]), + int(r["col"].iloc[i]), + ) + elif grid.is_vertex(ds): + icellid = ( + int(r["layer"].iloc[i]), + int(r["cellid"].iloc[i]), + ) + else: + icellid = None + if icellid is not None: + continuous.append((name, obs_type, icellid)) + + if fname is None: + fname = f"{obs_type}_obs.csv" + return {fname: continuous} + + +def obs( + gwf_or_gwt, + continuous: list | dict, + pname=None, + **kwargs, +): + """Add observation package to a MODFLOW 6 model. + + Parameters + ---------- + gwf_or_gwt : flopy.mf6.ModflowGwf or flopy.mf6.ModflowGwt + MODFLOW 6 groundwater flow or transport model object + continuous : list or dict + List of dictionaries containing observation information, e.g. + [{"file1": [(name, obs_type, icellid), ...], "file2": [...]}]. Also accepts + a single dictionary. + pname : str, optional + Name of the observation package, default is None, which will create a package + named "obs_{obs_type}" with a maximum length of 16 characters. If pname is + provided, it will be truncated to 16 characters if necessary. + **kwargs : additional keyword arguments + Additional keyword arguments to pass to the flopy.mf6.ModflowUtlobs constructor. + + Returns + ------- + obs : flopy.mf6.ModflowUtlobs + MODFLOW 6 observation package object + + See Also + -------- + nlmod.build.build_observations + Build observation package for a MODFLOW 6 model from a dataframe containing + observation locations. This function returns the `continuous` variable + that can be passed into this function. + """ + logger.info("creating mf6 OBS") + + return fp.mf6.ModflowUtlobs( + gwf_or_gwt, + digits=10, + print_input=True, + continuous=continuous, + pname=pname, + **kwargs, + ) diff --git a/nlmod/plot/__init__.py b/nlmod/plot/__init__.py index 2a0bfc7c..bc752833 100644 --- a/nlmod/plot/__init__.py +++ b/nlmod/plot/__init__.py @@ -7,11 +7,12 @@ facet_plot, geotop_lithok_in_cross_section, geotop_lithok_on_map, + geotop_strat_in_cross_section, + get_ahn_colormap, map_array, modelextent, modelgrid, surface_water, - get_ahn_colormap, ) from .plotutil import ( add_background_map, diff --git a/nlmod/plot/dcs.py b/nlmod/plot/dcs.py index 26de80ac..d5ba0fae 100644 --- a/nlmod/plot/dcs.py +++ b/nlmod/plot/dcs.py @@ -20,10 +20,13 @@ class DatasetCrossSection: - # assumes: - # x and y are 1d-vectors - # x is increasing, y is decreasing - # the layers are ordered from the top down + """Create a cross section from a dataset. + + Assumes: + - x and y are 1d-vectors + - x is increasing, y is decreasing + - the layers are ordered from the top down + """ def __init__( self, @@ -124,7 +127,7 @@ def __init__( self.ax.axis(extent) def get_grid_edges(self): - """Get the x and y values of all gridcell edges + """Get the x and y values of all gridcell edges. Returns ------- @@ -141,6 +144,18 @@ def get_grid_edges(self): return x, y def coordinates_in_dataset(self, xy): + """Check if coordinates are within the dataset extent. + + Parameters + ---------- + xy : tuple + x, y coordinates to check. + + Returns + ------- + bool + True if coordinates are within the dataset extent. + """ return ( xy[0] > self.xedge.min() and xy[1] > self.yedge.min() @@ -150,6 +165,17 @@ def coordinates_in_dataset(self, xy): @staticmethod def add_intersections(gr_line, cs_line, points): + """Add intersection points between grid line and cross section line. + + Parameters + ---------- + gr_line : LineString + Grid line. + cs_line : LineString + Cross section line. + points : list + List to which intersection points will be appended. + """ intersection = cs_line.intersection(gr_line) if intersection.geom_type == "Point": points.append(intersection) @@ -158,17 +184,29 @@ def add_intersections(gr_line, cs_line, points): points.append(point) def line_intersect_grid(self, cs_line): + """Find intersection points between cross section line and grid. + + Parameters + ---------- + cs_line : LineString + Cross section line. + + Returns + ------- + xys : np.ndarray + Array of x, y, distance-along-line for intersection points. + """ points = [] # add the starting point if self.coordinates_in_dataset(cs_line.coords[0]): points.append(Point(cs_line.coords[0])) # add all intersections with yedge for y in self.yedge: - gr_line = LineString(zip(self.xedge[[0, -1]], [y, y])) + gr_line = LineString(zip(self.xedge[[0, -1]], [y, y], strict=False)) self.add_intersections(gr_line, cs_line, points) # add all intersections with xedge for x in self.xedge: - gr_line = LineString(zip([x, x], self.yedge[[0, -1]])) + gr_line = LineString(zip([x, x], self.yedge[[0, -1]], strict=False)) self.add_intersections(gr_line, cs_line, points) # add the ending point if self.coordinates_in_dataset(cs_line.coords[-1]): @@ -190,8 +228,31 @@ def plot_layers( min_label_area=np.inf, fontsize=None, only_labels=False, + ha="center", **kwargs, ): + """Plot layers in the cross section. + + Parameters + ---------- + colors : list, str, pd.DataFrame, pd.Series, dict, optional + Colors for each layer. The default is None, which uses tab20 colormap. + min_label_area : float, optional + Minimum area for layer labels. The default is np.inf. + fontsize : float, optional + Font size for labels. The default is None. + only_labels : bool, optional + Only plot labels, not polygons. The default is False. + ha : str, optional + Horizontal alignment for labels. The default is "center". + **kwargs : dict + Additional keyword arguments passed to matplotlib PatchCollection. + + Returns + ------- + polygons : list + List of matplotlib Polygon objects. + """ if colors is None: cmap = plt.get_cmap("tab20") colors = [cmap(i) for i in range(len(self.layer))] @@ -217,7 +278,7 @@ def plot_layers( tots.append(z_not_nan[x] + 1) vans.append(z_not_nan[x + 1]) tots.append(z_not_nan[-1] + 1) - for van, tot in zip(vans, tots): + for van, tot in zip(vans, tots, strict=False): t = self.top[i, van:tot] b = self.bot[i, van:tot] n = tot - van @@ -230,7 +291,7 @@ def plot_layers( b[sorted(list(range(n)) * 2, reverse=True)], ) ) - xy = list(zip(x, y)) + xy = list(zip(x, y, strict=False)) # xy = np.vstack((x, y)).T color = colors[i] pol = matplotlib.patches.Polygon(xy, facecolor=color, **kwargs) @@ -258,7 +319,7 @@ def plot_layers( xt, yt, self.layer[i], - ha="center", + ha=ha, va="center", fontsize=fontsize, ) @@ -267,6 +328,19 @@ def plot_layers( return polygons def label_layers(self, min_label_area=None): + """Plot labels for layers in the cross section. + + Parameters + ---------- + min_label_area : float, optional + Minimum area for layer labels. The default is None, which uses + line.length * 1 (layers with average thickness of 1 meter). + + Returns + ------- + polygons : list + List of matplotlib Text objects. + """ if min_label_area is None: # plot labels of layers with an average thickness of 1 meter # in entire cross-section @@ -282,6 +356,28 @@ def plot_grid( ilayers=None, **kwargs, ): + """Plot grid lines in the cross section. + + Parameters + ---------- + edgecolor : str, optional + Edge color for grid lines. The default is "k". + facecolor : str, optional + Face color for grid cells. The default is "none". + horizontal : bool, optional + Plot horizontal grid lines. The default is True. + vertical : bool, optional + Plot vertical grid lines. The default is True. + ilayers : list, optional + List of layer indices to plot. The default is None (all layers). + **kwargs : dict + Additional keyword arguments passed to matplotlib Line2D. + + Returns + ------- + lines : list + List of matplotlib Line2D objects. + """ lines = [] if ilayers is None: ilayers = range(self.top.shape[0]) @@ -390,7 +486,10 @@ def plot_map_cs( return ax def iterate_active_cells(self, zcs): - """Iterate over the cell indices of the cells in an array that are visible in the cross section and active in the model. + """Iterate over the cell indices of the cells in an array. + + Iterate over the cells in an array that are visible in the cross section + and active in the model. Parameters ---------- @@ -472,6 +571,22 @@ def array_on_cs(self, z): return z[:, self.rows, self.cols] def plot_array(self, z, head=None, **kwargs): + """Plot an array on the cross section. + + Parameters + ---------- + z : np.ndarray or xr.DataArray + Array to plot with dimensions (layer, y, x) or (layer, cellid). + head : np.ndarray or xr.DataArray, optional + Head array for clipping. The default is None. + **kwargs : dict + Additional keyword arguments passed to matplotlib pcolormesh. + + Returns + ------- + QuadMesh + matplotlib QuadMesh object. + """ zcs = self.array_on_cs(z) if head is not None: @@ -490,7 +605,78 @@ def plot_array(self, z, head=None, **kwargs): self.ax.add_collection(patch_collection) return patch_collection + def hatch_layers(self, layers, color="k", hatch="/", **kwargs): + """Add hatch patterns to specified layers. + + Parameters + ---------- + layers : list + List of layer names to hatch. + color : str, optional + Hatch color. The default is "k". + hatch : str, optional + Hatch pattern. The default is "/". + **kwargs : dict + Additional keyword arguments passed to matplotlib Polygon. + + Returns + ------- + polygons : list + List of matplotlib Polygon objects. + """ + polygons = [] + fc = kwargs.pop("facecolor", "none") + ec = kwargs.pop("edgecolor", "none") + for i, lay in enumerate(self.ds.layer.values): + if lay in layers: + if np.all(np.isnan(self.bot[i]) | (self.bot[i] == self.zmax)): + continue + if np.all(np.isnan(self.top[i]) | (self.top[i] == self.zmin)): + continue + z_not_nan = np.where(~np.isnan(self.top[i]) & ~np.isnan(self.bot[i]))[0] + vans = [z_not_nan[0]] + tots = [] + for x in np.where(np.diff(z_not_nan) > 1)[0]: + tots.append(z_not_nan[x] + 1) + vans.append(z_not_nan[x + 1]) + tots.append(z_not_nan[-1] + 1) + for van, tot in zip(vans, tots, strict=False): + t = self.top[i, van:tot] + b = self.bot[i, van:tot] + n = tot - van + + x = self.s[van:tot].ravel() + x = np.concatenate((x, x[::-1])) + y = np.concatenate( + ( + t[sorted(list(range(n)) * 2)], + b[sorted(list(range(n)) * 2, reverse=True)], + ) + ) + xy = list(zip(x, y, strict=False)) + pol = matplotlib.patches.Polygon( + xy, facecolor=fc, edgecolor=ec, hatch=hatch, **kwargs + ) + pol._hatch_color = matplotlib.colors.to_rgba(color) + self.ax.add_patch(pol) + polygons.append(pol) + return polygons + def plot_surface(self, z, **kwargs): + """Plot a surface on the cross section. + + Parameters + ---------- + z : np.ndarray or xr.DataArray + Surface elevation array. + **kwargs : dict + Additional keyword arguments passed to matplotlib plot. + + Returns + ------- + Line2D + matplotlib Line2D object. + """ if isinstance(z, xr.DataArray): z = z.data # check if z has the same dimensions as ds @@ -527,7 +713,7 @@ def plot_wells( filter_rect_kwargs=None, tubeline_kwargs=None, ): - """plot filter screens in cross section from a DataFrame + """Plot filter screens in cross section from a DataFrame. Parameters ---------- @@ -588,7 +774,6 @@ def plot_wells( matplotlib.collections.PatchCollection """ - # check screen top and bot if (df[col_screen_top] < df[col_screen_bottom]).any(): logger.warning("screen top is lower than screen bottom") @@ -661,7 +846,7 @@ def plot_wells( # sort on distance along line if sort_by_dist: - df.sort_values("s", inplace=True) + df = df.sort_values("s") rectangles = [] legend_handles = {} @@ -718,6 +903,20 @@ def plot_wells( return patch_collection def get_top_and_bot(self, top, bot): + """Get top and bottom arrays for the cross section. + + Parameters + ---------- + top : str or np.ndarray + Top elevation data variable name or array. + bot : str or np.ndarray + Bottom elevation data variable name or array. + + Returns + ------- + tuple + (top, bot) arrays for the cross section. + """ # then determine the top and botm of each cell if isinstance(top, str): top = self.ds[top].data diff --git a/nlmod/plot/flopy.py b/nlmod/plot/flopy.py index 0efc2f90..61d05817 100644 --- a/nlmod/plot/flopy.py +++ b/nlmod/plot/flopy.py @@ -23,9 +23,7 @@ def _get_figure(ax=None, gwf=None, figsize=None): figsize = (figsize[0], np.round(figsize[1] / 0.02, 0) * 0.02) base = 10 ** int(np.log10(gwf.modelgrid.extent[1] - gwf.modelgrid.extent[0])) - f, ax = get_map( - gwf.modelgrid.extent, base=base, figsize=figsize, tight_layout=False - ) + f, ax = get_map(gwf.modelgrid.extent, base=base, figsize=figsize) ax.set_aspect("equal", adjustable="box") return f, ax @@ -149,8 +147,6 @@ def map_array( axprops = {"xlabel": xlabel, "ylabel": ylabel, "title": title} ax.set(**axprops) - f.tight_layout() - # colorbar divider = make_axes_locatable(ax) if colorbar: @@ -279,8 +275,6 @@ def contour_array( axprops = {"xlabel": xlabel, "ylabel": ylabel, "title": title} ax.set(**axprops) - f.tight_layout() - if animate: return f, ax, cs else: @@ -313,6 +307,64 @@ def animate_map( save=False, fname=None, ): + """Animate a map over time. + + Parameters + ---------- + arr : np.ndarray or xr.DataArray + Array to animate with shape (time, layer, y, x). + times : list + List of time values for each frame. + gwf : flopy ModflowGwf + Groundwater flow model object. + ilay : int, optional + Layer index to plot. The default is 0. + extent : tuple, optional + Extent for the plot. The default is None. + ax : matplotlib Axes, optional + Axes to plot on. The default is None. + title : str, optional + Plot title. The default is "". + xlabel : str, optional + X-axis label. The default is "X [km RD]". + ylabel : str, optional + Y-axis label. The default is "Y [km RD]". + datefmt : str, optional + Date format string. The default is "%Y-%m". + norm : matplotlib.colors.Normalize, optional + Color normalization. The default is None. + vmin : float, optional + Minimum value for color scale. The default is None. + vmax : float, optional + Maximum value for color scale. The default is None. + levels : list, optional + Contour levels. The default is None. + cmap : str, optional + Colormap name. The default is "viridis". + alpha : float, optional + Transparency. The default is 1.0. + colorbar : bool, optional + Show colorbar. The default is True. + colorbar_label : str, optional + Colorbar label. The default is "". + plot_grid : bool, optional + Plot grid. The default is True. + add_to_plot : callable, optional + Function to add additional elements to the plot. The default is None. + background : bool, optional + Use background. The default is False. + figsize : tuple, optional + Figure size. The default is (9.24, 10.042). + save : bool, optional + Save animation. The default is False. + fname : str, optional + Filename for saved animation. The default is None. + + Returns + ------- + FuncAnimation + matplotlib animation object. + """ # get data if isinstance(arr, xr.DataArray): arr = arr.values @@ -415,6 +467,52 @@ def facet_plot( plot_bc=None, plot_grid=False, ): + """Create a facet plot for model results. + + Parameters + ---------- + gwf : flopy ModflowGwf + Groundwater flow model object. + arr : np.ndarray + Array to plot with shape (time, layer, y, x) or (layer, y, x). + lbl : str, optional + Label for colorbar. The default is "". + plot_dim : str, optional + Dimension to plot ("layer" or "time"). The default is "layer". + layer : int, optional + Layer index to plot. The default is None (all layers). + period : int, optional + Stress period to plot. The default is None (all periods). + cmap : str, optional + Colormap name. The default is "viridis". + scale_cbar : bool, optional + Scale colorbar. The default is True. + vmin : float, optional + Minimum value for color scale. The default is None. + vmax : float, optional + Maximum value for color scale. The default is None. + norm : matplotlib.colors.Normalize, optional + Color normalization. The default is None. + xlim : tuple, optional + X-axis limits. The default is None. + ylim : tuple, optional + Y-axis limits. The default is None. + grid : bool, optional + Show grid. The default is False. + figsize : tuple, optional + Figure size. The default is (10, 8). + plot_bc : callable, optional + Function to plot boundary conditions. The default is None. + plot_grid : bool, optional + Plot grid. The default is False. + + Returns + ------- + fig : matplotlib Figure + Figure object. + axes : list + List of Axes objects. + """ if arr.ndim == 4 and plot_dim == "layer": nplots = arr.shape[1] elif arr.ndim == 4 and plot_dim == "time": diff --git a/nlmod/plot/plot.py b/nlmod/plot/plot.py index 265c0cef..5419eb5e 100644 --- a/nlmod/plot/plot.py +++ b/nlmod/plot/plot.py @@ -1,17 +1,14 @@ import logging -import warnings from functools import partial -import flopy as fp import matplotlib.pyplot as plt import numpy as np import pandas as pd import xarray as xr from matplotlib.animation import FFMpegWriter, FuncAnimation from matplotlib.collections import PatchCollection -from matplotlib.colors import ListedColormap, Normalize, LinearSegmentedColormap +from matplotlib.colors import LinearSegmentedColormap, ListedColormap, Normalize from matplotlib.patches import Patch -from mpl_toolkits.axes_grid1 import make_axes_locatable from ..dims.grid import ( get_affine_mod_to_world, @@ -35,16 +32,50 @@ def surface_water(model_ds, ax=None, **kwargs): + """Plot surface water from model dataset. + + Parameters + ---------- + model_ds : xarray.Dataset + Model dataset containing surface water data. + ax : matplotlib Axes, optional + Axes to plot on. The default is None. + **kwargs : dict + Additional keyword arguments passed to GeoDataFrame.plot. + + Returns + ------- + ax : matplotlib Axes + Axes with surface water plotted. + """ surf_water = rws.get_gdf_surface_water(model_ds) if ax is None: _, ax = plt.subplots() - surf_water.plot(ax=ax, **kwargs) + surf_water.plot(ax=ax, add_labels=kwargs.pop("add_labels", False), **kwargs) return ax def modelgrid(ds, ax=None, rotated=False, **kwargs): + """Plot model grid from dataset. + + Parameters + ---------- + ds : xarray.Dataset + Model dataset. + ax : matplotlib Axes, optional + Axes to plot on. The default is None. + rotated : bool, optional + Whether the grid is rotated. The default is False. + **kwargs : dict + Additional keyword arguments passed to modelgrid.plot. + + Returns + ------- + ax : matplotlib Axes + Axes with model grid plotted. + """ if ax is None: _, ax = plt.subplots(figsize=(10, 10)) ax.set_aspect("auto") @@ -97,112 +128,120 @@ def modelextent(ds, dx=None, ax=None, rotated=False, **kwargs): def facet_plot( - gwf, ds, - plot_var, - plot_time=None, - plot_bc=None, - color="k", - grid=False, - xlim=None, - ylim=None, + da, + dim="layer", + selection=None, + cmap="turbo", + rotated=True, + figsize=(10, 10), + layout=None, + show_titles=True, + show_ticks=True, + base=10_000, + fmt_base=1_000, + fmt="{:.0f}", + colorbar=True, + **kwargs, ): - """Make a 2d plot of every modellayer, store them in a grid. + """Plot a facet grid of maps for each layer in the dataset. Parameters ---------- - gwf : Groundwater flow - Groundwaterflow model. - ds : xr.DataSet - model dataset. - figdir : str - file path figures. - plot_var : str - variable in ds - plot_time : int, optional - time step if plot_var is time variant. The default is None. - plot_bc : list of str, optional - name of packages of which boundary conditions are plot. The default - is ['CHD']. - color : str, optional - color. The default is 'k'. - grid : bool, optional - if True a grid is plotted. The default is False. - xlim : tuple, optional - xlimits. The default is None. - ylim : tuple, optional - ylimits. The default is None. + ds : xarray.Dataset + The dataset containing the data to be plotted. Must contain a "layer" dimension. + da : str or xarray.DataArray + The name of the DataArray in ds to be plotted, or the DataArray itself. + dim : str, optional + The dimension along which to facet the plots. Default is "layer". + selection : list, optional + A list of dimension names i.e. layer names or timestamps to select from the + dataset before plotting. Default is None, which plots all layers. + cmap : str or Colormap, optional + The colormap to be used for the plots. Default is "turbo". + rotated : bool, optional + When True, plot the data in real-world coordinates for rotated grids. + The default is True, which plots the data in local coordinates. + figsize : tuple, optional + The size of the figure in inches. Default is (10, 10). + layout : str or None, optional + Matplotlib subplot layout mode passed to get_map. The default is None, + which avoids the overhead of constrained layout during batch rendering. + show_titles : bool, optional + When True, draw a title inside each facet. Default is True. + show_ticks : bool, optional + When True, draw axis ticks using base and fmt_base. Set to False for + faster batch rendering. Default is True. + base : int, optional + The base for the axis tick formatting. Default is 1,000. + fmt_base : int, optional + The base for the tick labels formatting. Default is 1,000. + fmt : str, optional + The format string for the tick labels. Default is "{:.0f}". + **kwargs + Additional keyword arguments to be passed to the data_array function for + plotting. Returns ------- - fig : TYPE - DESCRIPTION. - axes : TYPE - DESCRIPTION. + axes : numpy.ndarray + An array of matplotlib.axes.Axes objects corresponding to each layer plotted. """ - warnings.warn( - "this function is out of date and will probably be removed in a future version", - DeprecationWarning, - ) + if isinstance(da, str): + da = ds[da] - if plot_bc is not None: - for key in plot_bc: - if key not in gwf.get_package_list(): - raise ValueError( - f"cannot plot boundary condition {key} " - "because it is not in the package list" - ) + if selection is None: + selection = da[dim].values - nlay = len(ds.layer) + nselection = len(selection) + if nselection == 0: + raise ValueError("selection must contain at least one value") - plots_per_row = int(np.ceil(np.sqrt(nlay))) - plots_per_col = nlay // plots_per_row + 1 + ndim = np.sqrt(nselection) + nrows = np.floor(ndim).astype(int) + ncols = np.ceil(ndim).astype(int) + if nrows * ncols < nselection: + nrows += 1 - fig, axes = plt.subplots( - plots_per_col, - plots_per_row, - figsize=(11, 10), + f, axes = get_map( + get_extent(ds, rotated=rotated), + figsize=figsize, + nrows=nrows, + ncols=ncols, sharex=True, sharey=True, - dpi=150, + base=base if show_ticks else None, + fmt_base=fmt_base, + fmt=fmt, + layout=layout, ) - if plot_time is None: - plot_arr = ds[plot_var] - else: - plot_arr = ds[plot_var][plot_time] - - vmin = plot_arr.min() - vmax = plot_arr.max() - for ilay in range(nlay): - iax = axes.flat[ilay] - mp = fp.plot.PlotMapView(model=gwf, layer=ilay, ax=iax) - # mp.plot_grid() - qm = mp.plot_array(plot_arr[ilay].values, cmap="viridis", vmin=vmin, vmax=vmax) - # qm = mp.plot_array(hf[-1], cmap="viridis", vmin=-0.1, vmax=0.1) - # mp.plot_ibound() - # plt.colorbar(qm) - if plot_bc is not None: - for bc_var in plot_bc: - mp.plot_bc(bc_var, color=color, kper=0) - - iax.set_aspect("equal", adjustable="box") - iax.set_title(f"Layer {ilay}") - - iax.grid(grid) - if xlim is not None: - iax.set_xlim(xlim) - if ylim is not None: - iax.set_ylim(ylim) - - for iax in axes.flat[nlay:]: - iax.set_visible(False) - - cb = fig.colorbar(qm, ax=axes, shrink=1.0) - cb.set_label(f"{plot_var}", rotation=270) - fig.suptitle(f"{plot_var} Time = {(ds.nper * ds.perlen) / 365} year") - fig.tight_layout() - - return fig, axes + + plot_ds = ds + if "icell2d" in da.dims: + plot_ds = get_patches(ds, rotated=rotated) + + qm = None # please linter + for i, isel in enumerate(selection): + iax = axes.flat[i] if isinstance(axes, np.ndarray) else axes + qm = data_array( + da.loc[{dim: isel}], + plot_ds, + ax=iax, + cmap=cmap, + rotated=rotated, + **kwargs, + ) + if show_titles: + if isinstance(isel, np.datetime64): + isel = pd.Timestamp(isel) + title_inside(f"{isel}", ax=iax) + if isinstance(axes, np.ndarray): + for j in range(nselection, len(axes.flat)): + axes.flat[j].set_visible(False) + if qm is not None and colorbar: + cbar = f.colorbar(qm, ax=axes) + cbar.set_label(f"{da.name} [{da.attrs.get('units', '')}]") + return axes def data_array(da, ds=None, ax=None, rotated=False, edgecolor=None, **kwargs): @@ -274,6 +313,64 @@ def data_array(da, ds=None, ax=None, rotated=False, edgecolor=None, **kwargs): return ax.pcolormesh(x, y, da, shading=shading, edgecolor=edgecolor, **kwargs) +def geotop_strat_in_cross_section( + line, + gt=None, + ax=None, + legend=True, + legend_loc=None, + strat_props=None, + alpha=None, + **kwargs, +): + """Plot the stratigraphic-data of GeoTOP in a cross-section. + + Parameters + ---------- + line : shapely.LineString + The line along which the GeoTOP data is plotted + gt : xr.Dataset, optional + The voxel-dataset from GeoTOP. It is downloaded with the method + `nlmod.read.geotop.get_geotop()` if None. The default is None. + ax : matplotlib.Axes, optional + The axes in which the cross-section is plotted. Will default to the current axes + if None. The default is None. + legend : bool, optional + When True, add a legend to the plot with the lithology-classes. The default is + True. + legend_loc : None or str, optional + The location of the legend. See matplotlib documentation. The default is None. + strat_props : pd.DataFrame, optional + A DataFrame containing the properties of the stratigraphic classes. + Will call nlmod.read.geotop.get_strat_props() when None. The default is None. + alpha : float, optional + Opacity for plot_array function, The default is None. + **kwargs : dict + kwargs are passed onto DatasetCrossSection. + + Returns + ------- + cs : DatasetCrossSection + The instance of DatasetCrossSection that is used to plot the cross-section. + """ + if strat_props is None: + strat_props = geotop.get_strat_props() + + cs = geotop_var_in_cross_section( + line, + "strat", + gt=gt, + ax=ax, + legend=legend, + legend_loc=legend_loc, + var_props=strat_props, + alpha=alpha, + label_col="code", + **kwargs, + ) + return cs + + def geotop_lithok_in_cross_section( line, gt=None, @@ -311,6 +408,71 @@ def geotop_lithok_in_cross_section( **kwargs : dict kwargs are passed onto DatasetCrossSection. + Returns + ------- + cs : DatasetCrossSection + The instance of DatasetCrossSection that is used to plot the cross-section. + """ + if lithok_props is None: + lithok_props = geotop.get_lithok_props() + + cs = geotop_var_in_cross_section( + line, + "lithok", + gt=gt, + ax=ax, + legend=legend, + legend_loc=legend_loc, + var_props=lithok_props, + alpha=alpha, + **kwargs, + ) + return cs + + +def geotop_var_in_cross_section( + line, + var, + gt=None, + ax=None, + legend=True, + legend_loc=None, + var_props=None, + alpha=None, + label_col="name", + **kwargs, +): + """Plot a variable (lithoclass or stratigraphy) from GeoTOP in a cross-section. + + Parameters + ---------- + line : shapely.LineString + The line along which the GeoTOP data is plotted + var : str + The variable in the GeoTOP dataset to plot in the cross-section. Can be + 'lithok' or 'strat' + gt : xr.Dataset, optional + The voxel-dataset from GeoTOP. It is downloaded with the method + `nlmod.read.geotop.get_geotop()` if None. The default is None. + ax : matplotlib.Axes, optional + The axes in which the cross-section is plotted. Will default to the current axes + if None. The default is None. + legend : bool, optional + When True, add a legend to the plot with the lithology-classes. The default is + True. + legend_loc : None or str, optional + The location of the legend. See matplotlib documentation. The default is None. + var_props : pd.DataFrame, optional + A DataFrame containing the properties of the variable (lithoclasses or + stratigraphy). Will call nlmod.read.geotop.get_{var}_props() when None. + The default is None. + alpha : float, optional + Opacity for plot_array function, The default is None. + label_col : str, optional + column in the var_props dataframe to use a a label. The default is 'name'. + **kwargs : dict + kwargs are passed onto DatasetCrossSection. + Returns ------- cs : DatasetCrossSection @@ -329,16 +491,25 @@ def geotop_lithok_in_cross_section( if "top" not in gt or "botm" not in gt: gt = geotop.add_top_and_botm(gt) - if lithok_props is None: - lithok_props = geotop.get_lithok_props() + if var_props is None: + if var == "lithok": + var_props = geotop.get_lithok_props() + elif var == "strat": + var_props = geotop.get_strat_props() + else: + raise ValueError( + f"Variable {var} not recognized. Can only be 'lithok' or 'strat'." + ) cs = DatasetCrossSection(gt, line, layer="z", ax=ax, **kwargs) - array, cmap, norm = _get_geotop_cmap_and_norm(gt["lithok"], lithok_props) + array, cmap, norm = _get_geotop_cmap_and_norm(gt[var], var_props) cs.plot_array(array, norm=norm, cmap=cmap, alpha=alpha) if legend: # make a legend with dummy handles - _add_geotop_lithok_legend(lithok_props, ax, lithok=gt["lithok"], loc=legend_loc) + _add_geotop_var_legend( + var_props, ax, var=gt[var], label_col=label_col, loc=legend_loc + ) return cs @@ -390,21 +561,21 @@ def geotop_lithok_on_map( qm = ax.pcolormesh(lithok.x, lithok.y, array, norm=norm, cmap=cmap, **kwargs) if legend: # make a legend with dummy handles - _add_geotop_lithok_legend(lithok_props, ax, lithok=lithok, loc=legend_loc) + _add_geotop_var_legend(lithok_props, ax, var=lithok, loc=legend_loc) return qm -def _add_geotop_lithok_legend(lithok_props, ax, lithok=None, **kwargs): - """Add a legend with lithok-data.""" +def _add_geotop_var_legend(var_props, ax, var=None, label_col="name", **kwargs): + """Add a legend with lithok- or strat-data.""" handles = [] - if lithok is None: - lithoks = lithok_props.index + if var is None: + unique_vals = var_props.index else: - lithoks = np.unique(lithok) - lithoks = lithoks[~np.isnan(lithoks)] - for index in lithoks: - color = lithok_props.at[index, "color"] - label = lithok_props.at[int(index), "name"] + unique_vals = np.unique(var) + unique_vals = unique_vals[~np.isnan(unique_vals)] + for index in unique_vals: + color = var_props.at[index, "color"] + label = var_props.at[int(index), label_col] handles.append(Patch(facecolor=color, label=label)) return ax.legend(handles=handles, **kwargs) @@ -450,9 +621,7 @@ def _get_figure(ax=None, da=None, ds=None, figsize=None, rotated=False, extent=N fmt = "{:.1f}" else: fmt = "{:.0f}" - f, ax = get_map( - extent, base=base, figsize=figsize, tight_layout=False, layout=None, fmt=fmt - ) + f, ax = get_map(extent, base=base, figsize=figsize, layout=None, fmt=fmt) ax.set_aspect("equal", adjustable="box") return f, ax @@ -484,6 +653,66 @@ def map_array( animate=False, **kwargs, ): + """Plot a map of a data array from a model dataset. + + Parameters + ---------- + da : str, xr.DataArray, or np.ndarray + Data array or variable name to plot. + ds : xarray.Dataset + Model dataset. + ilay : int, optional + Layer index to plot. The default is 0. + iper : int, optional + Stress period index to plot. The default is 0. + extent : tuple, optional + Extent for the plot. The default is None. + ax : matplotlib Axes, optional + Axes to plot on. The default is None. + title : str, optional + Plot title. The default is "". + xlabel : str, optional + X-axis label. The default is "X [km RD]". + ylabel : str, optional + Y-axis label. The default is "Y [km RD]". + date_fmt : str, optional + Date format string. The default is "%Y-%m-%d". + norm : matplotlib.colors.Normalize, optional + Color normalization. The default is None. + vmin : float, optional + Minimum value for color scale. The default is None. + vmax : float, optional + Maximum value for color scale. The default is None. + levels : list, optional + Contour levels. The default is None. + cmap : str, optional + Colormap name. The default is "viridis". + alpha : float, optional + Transparency. The default is 1.0. + colorbar : bool, optional + Show colorbar. The default is True. + colorbar_label : str, optional + Colorbar label. The default is "". + plot_grid : bool, optional + Plot grid. The default is True. + rotated : bool, optional + Whether the grid is rotated. The default is False. + add_to_plot : callable, optional + Function to add additional elements to the plot. The default is None. + background : bool, optional + Use background. The default is False. + figsize : tuple, optional + Figure size. The default is None. + animate : bool, optional + Animate the plot. The default is False. + **kwargs : dict + Additional keyword arguments passed to matplotlib imshow. + + Returns + ------- + ax : matplotlib Axes + Axes with the array plotted. + """ # get data if isinstance(da, str): da = ds[da] @@ -527,7 +756,6 @@ def map_array( else: t = None - fig_tight_layout = ax is None f, ax = _get_figure( ax=ax, da=da, ds=ds, figsize=figsize, rotated=rotated, extent=extent ) @@ -573,16 +801,14 @@ def map_array( ax.set(**axprops) # colorbar - divider = make_axes_locatable(ax) if colorbar: - cax = divider.append_axes("right", size="5%", pad=0.1) - cbar = f.colorbar(pc, cax=cax, extend=kwargs.pop("extend", "neither")) + cbar = f.colorbar( + pc, ax=ax, extend=kwargs.pop("extend", "neither"), fraction=0.046, pad=0.04 + ) if levels is not None: cbar.set_ticks(levels) cbar.set_label(colorbar_label) - _ = f.tight_layout() if fig_tight_layout else None - if animate: return f, ax, pc else: @@ -802,7 +1028,8 @@ def get_ahn_colormap(name="ahn", N=256): Notes ----- The color progression is as follows: - dark blue → medium blue → light blue → dark green → light green → yellow → orange → light red → dark red + dark blue → medium blue → light blue → dark green → light green → + yellow → orange → light red → dark red """ colors = np.array( [ diff --git a/nlmod/prt/prt.py b/nlmod/prt/prt.py index e553a869..d107cf03 100644 --- a/nlmod/prt/prt.py +++ b/nlmod/prt/prt.py @@ -1,8 +1,9 @@ import logging from pathlib import Path -from pandas import read_csv, DataFrame + import flopy as fp from flopy.plot.plotutil import PRT_PATHLINE_DTYPE +from pandas import DataFrame, read_csv from ..gwf.gwf import _dis, _disv, _set_record from ..util import _get_value_from_ds_datavar @@ -201,6 +202,26 @@ def fmi(ds, prt, packagedata=None, **kwargs): def oc(ds, prt, save_budget=True, print_budget=False, **kwargs): + """Create output control package for particle tracking model. + + Parameters + ---------- + ds : xarray.Dataset + dataset with model data. + prt : ModflowPrt + modflow particle tracking object + save_budget : bool, optional + save budget. The default is True. + print_budget : bool, optional + print budget. The default is False. + **kwargs : dict + additional keyword arguments passed to ModflowPrtoc. + + Returns + ------- + oc : flopy ModflowPrtoc + output control package. + """ logger.info("creating mf6 OC") budget_filerecord = kwargs.pop("budget_filerecord", [f"{ds.model_name}_prt.cbc"]) @@ -228,7 +249,8 @@ def read_pathlines(path: str | Path, icell2d: int | None = None) -> DataFrame: - kper: stress period number - kstp: time step number - imdl: number of the model the particle originated in - - iprp: number of the particle release point (PRP) package the particle originated in + - iprp: number of the particle release point (PRP) package + the particle originated in - irpt: release point number - ilay: layer number - icell: cell number diff --git a/nlmod/read/__init__.py b/nlmod/read/__init__.py index e4aa418b..bfcd4678 100644 --- a/nlmod/read/__init__.py +++ b/nlmod/read/__init__.py @@ -11,6 +11,7 @@ jarkus, knmi, knmi_data_platform, + lgn, meteobase, nhi, regis, @@ -18,7 +19,7 @@ waterboard, webservices, ) -from .geotop import download_geotop, get_geotop -from .regis import download_regis, get_regis from .ahn import download_ahn, get_ahn from .bgt import download_bgt, get_bgt +from .geotop import download_geotop, get_geotop +from .regis import download_regis, get_regis diff --git a/nlmod/read/administrative.py b/nlmod/read/administrative.py index 2c899579..b9601cf4 100644 --- a/nlmod/read/administrative.py +++ b/nlmod/read/administrative.py @@ -1,7 +1,7 @@ import warnings -from . import waterboard, webservices from .. import cache +from . import waterboard, webservices def get_municipalities(*args, **kwargs): @@ -28,8 +28,10 @@ def get_municipalities(*args, **kwargs): """ warnings.warn( "this function is deprecated and will eventually be removed, " - "please use nlmod.read.administrative.download_municipalities_gdf() in the future.", + "please use nlmod.read.administrative.download_municipalities_gdf() " + "in the future.", DeprecationWarning, + stacklevel=2, ) return download_municipalities_gdf(*args, **kwargs) @@ -97,11 +99,11 @@ def get_provinces(*args, **kwargs): gpd.GeoDataFrame polygons of provinces """ - warnings.warn( "this function is deprecated and will eventually be removed, " "please use nlmod.read.administrative.download_provinces_gdf() in the future.", DeprecationWarning, + stacklevel=2, ) return download_provinces_gdf(*args, **kwargs) @@ -160,11 +162,12 @@ def get_netherlands(*args, **kwargs): gpd.GeoDataFrame polygons of the Netherlands """ - warnings.warn( "this function is deprecated and will eventually be removed, " - "please use nlmod.read.administrative.download_netherlands_gdf() in the future.", + "please use nlmod.read.administrative.download_netherlands_gdf() " + "in the future.", DeprecationWarning, + stacklevel=2, ) return download_netherlands_gdf(*args, **kwargs) @@ -221,8 +224,10 @@ def get_waterboards(**kwargs): """ warnings.warn( "this function is deprecated and will eventually be removed, " - "please use nlmod.read.administrative.download_waterboards_gdf() in the future.", + "please use nlmod.read.administrative.download_waterboards_gdf() " + "in the future.", DeprecationWarning, + stacklevel=2, ) return waterboard.download_polygons(**kwargs) @@ -243,3 +248,23 @@ def download_waterboards_gdf(**kwargs): polygons of the Netherlands """ return waterboard.download_polygons(**kwargs) + + +def download_kadaster_percelen(extent): + """Download percelen (cadastral parcels) from the Dutch Kadaster. + + Parameters + ---------- + extent : list, tuple or np.array + The extent to be downloaded, consisting of 4 floats: xmin, xmax, ymin, ymax. + + Returns + ------- + gdf : gpd.GeoDataFrame + GeoDataFrame with polygons of the parcels + + """ + url = "https://service.pdok.nl/kadaster/kadastralekaart/wfs/v5_0?service=WFS" + layer = "kadastralekaartv5:perceel" + gdf = webservices.wfs(url, layer, extent) + return gdf diff --git a/nlmod/read/ahn.py b/nlmod/read/ahn.py index b8a25d25..755349fd 100644 --- a/nlmod/read/ahn.py +++ b/nlmod/read/ahn.py @@ -1,8 +1,8 @@ import datetime as dt import logging import os -import warnings import tempfile +import warnings from typing import Literal import geopandas as gpd @@ -15,6 +15,7 @@ from rasterio.env import Env from requests.exceptions import HTTPError from rioxarray.merge import merge_arrays + from .. import NLMOD_DATADIR, cache from ..dims.grid import get_extent from ..dims.resample import structured_da_to_ds @@ -222,9 +223,11 @@ def get_ahn( """ if ds is None: warnings.warn( - "calling 'get_ahn' with ds=None is deprecated and will raise an error in the " - "future. Use 'nlmod.read.ahn.download_ahn' to get the ahn within an extent", + "calling 'get_ahn' with ds=None is deprecated and will raise an error " + "in the future. Use 'nlmod.read.ahn.download_ahn' to get the ahn " + "within an extent", DeprecationWarning, + stacklevel=2, ) if extent is None and ds is not None: @@ -274,9 +277,11 @@ def get_ahn_at_point( Resturn the mean of all non-nan pixels within buffer. Return the center pixel when False. The default is False. identifier : str, optional - The identifier passed onto download_latest_ahn_from_wcs. The default is "dsm_05m". + The identifier passed onto download_latest_ahn_from_wcs. + The default is "dsm_05m". res : float, optional - The resolution that is passed onto download_latest_ahn_from_wcs. The default is 0.5. + The resolution that is passed onto download_latest_ahn_from_wcs. + The default is 0.5. **kwargs : dict kwargs are passed onto the method download_latest_ahn_from_wcs. @@ -285,11 +290,11 @@ def get_ahn_at_point( float The surface level value at the requested point. """ - warnings.warn( "'get_ahn_at_point' is deprecated and will eventually be removed, " "please use nlmod.read.ahn.download_ahn_at_point() in the future.", DeprecationWarning, + stacklevel=2, ) return download_ahn_at_point( @@ -323,9 +328,11 @@ def download_ahn_at_point( Resturn the mean of all non-nan pixels within buffer. Return the center pixel when False. The default is False. identifier : str, optional - The identifier passed onto download_latest_ahn_from_wcs. The default is "dsm_05m". + The identifier passed onto download_latest_ahn_from_wcs. + The default is "dsm_05m". res : float, optional - The resolution that is passed onto download_latest_ahn_from_wcs. The default is 0.5. + The resolution that is passed onto download_latest_ahn_from_wcs. + The default is 0.5. **kwargs : dict kwargs are passed onto the method download_latest_ahn_from_wcs. @@ -392,6 +399,7 @@ def get_ahn_along_line( "'get_ahn_along_line' is deprecated and will eventually be removed, " "please use nlmod.read.ahn.download_ahn_along_line() in the future.", DeprecationWarning, + stacklevel=2, ) return download_ahn_along_line(line, ahn, dx, num, method, plot) @@ -444,7 +452,7 @@ def download_ahn_along_line( dx = float(ahn.x[1] - ahn.x[0]) s = np.arange(0.0, line.length, dx) - x, y = zip(*[p.xy for p in line.interpolate(s)]) + x, y = zip(*[p.xy for p in line.interpolate(s)], strict=False) x = np.array(x)[:, 0] y = np.array(y)[:, 0] @@ -515,6 +523,7 @@ def get_latest_ahn_from_wcs( "'get_latest_ahn_from_wcs' is deprecated and will eventually be removed, " "please use 'nlmod.read.ahn.download_latest_ahn_from_wcs()' in the future.", DeprecationWarning, + stacklevel=2, ) return download_latest_ahn_from_wcs( @@ -690,6 +699,7 @@ def get_ahn1( "'get_ahn1' is deprecated and will be removed in a future version. " "Use 'nlmod.read.ahn.download_ahn1' instead", DeprecationWarning, + stacklevel=2, ) return download_ahn1(extent, identifier, as_data_array, **kwargs) @@ -765,6 +775,7 @@ def get_ahn2( "'get_ahn2' is deprecated and will be removed in a future version. " "Use 'nlmod.read.ahn.download_ahn2' instead", DeprecationWarning, + stacklevel=2, ) return download_ahn2(extent, identifier, as_data_array, **kwargs) @@ -847,6 +858,7 @@ def get_ahn3( "'get_ahn3' is deprecated and will be removed in a future version. " "Use 'nlmod.read.ahn.download_ahn3' instead", DeprecationWarning, + stacklevel=2, ) return download_ahn3(extent, identifier, as_data_array, **kwargs) @@ -928,6 +940,7 @@ def get_ahn4( "'get_ahn4' is deprecated and will be removed in a future version. " "Use 'nlmod.read.ahn.download_ahn4' instead", DeprecationWarning, + stacklevel=2, ) return download_ahn4(extent, identifier, as_data_array, **kwargs) @@ -1008,6 +1021,7 @@ def get_ahn5( "'get_ahn5' is deprecated and will be removed in a future version. " "Use 'nlmod.read.ahn.download_ahn5' instead", DeprecationWarning, + stacklevel=2, ) return download_ahn5(extent, identifier, **kwargs) @@ -1052,7 +1066,6 @@ def download_ahn5( def download_ahn6(extent: list[float], tile_size: str = "1x1km", **kwargs): """Download AHN6. - Parameters ---------- extent : list, tuple or np.array @@ -1169,7 +1182,7 @@ def _download_and_save_json_file(url, fname, timeout=120.0): def get_configuration(): - """Get the location of json-files with positions of map tiles with AHN-data + """Get the location of json-files with positions of map tiles with AHN-data. Returns ------- @@ -1392,7 +1405,8 @@ def _download_tiles_hwh( except KeyError as e: raise KeyError(f"Data not available for {version}: {e}") from e if url.startswith("http"): - # write the tiles to a TemporaryDirectory, as direct read from url fails for large files + # write the tiles to a TemporaryDirectory, as direct read from url + # fails for large files with tempfile.TemporaryDirectory() as tmpdir: tmp_file = os.path.join(tmpdir, "data.json") _download_and_save_json_file(url, tmp_file, timeout=timeout) @@ -1440,10 +1454,10 @@ def _download_ahn_hwh( If True, the function returns a merged DataArray. If False, the function returns a list of DataArrays with the original tiles. The default is True. cut_extent : bool, optional - If True, only keep the requested extent from the data. The defualts is True. + If True, only keep the requested extent from the data. The default is True. config : dict, optional A dictionary with properties of the data sources of the different AHN-versions. - When None, the configuration is retreived from the method get_configuration(). + When None, the configuration is retrieved from the method get_configuration(). The default is None. Raises diff --git a/nlmod/read/bgt.py b/nlmod/read/bgt.py index 384b4d11..f06b9ece 100644 --- a/nlmod/read/bgt.py +++ b/nlmod/read/bgt.py @@ -20,15 +20,13 @@ def get_bgt(*args, **kwargs): - """Get geometries within an extent or polygon from the Basis Registratie - Grootschalige Topografie (BGT) + """Get geometries from the Basis Registratie Grootschalige Topografie (BGT). .. deprecated:: 0.10.0 `get_bgt` will be removed in nlmod 1.0.0, it is replaced by `download_bgt` because of new naming convention https://github.com/gwmod/nlmod/issues/47 - Parameters ---------- extent : list or tuple of length 4 or shapely Polygon @@ -68,11 +66,11 @@ def get_bgt(*args, **kwargs): A GeoDataFrame (when only one layer is requested) or a dict of GeoDataFrames containing all geometries and properties. """ - warnings.warn( "this function is deprecated and will eventually be removed, " "please use nlmod.read.bgt.download_bgt() in the future.", DeprecationWarning, + stacklevel=2, ) return download_bgt(*args, **kwargs) @@ -90,8 +88,7 @@ def download_bgt( add_bronhouder_names=True, timeout=1200, ): - """Get geometries within an extent or polygon from the Basis Registratie - Grootschalige Topografie (BGT) + """Get geometries from the Basis Registratie Grootschalige Topografie (BGT). Parameters ---------- @@ -282,6 +279,23 @@ def read_bgt_zipfile( def read_bgt_gml(fname, geometry="geometrie2dGrondvlak", crs="epsg:28992"): + """Read BGT data from a GML file. + + Parameters + ---------- + fname : str + Path to the GML file. + geometry : str, optional + The geometry attribute to use. The default is "geometrie2dGrondvlak". + crs : str, optional + The coordinate reference system. The default is "epsg:28992". + + Returns + ------- + list + List of dictionaries containing BGT features and their geometries. + """ + def get_xy(text): xy = [float(val) for val in text.split()] xy = np.array(xy).reshape(int(len(xy) / 2), 2) @@ -408,7 +422,7 @@ def read_label(child, d): def get_bgt_layers(timeout=1200): """ - Get the layers in the Basis Registratie Grootschalige Topografie (BGT) + Get the layers in the Basis Registratie Grootschalige Topografie (BGT). Parameters ---------- @@ -455,7 +469,7 @@ def get_bronhouder_names() -> Dict[str, str]: def _update_bronhouder_json(fname): """ - "Update the json-file within nlmod with bronhouder-names + "Update the json-file within nlmod with bronhouder-names. Parameters ---------- diff --git a/nlmod/read/bofek.py b/nlmod/read/bofek.py index 04321ff0..53ed1add 100644 --- a/nlmod/read/bofek.py +++ b/nlmod/read/bofek.py @@ -1,5 +1,5 @@ import logging -import shutil +import tempfile import warnings import zipfile from io import BytesIO @@ -15,11 +15,10 @@ def get_gdf_bofek(*args, **kwargs): - """Get geodataframe of bofek 2020 wihtin the extent of the model. + """Get geodataframe of bofek 2020 within the extent of the model. - It does so by downloading a zip file (> 100 MB) and extracting the relevant - geodatabase. Therefore the function can be slow, ~35 seconds depending on your - internet connection. + It does so by downloading a zip file and reading the included shapefile. + Therefore the function can be slow depending on your internet connection. .. deprecated:: 0.10.0 `get_gdf_bofek` will be removed in nlmod 1.0.0, it is replaced by @@ -44,15 +43,13 @@ def get_gdf_bofek(*args, **kwargs): Notes ----- - An attempt was made to read the geodatabase in memory from the zip file wihtout - writing data to disk, but this was not successful. Mainly because of the difficulty - to read the geodatabase in memory. + The shapefile is extracted temporarily before reading it. """ - warnings.warn( "this function is deprecated and will eventually be removed, " "please use nlmod.read.bofek.download_bofek_gdf() in the future.", DeprecationWarning, + stacklevel=2, ) return download_bofek_gdf(*args, **kwargs) @@ -60,11 +57,10 @@ def get_gdf_bofek(*args, **kwargs): @cache.cache_pickle def download_bofek_gdf(extent, dirname, timeout=3600): - """Get geodataframe of bofek 2020 wihtin the extent of the model. + """Get geodataframe of bofek 2020 within the extent of the model. - It does so by downloading a zip file (> 100 MB) and extracting the relevant - geodatabase. Therefore the function can be slow, ~35 seconds depending on your - internet connection. + It does so by downloading a zip file and reading the included shapefile. + Therefore the function can be slow depending on your internet connection. Parameters ---------- @@ -84,21 +80,15 @@ def download_bofek_gdf(extent, dirname, timeout=3600): Notes ----- - An attempt was made to read the geodatabase in memory from the zip file wihtout - writing data to disk, but this was not successful. Mainly because of the difficulty - to read the geodatabase in memory. + The shapefile is extracted temporarily before reading it. """ - import py7zr - - # set paths dirname = Path(dirname) - fname_bofek_gdb = dirname / "GIS" / "BOFEK2020_bestanden" / "BOFEK2020.gdb" # create directories if they do not exist dirname.mkdir(exist_ok=True, parents=True) # url - bofek_zip_url = "https://www.wur.nl/nl/show/bofek-2020-gis-1.htm" + bofek_zip_url = "https://bodemdata.nl/files/download/BOFEK_2020_Shape.zip" # download zip logger.info("Downloading BOFEK2020 GIS data (~35 seconds)") @@ -116,24 +106,16 @@ def download_bofek_gdf(extent, dirname, timeout=3600): progress_bar.update(len(data)) file_unzipped.write(data) - # extract geodatabase from 7z - with zipfile.ZipFile(file_unzipped, mode="r") as zf: - with py7zr.SevenZipFile(BytesIO(zf.read(zf.filelist[0])), mode="r") as z: - z.extract( - targets=["GIS/BOFEK2020_bestanden/BOFEK2020.gdb"], - path=dirname, - recursive=True, + with tempfile.TemporaryDirectory(dir=dirname) as extract_dir: + with zipfile.ZipFile(file_unzipped, mode="r") as zf: + zf.extractall( + path=extract_dir, + members=[ + f"BOFEK_2020.{ext}" for ext in ("shp", "shx", "dbf", "prj", "cpg") + ], ) - # read geodatabase - logger.debug("convert geodatabase to geojson") - gdf_bofek = gpd.read_file(fname_bofek_gdb) - - # slice to extent - gdf_bofek = util.gdf_within_extent(gdf_bofek, extent) - - # clean up - logger.debug("Remove geodatabase") - shutil.rmtree(fname_bofek_gdb) + gdf_bofek = gpd.read_file(Path(extract_dir) / "BOFEK_2020.shp") + gdf_bofek = util.gdf_within_extent(gdf_bofek, extent) return gdf_bofek diff --git a/nlmod/read/bro.py b/nlmod/read/bro.py index 7b7e3f19..a18a50fa 100644 --- a/nlmod/read/bro.py +++ b/nlmod/read/bro.py @@ -3,8 +3,7 @@ import numpy as np import pandas as pd - -from pyproj import Transformer +import pyproj from .. import cache, util @@ -38,11 +37,11 @@ def get_bro(*args, **kwargs): ------- ObsCollection """ - warnings.warn( "this function is deprecated and will eventually be removed, " "please use nlmod.read.bro.download_bro_groundwater() in the future.", DeprecationWarning, + stacklevel=2, ) return download_bro_groundwater(*args, **kwargs) @@ -80,7 +79,7 @@ def download_bro_groundwater( """ # convert extent to epsg 4326 if epsg != 4326: - transformer = Transformer.from_crs(epsg, 4326) + transformer = pyproj.Transformer.from_crs(epsg, 4326) lat1, lon1 = transformer.transform(extent[0], extent[2]) lat2, lon2 = transformer.transform(extent[1], extent[3]) extent = (lon1, lon2, lat1, lat2) @@ -131,7 +130,8 @@ def download_bro_groundwater( oc_list.append(oc) except Exception as e: logger.error( - f"could not download BRO data in extent {xmin}, {xmax}, {ymin}, {ymax}" + f"could not download BRO data in extent " + f"{xmin}, {xmax}, {ymin}, {ymax}" ) logger.error(e) else: @@ -146,6 +146,7 @@ def download_bro_groundwater( ) oc_list.append(oc) oc = pd.concat(oc_list) + oc.set_crs(4326) # crs info is lost after pd.concat operation else: name = "BRO_" + "_".join(map(str, extent)) oc = _get_bro_within_extent( @@ -160,12 +161,15 @@ def download_bro_groundwater( if oc.empty: logger.warning("no observation wells within extent") + if oc.crs != pyproj.CRS(epsg): + oc = oc.to_crs(epsg) + return oc @cache.cache_pickle def _get_bro_within_extent(extent, name, ignore_max_obs, epsg, **kwargs): - """Get bro groundwater measurements within extent + """Get bro groundwater measurements within extent. Parameters ---------- @@ -187,5 +191,5 @@ def _get_bro_within_extent(extent, name, ignore_max_obs, epsg, **kwargs): """ hpd = util.import_hydropandas(method="nlmod.read.bro.download_bro_groundwater()") return hpd.read_bro( - extent, name=name, ignore_max_obs=ignore_max_obs, epsg=epsg, **kwargs + extent, name=name, ignore_max_obs=ignore_max_obs, crs=epsg, **kwargs ) diff --git a/nlmod/read/brp.py b/nlmod/read/brp.py index 97113cad..e9d4ccc9 100644 --- a/nlmod/read/brp.py +++ b/nlmod/read/brp.py @@ -12,11 +12,11 @@ def get_percelen(*args, **kwargs): https://github.com/gwmod/nlmod/issues/47 """ - warnings.warn( "this function is deprecated and will eventually be removed, " "please use nlmod.read.brp.download_percelen_gdf() in the future.", DeprecationWarning, + stacklevel=2, ) return download_percelen_gdf(*args, **kwargs) @@ -30,8 +30,5 @@ def download_percelen_gdf(extent, year=None): gdf = webservices.wfs(url, layer, extent) gdf = gdf.set_index("fuuid") else: - if year < 2009 or year > 2021: - raise (ValueError("Only data available from 2009 up to and including 2021")) - url = f"https://services.arcgis.com/nSZVuSZjHpEZZbRo/ArcGIS/rest/services/BRP_{year}/FeatureServer" - gdf = webservices.arcrest(url, 0, extent=extent) + raise ValueError("The argument `year` is not supported any more.") return gdf diff --git a/nlmod/read/brt.py b/nlmod/read/brt.py index 4678a4cf..fa990a4c 100644 --- a/nlmod/read/brt.py +++ b/nlmod/read/brt.py @@ -72,6 +72,7 @@ def get_brt(*args, **kwargs): "this function is deprecated and will eventually be removed, " "please use nlmod.read.brt.download_brt_gdf() in the future.", DeprecationWarning, + stacklevel=2, ) return download_brt_gdf(*args, **kwargs) diff --git a/nlmod/read/geotop.py b/nlmod/read/geotop.py index 473f2985..d84be029 100644 --- a/nlmod/read/geotop.py +++ b/nlmod/read/geotop.py @@ -17,6 +17,18 @@ def get_lithok_props(rgb_colors=True): + """Get lithok properties from GeoTOP. + + Parameters + ---------- + rgb_colors : bool, optional + If True, add RGB color values to the DataFrame. The default is True. + + Returns + ------- + pd.DataFrame + DataFrame with lithok properties. + """ fname = os.path.join(NLMOD_DATADIR, "geotop", "litho_eenheden.csv") df = pd.read_csv(fname, index_col=0) if rgb_colors: @@ -25,6 +37,13 @@ def get_lithok_props(rgb_colors=True): def get_lithok_colors(): + """Get RGB color values for lithok classes. + + Returns + ------- + dict + Dictionary mapping lithok class values to RGB color tuples (normalized to 0-1). + """ colors = { 0: (200, 200, 200), 1: (157, 78, 64), @@ -41,6 +60,13 @@ def get_lithok_colors(): def get_strat_props(): + """Get stratigraphic properties from GeoTOP. + + Returns + ------- + pd.DataFrame + DataFrame with stratigraphic properties including code, name, and color. + """ fname = os.path.join(NLMOD_DATADIR, "geotop", "REF_GTP_STR_UNIT.csv") df = pd.read_csv(fname, keep_default_na=False, na_values="") # rename the columns to previously used values @@ -63,6 +89,19 @@ def get_strat_props(): def get_kh_kv_table(kind="Brabant"): + """Get the table with hydraulic conductivities and vertical anisotropy for GeoTOP. + + Parameters + ---------- + kind : str, optional + The kind of table to return. The default is "Brabant". + + Returns + ------- + pd.DataFrame + DataFrame with columns "strat", "kh", "kv" and "vani". + The index is the stratigraphic unit code. + """ if kind == "Brabant": fname = os.path.join( NLMOD_DATADIR, @@ -75,6 +114,277 @@ def get_kh_kv_table(kind="Brabant"): return df +def split_layers_on_geul(strat, units_no_geul, geulen): + """Insert paleochannels (geulen) into the stratigraphic data. + + Modifies the stratigraphic data from geotop in such a way that every stratigraphic + unit is completely above or completely below any other unit (and never both above + and below the same unit). This is useful for creating a layer model from the + stratigraphic units. + + This function splits the geulen over multiple layers and splits other layers + locally when a geul is partly above and partly below that layer. + + Some extra logic is added to minimize the number of geul layers by finding the + most efficient layer to add the geul to. + + Parameters + ---------- + strat : xarray DataArray + with dimensions z, y and x, with values corresponding to the stratigraphic + unit in each voxel. + units_no_geul : list + Ordered list of stratigraphic units without geulen. + geulen : list + Ordered list of geulen units. + + Returns + ------- + strat : xarray DataArray + Modified stratigraphic data, including all the new geul units and the non-geul + units that are split because the geul was in between them. + new_unit_order : list + Ordered list with new stratigraphic units, including all the new geul units + and the non-geul units that are split because the geul was in between them. + + Notes + ----- + the new stratigraphic data contains more unit numbers than the original + stratigraphic data. When a layer is split by a geul, the part of the layer + below the geul gets a new unit number. The same goes for the geul itself when it + occurs across multiple layers. + """ + unit_order = units_no_geul.copy() + strat = strat.copy() + + if not np.unique(strat.z.diff(dim="z")) == -0.5: + raise ValueError( + "this function assumes a layer thickness of 0.5 m, " + "please check the z values of the strat DataArray" + ) + + z = ( + strat["z"] + .data[:, np.newaxis, np.newaxis] + .repeat(len(strat.y), 1) + .repeat(len(strat.x), 2) + ) + + for ilay_geul, geul in enumerate(geulen): + # 1 get top/bot units + + # a. get top/bot for this geul + mask = strat == geul + maxz = np.max(np.where(mask, z, -np.inf), axis=0) + minz = np.min(np.where(mask, z, np.inf), axis=0) + + top_geul = np.where(np.isfinite(maxz), maxz + 0.25, np.nan) + bot_geul = np.where(np.isfinite(minz), minz - 0.25, np.nan) + + # b. get top and bottom height (m NAP) of all other stratographic layers + # top layer is a dummy layer in order to obtain a different value for the + # case where no maximum is found (0) and the case where the maximum is found + # in the top layer (1). + shape_no_geul = (len(unit_order) + 1, len(strat.y), len(strat.x)) + top = np.full(shape_no_geul, np.nan) + bot = np.full(shape_no_geul, np.nan) + + for layer, unit in enumerate(unit_order): + mask = strat == unit + # Use finite sentinels to avoid all-NaN slice warnings for empty cells. + maxz = np.max(np.where(mask, z, -np.inf), axis=0) + minz = np.min(np.where(mask, z, np.inf), axis=0) + top[layer + 1] = np.where(np.isfinite(maxz), maxz + 0.25, np.nan) + bot[layer + 1] = np.where(np.isfinite(minz), minz - 0.25, np.nan) + + # c. get top and bottom height of the model + lay_top = (np.isfinite(strat)).argmax(dim="z").values + z_top = ( + np.take_along_axis(z, lay_top[np.newaxis, ...], axis=0).squeeze(axis=0) + + 0.25 + ) # z value of top layer + lay_bot = (np.isfinite(strat[::-1])).argmax(dim="z").values + z_bot = ( + np.take_along_axis(z[::-1], lay_bot[np.newaxis, ...], axis=0).squeeze( + axis=0 + ) + - 0.25 + ) # z value of bottom layer (nearly always -50.25) + + # 2 find units above and below the geul + # a find for each cell the layer on top of the geul + top_lay_geul = (top_geul == bot).argmax( + axis=0 + ) # index (+1) of unit directly above the geul + top_lay_geul[ + np.isnan(top_geul) + ] = -999 # -999 if geul is not present in vertical + geul_between_lay = ((top_geul < top) & (top_geul > bot)).argmax(axis=0) + top_lay_geul = np.where( + geul_between_lay != 0, -geul_between_lay, top_lay_geul + ) # -index if geul is in between a unit. + top_lay_geul[top_lay_geul == 0] = -888 # -888 if layer above geul is also geul. + top_lay_geul[top_geul == z_top] = 0 # 0 if geul is the top layer + + # b find for each cell the layer below the geul + bot_lay_geul = (bot_geul == top).argmax( + axis=0 + ) # index (+1) of unit directly below the geul + bot_lay_geul[ + np.isnan(bot_geul) + ] = -999 # -999 if geul is not present in vertical + bot_lay_geul[bot_lay_geul == 0] = -888 # -888 if layer below geul is also geul. + bot_lay_geul[bot_geul == z_bot] = ( + len(unit_order) + 1 + ) # 999 if geul is the bottom layer + + bot_lay_geul = np.where( + bot_lay_geul == -888, (bot_geul > bot).argmax(axis=0), bot_lay_geul + ) # if layer below geul is also geul, take the first non-geul layer. + geul_between_lay = ((bot_geul > bot) & (bot_geul < top)).argmax(axis=0) + bot_lay_geul = np.where( + geul_between_lay != 0, -geul_between_lay, bot_lay_geul + ) # -index if geul is in between a unit. + + # check assumption if geul is in both arrays in between a unit + if not ( + (bot_lay_geul < 0) & (bot_lay_geul > -100) + == (top_lay_geul < 0) & (top_lay_geul > -100) + ).all(): + raise ValueError( + "unexpected geulen configuration, probably because a geul is cut by " + "another stratigraphic unit." + ) + + # 3 decide where to add the geul layer + # create an empty array to store for each cell the index number of the unit. + # The geul will be added right below this unit. + lay_geul = np.ones_like(top_lay_geul) * np.nan + + # Get the layers where the geul is in between a unit. + layers, counts = np.unique( + bot_lay_geul[(bot_lay_geul < 0) & (bot_lay_geul != -999)], + return_counts=True, + ) + if len(layers) == 0: # a. The geul is never in between a unit. + layers, counts = np.unique( + [top_lay_geul, bot_lay_geul - 1], return_counts=True + ) + layers = layers[np.argsort(counts)][::-1] + layers = [lay for lay in layers if lay >= 0] + for lay in layers: + mask = np.isnan(lay_geul) & (~np.isnan(top_geul)) + if (~mask).all(): + break # all cells have a layer assigned + lay_geul = np.where( + mask & (top_lay_geul <= lay) & (bot_lay_geul - 1 >= lay), + lay, + lay_geul, + ) # assign geul to unit + else: # b. In some places the geul is in between a unit. + layers = layers[np.argsort(counts)] + for lay in layers: + # Assign geul where it is in between a unit + lay_geul[lay == top_lay_geul] = lay + # Assign geul to same unit wherever that is possible + lay_geul[(abs(lay) >= top_lay_geul) & (abs(lay) <= bot_lay_geul)] = abs( + lay + ) + + # c. In some places the geul is still not assigned to a unit. + # In those cases, assign the geul to the layer closest to the in + # between unit. + + # absolute difference between possible top and in between unit + dif_top = np.ones((len(layers), *top_geul.shape)) * np.nan + # absolute difference between possible bottom and in between unit + dif_bot = np.ones((len(layers), *top_geul.shape)) * np.nan + for i, lay in enumerate(layers): + lay = abs(lay) + dif_top[i] = np.abs(np.abs(top_lay_geul) - lay) + dif_bot[i] = np.abs(np.abs(bot_lay_geul) - 1 - lay) + + top_min = np.min( + dif_top, axis=0 + ) # minimal difference between possible top and in between unit + bot_min = np.min( + dif_bot, axis=0 + ) # minimal difference between possible bottom and in between unit + topbot_min = np.argmin((top_min, bot_min), axis=0) + # closest possible layer to inbetween unit + closest_lay = np.where(topbot_min == 1, bot_lay_geul - 1, top_lay_geul) + + # assign geul to closest layer where it is not yet assigned + mask = np.isnan(lay_geul) & (~np.isnan(top_geul)) + lay_geul = np.where(mask, closest_lay, lay_geul) + + # 4. Modify strat in such a way that the geul is inserted as separate layers. + new_unit_order = unit_order.copy() + layers = np.unique(lay_geul) + layers_abs = np.unique(np.abs(layers))[ + ::-1 + ] # sort absolute values in descending order + layers_ordered = [ + -ilay if (-ilay in layers) else ilay for ilay in layers_abs + ] # use negative value if available + + for geul_lay_count, lay in enumerate(layers_ordered): + if np.isnan(lay): + continue + + geul_subset = geul + (10000 * (geul_lay_count + 1)) + + if lay == 0: # geul is the top layer + logger.debug(f"adding geul {geul} on top of model as {geul_subset}") + new_unit_order = [geul_subset] + new_unit_order + mask1 = np.abs(lay_geul) == np.abs(lay) + mask4 = strat == geul + strat = xr.where(mask4 & mask1, geul_subset, strat) + continue + + ilay = abs(int(lay)) - 1 # correction for dummy layer + unit = unit_order[ilay] + + # Add new geul units to strat and add the geul units to the ordered units. + if lay < 0: # geul is in between a unit + logger.debug(f"geul {geul} below and above unit {unit}") + unit_above = unit + (10000 * (ilay_geul + 1)) + logger.debug(f"split {unit}, part above geul is {unit_above}") + mask2 = strat == unit # 3d mask of where unit is present + mask3 = z > bot_geul # 3d mask of where z value is above geul + strat = xr.where( + (mask2 & mask3), unit_above, strat + ) # add unit above geul + logger.debug( + f"adding geul {geul} below unit {unit_above} as {geul_subset}" + ) + mask1 = np.abs(lay_geul) == np.abs( + lay + ) # 2d mask of where geul can be added to this layer + mask4 = strat == geul # 3d mask of where geul is present + strat = xr.where(mask4 & mask1, geul_subset, strat) # add geul + new_unit_order = ( + new_unit_order[:ilay] + + [unit_above, geul_subset] + + new_unit_order[ilay:] + ) # update order with (part of) unit above geul and geul + else: + # add geul below unit + logger.debug(f"adding geul {geul} below unit {unit} as {geul_subset}") + mask1 = np.abs(lay_geul) == np.abs(lay) + mask4 = strat == geul + strat = xr.where(mask4 & mask1, geul_subset, strat) + new_unit_order = ( + new_unit_order[: ilay + 1] + + [geul_subset] + + new_unit_order[ilay + 1 :] + ) + logger.debug(f"new order of units: {new_unit_order}") + unit_order = new_unit_order.copy() + + return strat, new_unit_order + + @cache.cache_netcdf() def to_model_layers( geotop_ds, @@ -105,7 +415,8 @@ def to_model_layers( the 'geul'. The method "add_as_layer" tries to add the 'geulen' as one or more layers, which can fail if a 'geul' is locally both below the top and above the bottom of another layer (splitting the layer in two, which is not supported). - The default is "add_to_layer_below". + The method "split_layers" splits layers when a 'geul' is both below the top and + above the bottom of another layer. The default is "add_to_layer_below". drop_layer_dim_from_top : bool, optional When True, fill NaN values in top and botm and drop the layer dimension from top. This will transform top and botm to the data model in MODFLOW. An advantage @@ -123,10 +434,9 @@ def to_model_layers( strat_props = get_strat_props() # get all strat-units in Dataset - strat = geotop_ds["strat"].values + strat = geotop_ds["strat"] units = np.unique(strat) units = units[~np.isnan(units)].astype(int) - shape = (len(units), len(geotop_ds.y), len(geotop_ds.x)) if "SEQ_NR" in strat_props.columns: # sort units based on SEQ_NR in strat_props @@ -136,6 +446,17 @@ def to_model_layers( if (2000 in units) and (1130 in units): units[(units == 2000) + (units == 1130)] = [2000, 1130] + if method_geulen == "split_layers": + # remove geulen from units + logger.warning( + "the 'split_layers' method for geulen is still experimental and not " + "yet thoroughly tested." + ) + units_no_geul = [unit for unit in units if unit < 6000] + geulen = [unit for unit in units if unit >= 6000] + strat, units = split_layers_on_geul(strat, units_no_geul, geulen) + shape = (len(units), len(geotop_ds.y), len(geotop_ds.x)) + # fill top and bot top = np.full(shape, np.nan) bot = np.full(shape, np.nan) @@ -147,25 +468,45 @@ def to_model_layers( .repeat(len(geotop_ds.x), 2) ) layers = [] - geulen = [] + geulen = geulen if method_geulen == "split_layers" else [] + uc = np.unique([str(u)[-4:] for u in units], return_counts=True) + split_unit_counter = {int(unit): count for unit, count in zip(*uc, strict=False)} for layer, unit in enumerate(units): - mask = strat == unit - top[layer] = np.nanmax(np.where(mask, z, np.nan), 0) + 0.25 - bot[layer] = np.nanmin(np.where(mask, z, np.nan), 0) - 0.25 + mask = strat.values == unit + # Use finite sentinels to avoid all-NaN slice warnings for empty cells. + maxz = np.max(np.where(mask, z, -np.inf), axis=0) + minz = np.min(np.where(mask, z, np.inf), axis=0) + top[layer] = np.where(np.isfinite(maxz), maxz + 0.25, np.nan) + bot[layer] = np.where(np.isfinite(minz), minz - 0.25, np.nan) if int(unit) in strat_props.index: layers.append(strat_props.at[unit, "code"]) else: - logger.warning(f"Unknown strat-value: {unit}") - layers.append(str(unit)) - if unit >= 6000: + str_unit = str(int(unit)) + if method_geulen == "split_layers" and len(str_unit) > 4: + unit = int(str_unit[-4:]) + if unit in split_unit_counter: + split_unit_counter[unit] -= 1 + else: + logger.warning(f"Unknown strat-value: {str_unit}") + subset = split_unit_counter[unit] + if unit in strat_props.index: + layers.append(f"{strat_props.at[unit, 'code']}_{subset}") + else: + logger.warning(f"Unknown strat-value: {unit}") + layers.append(unit) + else: + logger.warning(f"Unknown strat-value: {unit}") + layers.append(unit) + if unit >= 6000 and method_geulen != "split_layers": geulen.append(layers[-1]) dims = ("layer", "y", "x") coords = {"layer": layers, "y": geotop_ds.y, "x": geotop_ds.x} ds = xr.Dataset({"top": (dims, top), "botm": (dims, bot)}, coords=coords) - if method_geulen is None: + if method_geulen is None or method_geulen == "split_layers": pass + elif method_geulen == "add_as_layer": top = ds["top"].copy(deep=True) bot = ds["botm"].copy(deep=True) @@ -173,6 +514,7 @@ def to_model_layers( ds = remove_layer(ds, geul) for geul in geulen: ds = insert_layer(ds, geul, top.loc[geul], bot.loc[geul]) + elif method_geulen == "add_to_layer_below": top = ds["top"].copy(deep=True) bot = ds["botm"].copy(deep=True) @@ -196,7 +538,8 @@ def to_model_layers( # idomain = get_idomain(ds) # fal = get_last_active_layer_from_idomain(idomain) logger.warning( - f"Geul {geul} is at the bottom of the GeoTOP-dataset in {int(todo.sum())} cells, where it is ignored" + f"Geul {geul} is at the bottom of the GeoTOP-dataset in " + f"{int(todo.sum())} cells, where it is ignored" ) elif method_geulen == "add_to_layer_above": @@ -222,7 +565,8 @@ def to_model_layers( # idomain = get_idomain(ds) # fal = get_first_active_layer_from_idomain(idomain) logger.warning( - f"Geul {geul} is at the top of the GeoTOP-dataset in {int(todo.sum())} cells, where it is ignored" + f"Geul {geul} is at the top of the GeoTOP-dataset in " + f"{int(todo.sum())} cells, where it is ignored" ) else: raise (ValueError(f"Unknown method to deal with geulen: '{method_geulen}'")) @@ -249,9 +593,10 @@ def to_model_layers( def get_geotop(*args, **kwargs): - """Get a slice of the geotop netcdf url within the extent, set the x and y - coordinates to match the cell centers and keep only the strat and lithok data - variables. + """Get a slice of the geotop netcdf url within the extent. + + Set the x and y coordinates to match the cell centers and keep only the strat and + lithok data variables. .. deprecated:: 0.10.0 `get_geotop` will be removed in nlmod 1.0.0, it is replaced by @@ -277,15 +622,17 @@ def get_geotop(*args, **kwargs): "this function is deprecated and will eventually be removed, " "please use nlmod.read.geotop.download_geotop() in the future.", DeprecationWarning, + stacklevel=2, ) return download_geotop(*args, **kwargs) @cache.cache_netcdf() def download_geotop(extent, url=None, probabilities=False, chunks="auto"): - """Get a slice of the geotop netcdf url within the extent, set the x and y - coordinates to match the cell centers and keep only the strat and lithok data - variables. + """Get a slice of the geotop netcdf url within the extent. + + Set the x and y coordinates to match the cell centers and keep only the strat and + lithok data variables. Parameters ---------- @@ -530,13 +877,33 @@ def add_kh_and_kv( kv_ar = kv_ar + (probability / kvi) probability_total += probability if kh_method == "arithmetic_mean": - kh_ar = kh_ar / probability_total + kh_ar = np.divide( + kh_ar, + probability_total, + out=np.full_like(kh_ar, np.nan), + where=probability_total > 0, + ) else: - kh_ar = probability_total / kh_ar + kh_ar = np.divide( + probability_total, + kh_ar, + out=np.full_like(kh_ar, np.nan), + where=kh_ar != 0, + ) if kv_method == "arithmetic_mean": - kv_ar = kv_ar / probability_total + kv_ar = np.divide( + kv_ar, + probability_total, + out=np.full_like(kv_ar, np.nan), + where=probability_total > 0, + ) else: - kv_ar = probability_total / kv_ar + kv_ar = np.divide( + probability_total, + kv_ar, + out=np.full_like(kv_ar, np.nan), + where=kv_ar != 0, + ) else: raise (ValueError(f"Unsupported value for stochastic: '{stochastic}'")) @@ -591,8 +958,9 @@ def _handle_nans_in_stochastic_approach(kh, kv, kh_method, kv_method): def aggregate_to_ds( gt, ds, kh="kh", kv="kv", kd="kD", c="c", kh_gt="kh", kv_gt="kv", add_kd_and_c=False ): - """Aggregate voxels from GeoTOP to layers in a model DataSet with top and botm, to - calculate kh and kv. + """Aggregate voxels from GeoTOP to layers in a model DataSet. + + Uses top and botm, to calculate kh and kv. Parameters ---------- @@ -626,7 +994,8 @@ def aggregate_to_ds( ds : xr.Dataset The Dataset ds, with added variables kh and kv (and optionally kd and c). """ - assert (ds.x == gt.x).all() and (ds.y == gt.y).all() + assert (ds.x == gt.x).all() + assert (ds.y == gt.y).all() msg = "Please add '{}' to geotop-Dataset first, using add_kh_and_kv()" if kh_gt not in gt: raise (MissingValueError(msg.format(kh_gt))) @@ -677,7 +1046,9 @@ def aggregate_to_ds( def _save_excel_files_as_csv(): """ - This method takes the files REF_GTP_STR_UNIT.xlsx and REF_GTP_LITHO_CLASS.xlsx that + Convert REGIS AND GeoTOP Excel files to CSV format. + + Takes the files REF_GTP_STR_UNIT.xlsx and REF_GTP_LITHO_CLASS.xlsx that are taken from the GeoTOP 1.6 zipfile downloaded from DINOloket, and saves them as csv-files. In this way version-control can better process the changes in future versions of GeoTOP. diff --git a/nlmod/read/jarkus.py b/nlmod/read/jarkus.py index 25377d7b..a3d29827 100644 --- a/nlmod/read/jarkus.py +++ b/nlmod/read/jarkus.py @@ -42,7 +42,6 @@ def download_bathymetry(extent, kind="jarkus"): da_bathymetry : xarray.DataArray bathymetry data """ - # try to get bathymetry via opendap jarkus_ds = get_dataset_jarkus(extent=extent, kind=kind) @@ -115,6 +114,7 @@ def get_bathymetry( "error in the future. Use 'nlmod.read.jarkus.download_bathymetry' to get " "the bathymetry data within an extent", DeprecationWarning, + stacklevel=2, ) if extent is None and ds is not None: @@ -188,7 +188,6 @@ def discretize_bathymetry( data is resampled to the modelgrid. Maybe we can speed up things by changing the order in which operations are executed. """ - # fill nan values in bathymetry da_bathymetry_filled = fillnan_da(bathymetry_da) @@ -208,8 +207,10 @@ def discretize_bathymetry( @cache.cache_netcdf() def get_dataset_jarkus(extent, kind="jarkus", return_tiles=False, time=-1): - """Get bathymetry from Jarkus within a certain extent. If return_tiles is False, the - following actions are performed: + """Get bathymetry from Jarkus within a certain extent. + + If return_tiles is False, the following actions are performed: + 1. find Jarkus tiles within the extent 2. download netcdf files of Jarkus tiles 3. read Jarkus tiles and combine the 'z' parameter of the last time step of each diff --git a/nlmod/read/knmi.py b/nlmod/read/knmi.py index 835d797a..9142ba65 100644 --- a/nlmod/read/knmi.py +++ b/nlmod/read/knmi.py @@ -6,9 +6,9 @@ import pandas as pd from .. import cache, util +from ..dims.base import get_ds from ..dims.grid import get_affine_mod_to_world, is_structured, is_vertex from ..dims.layers import get_first_active_layer -from ..dims.base import get_ds from ..dims.shared import get_area from ..dims.time import ds_time_to_pandas_index @@ -92,7 +92,7 @@ def discretize_knmi( to_model_time=True, hourly_precision=None, ): - """discretize knmi data to model grid + """Discretize knmi data to model grid. Create a dataset with recharge (and evaporation) data by following these steps: 1. Check for each cell (structured or vertex) which knmi measurement @@ -123,12 +123,13 @@ def discretize_knmi( When True, only use data from the station that is most common in the model area. The default is True add_stn_dimensions : bool, optional - When True, add the dimension `time` to the variable `recharge` (and `evaporation` - when `method='seperate'`). When True, add dimension `stn_RD` and `stn_EV24` to - the variable `recharge` and `evaporation`, and add variables "recharge_stn" and - "evaporation_stn" that specify for every grid cell which KNMI-stations are used. - When `add_stn_dimensions` is False, specify recharge (and evaporation when - `method='seperate'`) for every gridcell. The default is False. + When True, add the dimension `time` to the variable `recharge` (and + `evaporation` when `method='seperate'`). When True, add dimension `stn_RD` and + `stn_EV24` to the variable `recharge` and `evaporation`, and add variables + "recharge_stn" and "evaporation_stn" that specify for every grid cell which + KNMI-stations are used. When `add_stn_dimensions` is False, specify recharge + (and evaporation when `method='seperate'`) for every gridcell. The default is + False. to_model_time : bool, optional When True, resample the recharge and evaporation to the dimension `time` in ds. When False, save the original times of the KNMI-data in variables `time_RD` and @@ -154,7 +155,6 @@ def discretize_knmi( ValueError if grid is not vertex """ - # check time settings if "time" not in ds: raise ( @@ -200,8 +200,7 @@ def discretize_knmi( nodata = -999 shape = [len(ds_out[dim]) for dim in dims] variables = {"recharge": "stn_RD", "evaporation": "stn_EV24"} - for var in variables: - stn_var = variables[var] + for var, stn_var in variables.items(): ds_out[f"{var}_stn"] = dims, np.full(shape, nodata) values = [int(x.split("_")[-1]) for x in locations[stn_var]] if is_structured(ds): @@ -384,7 +383,7 @@ def _get_locations_structured(ds): # transform coordinates into real-world coordinates affine = get_affine_mod_to_world(ds) x, y = affine * (x, y) - layers = [fal.data[row, col] for row, col in zip(rows, columns)] + layers = [fal.data[row, col] for row, col in zip(rows, columns, strict=False)] hpd = util.import_hydropandas() locations = hpd.ObsCollection( pd.DataFrame( @@ -405,8 +404,7 @@ def download_knmi( end=None, most_common_station=False, ): - """Get precipitation (RD) and evaporation (EV24) data from the knmi at the grid - cells. + """Get precipitation (RD) and evaporation (EV24) data from KNMI at the grid cells. Parameters ---------- @@ -443,16 +441,14 @@ def download_knmi( for obs in oc_knmi["obs"]: msg = f"No data available for time series'{obs.name}'" if obs.empty: - raise (ValueError(msg)) - + raise ValueError(msg) if obs.index[-1] < end: - raise ValueError(f"{msg} untill date {end}") + raise ValueError(f"{msg} until date {end}") return oc_knmi def get_knmi(ds, most_common_station=False, start=None, end=None): - """Get precipitation (RD) and evaporation (EV24) data from the knmi at the grid - cells. + """Get precipitation (RD) and evaporation (EV24) data from KNMI at the grid cells. .. deprecated:: 0.10.0 `get_knmi` will be removed in nlmod 1.0.0, it is replaced by @@ -480,6 +476,7 @@ def get_knmi(ds, most_common_station=False, start=None, end=None): "'get_knmi' is deprecated and will raise an error in the " "future. Use 'nlmod.read.knmi.download_knmi' to get knmi data for your model", DeprecationWarning, + stacklevel=2, ) if start is None: @@ -493,8 +490,9 @@ def get_knmi(ds, most_common_station=False, start=None, end=None): def get_locations(ds, oc_knmi=None, most_common_station=False): - """Get the locations of the active grid cells in ds and the nearest (or most common) - precipitation and evaporation station. + """Get the locations the nearest (or most common) prec and evap stations. + + Uses the active grid cells in ds. Parameters ---------- @@ -556,7 +554,7 @@ def get_locations(ds, oc_knmi=None, most_common_station=False): def _download_knmi_at_locations(locations, start=None, end=None): - """Get precipitation (RD) and evaporation (EV24) data from the knmi at the locations + """Get precipitation (RD) and evaporation (EV24) data from KNMI at the locations. Parameters ---------- @@ -600,7 +598,7 @@ def _download_knmi_at_locations(locations, start=None, end=None): def get_knmi_at_locations(locations, ds=None, start=None, end=None): - """Get precipitation (RD) and evaporation (EV24) data from the knmi at the locations + """Get precipitation (RD) and evaporation (EV24) data from KNMI at the locations. .. deprecated:: 0.10.0 `get_knmi_at_locations` will be removed in nlmod 1.0.0, it is replaced by @@ -628,6 +626,7 @@ def get_knmi_at_locations(locations, ds=None, start=None, end=None): "'get_knmi_at_locations' is deprecated and will raise an error in the " "future. Use 'nlmod.read.knmi._download_knmi_at_locations' to get knmi data", DeprecationWarning, + stacklevel=2, ) if start is None: diff --git a/nlmod/read/knmi_data_platform.py b/nlmod/read/knmi_data_platform.py index 857b388c..5a2305a1 100644 --- a/nlmod/read/knmi_data_platform.py +++ b/nlmod/read/knmi_data_platform.py @@ -11,6 +11,7 @@ import xarray as xr from numpy import arange, array, ndarray from pandas import Timedelta, Timestamp + from ..util import tqdm logger = logging.getLogger(__name__) @@ -101,7 +102,7 @@ def download_file( dataset_name: str, dataset_version: str, fname: str, - dirname: str = ".", + dirname: Union[str, os.PathLike] = ".", api_key: Optional[str] = None, timeout: int = 120, ) -> None: @@ -114,6 +115,7 @@ def download_file( ) r = requests.get(url, headers={"Authorization": api_key}, timeout=timeout) rjson = r.json() + dirname = os.fspath(dirname) if not os.path.isdir(dirname): os.makedirs(dirname) logger.info(f"Download {fname} to {dirname}") @@ -133,7 +135,7 @@ def download_files( dataset_name: str, dataset_version: str, fnames: List[str], - dirname: str = ".", + dirname: Union[str, os.PathLike] = ".", api_key: Optional[str] = None, timeout: int = 120, ) -> None: @@ -260,9 +262,10 @@ def read_grib( def read_dataset_from_zip( - fname: str, hour: Optional[int] = None, **kwargs: dict + fname: Union[str, os.PathLike], hour: Optional[int] = None, **kwargs: dict ) -> xr.Dataset: """Read KNMI data platfrom .zip file to xarray Dataset.""" + fname = os.fspath(fname) if fname.endswith(".zip"): with ZipFile(fname) as zipfo: fnames = sorted([x for x in zipfo.namelist() if not x.endswith("/")]) diff --git a/nlmod/read/lgn.py b/nlmod/read/lgn.py new file mode 100644 index 00000000..97f7eb61 --- /dev/null +++ b/nlmod/read/lgn.py @@ -0,0 +1,408 @@ +# %% +from pathlib import Path +from typing import Literal + +import geopandas as gpd +import matplotlib.colors as mcolors +import matplotlib.patches as mpatches +import numpy as np +import pandas as pd +import xarray as xr + +from nlmod.dims.grid import modelgrid_from_ds +from nlmod.epsg28992 import EPSG_28992 + +# %% + +# Rebuild with QGIS: +# Load LGN.qlr file > Properties > Symbology > Export Color Map to File... +# Run nlmod.read.lgn._load_clr_file() to get this dictionary +LGN_COLOR_DICT = { + 1: {"color": "#73df1f", "rgb": (115, 223, 31), "label": "agrarisch gras"}, + 2: {"color": "#e89919", "rgb": (232, 153, 25), "label": "mais"}, + 3: {"color": "#b26600", "rgb": (178, 102, 0), "label": "aardappelen"}, + 4: {"color": "#e51f7f", "rgb": (229, 31, 127), "label": "bieten"}, + 5: {"color": "#ffff00", "rgb": (255, 255, 0), "label": "granen"}, + 6: {"color": "#ff00c5", "rgb": (255, 0, 197), "label": "overige landbouwgewassen"}, + 8: {"color": "#46ffcf", "rgb": (70, 255, 207), "label": "glastuinbouw"}, + 9: {"color": "#3cef45", "rgb": (60, 239, 69), "label": "boomgaarden"}, + 10: {"color": "#ac81a8", "rgb": (172, 129, 168), "label": "bloembollen"}, + 11: {"color": "#33c800", "rgb": (51, 200, 0), "label": "loofbos"}, + 12: {"color": "#009900", "rgb": (0, 153, 0), "label": "naaldbos"}, + 16: {"color": "#2473ff", "rgb": (36, 115, 255), "label": "zoet water"}, + 17: {"color": "#000099", "rgb": (0, 0, 153), "label": "zout water"}, + 18: { + "color": "#ff0000", + "rgb": (255, 0, 0), + "label": "bebouwing in primair bebouwd gebied", + }, + 19: { + "color": "#730000", + "rgb": (115, 0, 0), + "label": "bebouwing in secundair bebouwd gebied", + }, + 20: { + "color": "#93d600", + "rgb": (147, 214, 0), + "label": "bos in primair bebouwd gebied", + }, + 22: { + "color": "#93aa00", + "rgb": (147, 170, 0), + "label": "bos in secundair bebouwd gebied", + }, + 23: { + "color": "#93ff00", + "rgb": (147, 255, 0), + "label": "gras in primair bebouwd gebied", + }, + 24: { + "color": "#ffff66", + "rgb": (255, 255, 102), + "label": "kale grond in bebouwd gebied", + }, + 26: { + "color": "#761818", + "rgb": (118, 24, 24), + "label": "bebouwing in buitengebied", + }, + 27: { + "color": "#ff645a", + "rgb": (255, 100, 90), + "label": "overig grondgebruik in buitengebied", + }, + 28: { + "color": "#a8ef44", + "rgb": (168, 239, 68), + "label": "gras in secundair bebouwd gebied", + }, + 29: {"color": "#000000", "rgb": (0, 0, 0), "label": "zonneparken"}, + 30: {"color": "#b03060", "rgb": (176, 48, 96), "label": "kwelders"}, + 31: { + "color": "#e6fb00", + "rgb": (230, 251, 0), + "label": "open zand in kustgebied", + }, + 32: { + "color": "#89d42b", + "rgb": (137, 212, 43), + "label": "duinen met lage vegetatie", + }, + 33: { + "color": "#5aba40", + "rgb": (90, 186, 64), + "label": "duinen met hoge vegetatie", + }, + 34: {"color": "#750075", "rgb": (117, 0, 117), "label": "duinheide"}, + 35: { + "color": "#ffff00", + "rgb": (255, 255, 0), + "label": "open stuifzand en/of rivierzand", + }, + 36: {"color": "#750075", "rgb": (117, 0, 117), "label": "heide"}, + 37: {"color": "#a42353", "rgb": (164, 35, 83), "label": "matig vergraste heide"}, + 38: {"color": "#ad8b06", "rgb": (173, 139, 6), "label": "sterk vergraste heide"}, + 39: {"color": "#249996", "rgb": (36, 153, 150), "label": "hoogveen"}, + 40: {"color": "#065a4c", "rgb": (6, 90, 76), "label": "bos in hoogveengebied"}, + 41: { + "color": "#ffc0cb", + "rgb": (255, 192, 203), + "label": "overige moeras vegetatie", + }, + 42: {"color": "#ffa500", "rgb": (255, 165, 0), "label": "rietvegetatie"}, + 43: {"color": "#006400", "rgb": (0, 100, 0), "label": "bos in moerasgebied"}, + 45: { + "color": "#b6b639", + "rgb": (182, 182, 57), + "label": "natuurlijk beheerde agrarische graslanden", + }, + 46: { + "color": "#f5e10f", + "rgb": (245, 225, 15), + "label": "gras in het kustgebied", + }, + 47: {"color": "#969639", "rgb": (150, 150, 57), "label": "overig gras"}, + 61: {"color": "#ffb3a8", "rgb": (255, 179, 168), "label": "boomkwekerijen"}, + 62: {"color": "#e3ff70", "rgb": (227, 255, 112), "label": "fruitkwekerijen"}, + 251: { + "color": "#871b00", + "rgb": (135, 27, 0), + "label": "hoofdinfrastructuur en spoorbaanlichamen", + }, + 252: { + "color": "#b02300", + "rgb": (176, 35, 0), + "label": ( + "halfverharde wegen, infrastructuur langzaam verkeer en overige" + " infrastructuur" + ), + }, + 253: {"color": "#a80000", "rgb": (168, 0, 0), "label": "smalle wegen"}, + 321: { + "color": "#89d42b", + "rgb": (137, 212, 43), + "label": "struikvegetatie in hoogveengebied (laag)", + }, + 322: { + "color": "#89d42b", + "rgb": (137, 212, 43), + "label": "struikvegetatie in moerasgebied (laag)", + }, + 323: { + "color": "#89d42b", + "rgb": (137, 212, 43), + "label": "overige struikvegetatie (laag)", + }, + 331: { + "color": "#5aba40", + "rgb": (90, 186, 64), + "label": "struikvegetatie in hoogveengebied (hoog)", + }, + 332: { + "color": "#5aba40", + "rgb": (90, 186, 64), + "label": "struikvegetatie in moerasgebied (hoog)", + }, + 333: { + "color": "#5aba40", + "rgb": (90, 186, 64), + "label": "overige struikvegetatie (hoog)", + }, +} + + +def _load_clr_file(file_path): + """Parses a QGIS .clr file into a Python dictionary and Matplotlib colormap.""" + lgn_color_dict = {} + + with open(file_path, "r") as f: + for line in f: + line = line.strip() + if not line: + continue + + # Split line at '-' to separate numerical data from the description label + parts = line.split(" - ") + label = parts[1].strip() if len(parts) > 1 else "" + + # Parse numbers: [Code, Red, Green, Blue, Alpha, Duplicate_Code] + num_parts = parts[0].split() + if len(num_parts) >= 4: + code = int(num_parts[0]) + r, g, b = int(num_parts[1]), int(num_parts[2]), int(num_parts[3]) + + # Convert RGB to HEX format (#RRGGBB) + hex_color = f"#{r:02x}{g:02x}{b:02x}" + + lgn_color_dict[code] = { + "color": hex_color, + "rgb": (r, g, b), + "label": label, + } + + return lgn_color_dict + + +def remap_lgn(lgn: xr.DataArray, mappings: dict, fill_value=-999): + """Remap LGN values to a simplified set of values. + + Parameters + ---------- + lgn : xr.DataArray + LGN data array with original values. + mappings : dict + Dictionary mapping original LGN values to new values. + fill_value : int, optional + Value to use for unmapped LGN values. Default is -999. + + Returns + ------- + xr.DataArray + Remapped LGN data array. + """ + # Fast path for integer codes: use lookup array + max_code = int(max(mappings)) + lut = np.full(max_code + 1, fill_value, dtype=np.int32) + for c, v in mappings.items(): + lut[c] = v + + arr = lgn.values.astype(np.int64, copy=False) + out = np.full(arr.shape, fill_value, dtype=np.int32) + + valid = (arr >= 0) & (arr <= max_code) + out[valid] = lut[arr[valid]] + return xr.DataArray( + out, + coords=lgn.coords, + dims=lgn.dims, + name="lgn_simplified", + attrs=lgn.attrs, + ) + + +def download_lgn(): + """Download LGN data from the LGN website.""" + print("Download LGN data from: https://lgn.nl/bestanden") + + +def load_lgn_within_extent(lgn_tif_path: str | Path, extent: list): + """Load LGN tif file and subset to the given extent. + + Parameters + ---------- + lgn_tif_path : str or Path + Path to the LGN tiff file. + extent : list + list of [xmin, xmax, ymin, ymax] + + Returns + ------- + xr.DataArray + LGN data array. + """ + lgn = xr.open_dataset(lgn_tif_path, engine="rasterio") + lgnsel = lgn.sel(x=slice(extent[0], extent[1]), y=slice(extent[3], extent[2]))[ + "band_data" + ] + return lgnsel.sel(band=1, drop=True) + + +def get_lgn_cmap_norm(lgn_color_dict=None): + """Builds a ListedColormap and BoundaryNorm for LGN. + + Parameters + ---------- + lgn_color_dict : dict, optional + Dictionary mapping LGN codes to color and label information. Defaults to + predefined LGN_COLOR_DICT derived from default QGIS symbology. + + Returns + ------- + cmap : matplotlib.colors.ListedColormap + Colormap for LGN codes. + norm : matplotlib.colors.BoundaryNorm + Normalization for LGN codes. + """ + if lgn_color_dict is None: + lgn_color_dict = LGN_COLOR_DICT + + max_code = max(lgn_color_dict.keys()) + + # Default unmapped values to fully transparent white (#FFFFFF00) + colors = ["#FFFFFF00"] * (max_code + 1) + + for code, data in lgn_color_dict.items(): + colors[code] = data["color"] + + cmap = mcolors.ListedColormap(colors) + + # Boundaries centered around integer raster codes + bounds = np.arange(-0.5, max_code + 1.5, 1) + norm = mcolors.BoundaryNorm(bounds, cmap.N) + + return cmap, norm + + +def get_lgn_legend(lgn: xr.DataArray): + """Get a legend for the LGN data array. + + Parameters + ---------- + lgn : xr.DataArray + LGN data array. + + Returns + ------- + list + List of matplotlib.patches.Patch objects for the legend. + + Example + ------- + Use to build matplotlib legend:: + + lgn = get_lgn(ds) + f, ax = plt.subplots() + lgn.plot(ax=ax) + legend_patches = get_lgn_legend(lgn) + ax.legend(handles=legend_patches, loc=(1, 0), fontsize="small") + + """ + return [ + mpatches.Patch(color=info["color"], label=f"{code}: {info['label']}") + for code, info in LGN_COLOR_DICT.items() + if code in np.unique(lgn) + ] + + +def lgn_to_grid( + lgn: xr.DataArray, + ds: xr.Dataset, + engine: Literal["xrspatial", "exactextract"] = "xrspatial", + crs: str = EPSG_28992, +) -> gpd.GeoDataFrame: + """Compute area fractions per model cell from LGN. + + Requires xrspatial or exactextract to be installed. Install with + `pip install nlmod[lgn]`. + + Parameters + ---------- + lgn : xr.DataArray + LGN raster data array. + ds : xr.Dataset + Model dataset. + engine : str, optional + Engine to use for mapping. Options are "xrspatial" or "exactextract". + Defaults to "xrspatial". `xrspatial.zonal_crosstab` is faster but only counts + pixels, while `exact_extract.exactextract` calculates area fractions. + + Returns + ------- + gpd.GeoDataFrame + lgn area fractions per model cell + """ + grid = modelgrid_from_ds(ds).to_geodataframe() + grid = grid.set_crs(crs) + + if engine == "xrspatial": + import xrspatial as xrs + + lgn_grid = ( + xrs.zonal_crosstab(grid, lgn, column="node").iloc[:, 1:].astype(float) + ) # drop node/zone column + lgn_grid /= lgn_grid.sum(axis=1) # convert counts to area + lgn_grid.columns = lgn_grid.columns.astype(int) + lgn_grid.columns.name = "LGN_code" + lgn_grid.index.name = "icell2d" + lgn_grid = gpd.GeoDataFrame(lgn_grid, geometry=grid["geometry"]) + + elif engine == "exactextract": + from exactextract import exact_extract + + lgn_grid = exact_extract( + lgn, + grid, + ["unique", "frac"], + include_cols=["node"], + include_geom=True, + strategy="raster-sequential", # faster + output="pandas", + ) + # unstack unique LGN codes into columns and fill in fracs + lgn_grid_dict = {} + for node, row in lgn_grid.iterrows(): + lgn_grid_dict[node] = { + int(code): frac + for code, frac in zip(row["unique"], row["frac"], strict=True) + } + lgn_grid = gpd.GeoDataFrame( + pd.DataFrame(lgn_grid_dict).T.sort_index(axis=1), + geometry=grid["geometry"], + ) + lgn_grid.columns.name = "LGN_code" + lgn_grid.index.name = "icell2d" + + else: + raise ValueError(f"Unknown engine: {engine}") + + return lgn_grid diff --git a/nlmod/read/meteobase.py b/nlmod/read/meteobase.py index 5b3a9e11..5543562a 100644 --- a/nlmod/read/meteobase.py +++ b/nlmod/read/meteobase.py @@ -11,7 +11,7 @@ class MeteobaseType(Enum): - """Enum class to couple folder names to observation type (from in LEESMIJ.txt)""" + """Enum class to couple folder names to observation type (from in LEESMIJ.txt).""" NEERSLAG = "Neerslagradargegevens in Arc/Info-formaat." MAKKINK = "Verdampingsgegevens volgens Makkink." @@ -55,7 +55,13 @@ def read_leesmij(fo: FileIO) -> Dict[str, Dict[str, str]]: def get_timestamp_from_fname(fname: str) -> Timestamp: - """Get the Timestamp from a filename (with some assumptions about the formatting)""" + """Get the Timestamp from a filename (with some assumptions about the formatting). + + Parameters + ---------- + fname : str + Filename with date and hour in the format YYYYMMDD_HH + """ datestr = re.search("([0-9]{8})", fname) # assumes YYYYMMDD if datestr is not None: match = datestr.group(0) diff --git a/nlmod/read/nhi.py b/nlmod/read/nhi.py index 67a5e6f9..0671598f 100644 --- a/nlmod/read/nhi.py +++ b/nlmod/read/nhi.py @@ -86,6 +86,7 @@ def add_buisdrainage( depth_var="buisdrain_depth", cond_method="average", depth_method="mode", + fix_crs=None, ): """Add data about the buisdrainage to the model Dataset. @@ -114,6 +115,11 @@ def add_buisdrainage( depth_method : str, optional The method to transform the depth of buisdrainage to the model Dataset. The default is "mode". + fix_crs : bool, optional + Fix the coordinate shift (~100 m to the northeast) in the original NHI + buisdrainage dataset caused by a missing datum transformation (Bessel 1841 to + WGS84) in the source dataset. If None, a warning is issued and fix_crs + defaults to False to preserve backwards compatibility. The default is None. Returns ------- @@ -121,20 +127,21 @@ def add_buisdrainage( The model dataset with added variables with the names `cond_var` and `depth_var`. """ - warnings.warn( "'add_buisdrainage' is deprecated and will be removed in a future version. " "Use 'ds.update(nlmod.read.nhi.discretize_buisdrainage(ds))' to project the " "buisdrainage on the model grid", DeprecationWarning, + stacklevel=2, ) ds_out = discretize_buisdrainage( ds, - pathname, - cond_var, - depth_var, - cond_method, - depth_method, + pathname=pathname, + cond_var=cond_var, + depth_var=depth_var, + cond_method=cond_method, + depth_method=depth_method, + fix_crs=fix_crs, ) ds.update(ds_out) return ds @@ -147,6 +154,7 @@ def discretize_buisdrainage( depth_var="buisdrain_depth", cond_method="average", depth_method="mode", + fix_crs=None, ): """Add data about the buisdrainage to the model Dataset. @@ -180,6 +188,12 @@ def discretize_buisdrainage( depth_method : str, optional The method to transform the depth of buisdrainage to the model Dataset. The default is "mode". + fix_crs : bool, optional + Fix the coordinate shift (~100 m to the northeast) in the original NHI + buisdrainage dataset caused by a missing datum transformation (Bessel 1841 to + WGS84) in the source dataset. If None, a warning is issued and fix_crs + defaults to False to preserve backwards compatibility. For more information, see + https://github.com/gwmod/nlmod/pull/598. The default is None. Returns ------- @@ -187,6 +201,15 @@ def discretize_buisdrainage( The model dataset with added variables with the names `cond_var` and `depth_var`. """ + if fix_crs is None: + logger.warning( + "`fix_crs` was not specified and defaults to False. Set `fix_crs=True` to " + "fix the ~100 m coordinate shift (to the northeast) in the original NHI " + "buisdrainage dataset caused by a missing datum transformation in the " + "source dataset." + ) + fix_crs = False + if pathname is None: pathname = ds.cachedir # download files if needed @@ -198,10 +221,17 @@ def discretize_buisdrainage( ds_out = get_ds_empty(ds, keep_coords=("y", "x")) + src_crs = ( + "+proj=longlat +ellps=bessel " + "+towgs84=565.417,50.3319,465.552,-0.398957,0.343988,-1.8774,4.0725 +no_defs" + ) + # use cond_methd for conductance # (default is "average" to account for locations without pipe drainage, where the # conductance is 0) buisdrain_c = rioxarray.open_rasterio(fname_c, mask_and_scale=True)[0] + if fix_crs: + buisdrain_c = buisdrain_c.rio.write_crs(src_crs) # calculate a conductance (per m2) from a resistance cond = 1 / buisdrain_c # set conductance to 0 where resistance is infinite or 0 @@ -216,6 +246,8 @@ def discretize_buisdrainage( # (default is "mode" for depth that occurs most in each cell) mask_and_scale = False buisdrain_d = rioxarray.open_rasterio(fname_d, mask_and_scale=mask_and_scale)[0] + if fix_crs: + buisdrain_d = buisdrain_d.rio.write_crs(src_crs) if mask_and_scale: nodata = np.nan else: @@ -295,6 +327,7 @@ def get_gwo_wells( "'get_gwo_wells' is deprecated and will be removed in a future version. " "Use 'nlmod.read.nhi.download_gwo_wells' instead", DeprecationWarning, + stacklevel=2, ) return download_gwo_wells( @@ -461,6 +494,7 @@ def get_gwo_measurements( "'get_gwo_measurements' is deprecated and will be removed in a future version. " "Use 'nlmod.read.nhi.download_gwo_measurements' instead", DeprecationWarning, + stacklevel=2, ) return download_gwo_measurements( diff --git a/nlmod/read/regis.py b/nlmod/read/regis.py index 25f841dc..9f58ec84 100644 --- a/nlmod/read/regis.py +++ b/nlmod/read/regis.py @@ -145,19 +145,19 @@ def get_regis( -9999. rename_layers_to_version_2_2_2 : bool, toptional From version 2.2.3 of regis, the names of stratigraphic layers change, compared - to previous versions. If rename_layers_to_version_2_2_2 is True, the layer-names are - renamed back to their original names. The default is True. + to previous versions. If rename_layers_to_version_2_2_2 is True, the layer-names + are renamed back to their original names. The default is True. Returns ------- regis_ds : xarray dataset dataset with regis data projected on the modelgrid. """ - warnings.warn( "'get_regis' is deprecated and will eventually be removed, " "please use 'nlmod.read.regis.download_regis()' in the future.", DeprecationWarning, + stacklevel=2, ) return download_regis( @@ -213,8 +213,8 @@ def download_regis( -9999. rename_layers_to_version_2_2_2 : bool, toptional From version 2.2.3 of regis, the names of stratigraphic layers change, compared - to previous versions. If rename_layers_to_version_2_2_2 is True, the layer-names are - renamed back to their original names. The default is True. + to previous versions. If rename_layers_to_version_2_2_2 is True, the layer-names + are renamed back to their original names. The default is True. Returns ------- @@ -235,7 +235,10 @@ def download_regis( ds = ds.sel(x=slice(extent[0], extent[1]), y=slice(extent[2], extent[3])) if len(ds.x) == 0 or len(ds.y) == 0: - msg = "No data found. Please supply valid extent in the Netherlands in RD-coordinates" + msg = ( + "No data found. Please supply valid extent in the Netherlands in " + "RD-coordinates" + ) raise (ValueError(msg)) ds["layer"] = get_layer_names( @@ -281,7 +284,10 @@ def download_regis( ds = ds.sel(layer=mask_layer) if len(ds.layer) == 0: - msg = "No data found. Please supply valid extent in the Netherlands in RD-coordinates" + msg = ( + "No data found. Please supply valid extent in the Netherlands " + "in RD-coordinates" + ) raise (Exception(msg)) if drop_layer_dim_from_top and "botm" in ds and "top" in ds: @@ -317,8 +323,10 @@ def add_geotop_to_regis_layers( anisotropy=1.0, gt_layered=None, ): - """Combine geotop and regis in such a way that the one or more layers in Regis are - replaced by the geo_eenheden of geotop. + """Combine geotop and regis. + + In such a way that the one or more layers in Regis are replaced by the geo_eenheden + of geotop. Parameters ---------- @@ -353,12 +361,14 @@ def add_geotop_to_regis_layers( if "kh" not in gt or "kv" not in gt: if "kv" in gt: logger.info( - f"Calculating kh of geotop by multiplying kv with an anisotropy of {anisotropy}" + "Calculating kh of geotop by multiplying kv with an " + f"anisotropy of {anisotropy}" ) gt["kh"] = gt["kv"] * anisotropy elif "kh" in gt: logger.info( - f"Calculating kv of geotop by dividing kh by an anisotropy of {anisotropy}" + "Calculating kv of geotop by dividing kh by an " + f"anisotropy of {anisotropy}" ) gt["kv"] = gt["kh"] / anisotropy else: @@ -370,7 +380,10 @@ def add_geotop_to_regis_layers( # copy the regis-dataset, before altering it rg = rg.copy(deep=True) if "layer" in rg["top"].dims: - msg = "Top in rg has a layer dimension. add_geotop_to_regis_layers will remove the layer dimension from top in rg." + msg = ( + "Top in rg has a layer dimension. add_geotop_to_regis_layers will " + "remove the layer dimension from top in rg." + ) logger.warning(msg) else: # temporarily add layer dimension to top in rg @@ -400,7 +413,8 @@ def add_geotop_to_regis_layers( gtl = gtl.sel(layer=(th > 0).any(th.dims[1:])) # add kh and kv from gt to gtl - gtl = geotop.aggregate_to_ds(gt, gtl) + if "kh" not in gtl or "kv" not in gtl: + gtl = geotop.aggregate_to_ds(gt, gtl) # add gtl-layers to rg-layers lay = np.where(rg.layer == layer)[0][0] @@ -512,6 +526,7 @@ def get_legend(kind="REGIS"): def get_legend_lithoclass(): + """Get a legend (DataFrame) with the lithoclasses of GeoTOP layers.""" dir_path = os.path.dirname(os.path.realpath(__file__)) fname = os.path.join(dir_path, "..", "data", "geotop", "Lithoklasse.voleg") leg = read_voleg(fname) @@ -519,6 +534,7 @@ def get_legend_lithoclass(): def get_legend_lithostratigraphy(): + """Get a legend (DataFrame) with the lithostratigraphy of GeoTOP layers.""" dir_path = os.path.dirname(os.path.realpath(__file__)) fname = os.path.join(dir_path, "..", "data", "geotop", "Lithostratigrafie.voleg") leg = read_voleg(fname) @@ -526,14 +542,16 @@ def get_legend_lithostratigraphy(): def read_gleg(fname): + """Read a gleg file and return a DataFrame with the legend.""" leg = pd.read_csv( fname, sep="\t", header=None, names=["naam", "beschrijving", "r", "g", "b", "a", "x"], + na_filter=False, ) leg["naam"] = leg["naam"].str.replace("-", "") - leg.set_index("naam", inplace=True) + leg = leg.set_index("naam") clrs = np.array(leg.loc[:, ["r", "g", "b"]]) clrs = [tuple(rgb / 255.0) for rgb in clrs] leg["color"] = clrs @@ -542,13 +560,14 @@ def read_gleg(fname): def read_voleg(fname): + """Read a voleg file and return a DataFrame with the legend.""" leg = pd.read_csv( fname, sep="\t", header=None, names=["code", "naam", "r", "g", "b", "a", "beschrijving"], ) - leg.set_index("code", inplace=True) + leg = leg.set_index("code") clrs = np.array(leg.loc[:, ["r", "g", "b"]]) clrs = [tuple(rgb / 255.0) for rgb in clrs] leg["color"] = clrs diff --git a/nlmod/read/rws.py b/nlmod/read/rws.py index 87aa1f8b..62e8b918 100644 --- a/nlmod/read/rws.py +++ b/nlmod/read/rws.py @@ -7,6 +7,7 @@ import geopandas as gpd import numpy as np import xarray as xr +from rasterio.env import Env from rioxarray.merge import merge_arrays from .. import NLMOD_DATADIR, cache, dims, util @@ -18,8 +19,7 @@ @cache.cache_pickle def get_gdf_surface_water(ds=None, extent=None): - """Read a shapefile with surface water as a geodataframe, cut by the extent of the - model. + """Read a shapefile with surface water as a geodataframe, cut by model extent. Parameters ---------- @@ -54,7 +54,7 @@ def get_gdf_surface_water(ds=None, extent=None): @cache.cache_netcdf(coords_3d=True) def get_surface_water(ds, gdf=None, da_basename="rws_oppwater"): - """Create 3 data-arrays from the shapefile with surface water: + """Create 3 data-arrays from the shapefile with surface water. .. deprecated:: 0.10.0 `get_surface_water` will be removed in nlmod 1.0.0, it is replaced by @@ -81,11 +81,12 @@ def get_surface_water(ds, gdf=None, da_basename="rws_oppwater"): ds : xarray.Dataset dataset with modelgrid data. """ - warnings.warn( "'get_surface_water' is deprecated and will be removed in a future version. " - "Use 'nlmod.read.rws.discretize_surface_water' to project the surface water on the model grid", + "Use 'nlmod.read.rws.discretize_surface_water' to project the surface water " + "on the model grid", DeprecationWarning, + stacklevel=2, ) if gdf is None: @@ -178,8 +179,10 @@ def get_northsea(ds, gdf=None, da_name="northsea"): """ warnings.warn( "'get_northsea' is deprecated and will be removed in a future version. " - "Use 'nlmod.read.rws.discretize_northsea' to project the northsea on the model grid", + "Use 'nlmod.read.rws.discretize_northsea' to project the northsea on the " + "model grid", DeprecationWarning, + stacklevel=2, ) return discretize_northsea(ds, gdf, da_name) @@ -338,9 +341,9 @@ def get_gdr_configuration() -> dict: config["bodemhoogte_1m"] = { "url": ( "https://geo.rijkswaterstaat.nl/arcgis/rest/services/GDR/" - "bodemhoogte_index/FeatureServer" + "bodemhoogte_index/MapServer" ), - "layer": 1, + "layer": 1, # bodemhoogte_1mtr } # NOTE: the 20m resolution is no longer available from the GDR service via a # geodataframe containing the url. @@ -381,8 +384,10 @@ def get_bathymetry_gdf( """ warnings.warn( "'get_bathymetry_gdf' is deprecated and will be removed in a future version. " - "Use 'nlmod.read.rws.download_bathymetry_gdf' to project the buisdrainage on the model grid", + "Use 'nlmod.read.rws.download_bathymetry_gdf' to project the buisdrainage " + "on the model grid", DeprecationWarning, + stacklevel=2, ) return download_bathymetry_gdf(resolution, extent, config) @@ -472,6 +477,7 @@ def get_bathymetry( "'get_bathymetry' is deprecated and will be removed in a future version. " "Use 'nlmod.read.rws.download_bathymetry' to download bathymetry data", DeprecationWarning, + stacklevel=2, ) return download_bathymetry(extent, resolution, res, method, chunks, config) @@ -524,34 +530,38 @@ def download_bathymetry( xmin, xmax, ymin, ymax = extent dataarrays = [] - for _, row in tqdm( - gdf.iterrows(), desc="Downloading bathymetry", total=gdf.index.size - ): - url = row["geotiff"] - ds = xr.open_dataset(url, engine="rasterio") - ds = ds.assign_coords({"y": ds["y"].round(0), "x": ds["x"].round(0)}) - da = ( - ds["band_data"] - .sel(band=1, x=slice(xmin, xmax), y=slice(ymax, ymin)) - .drop_vars("band") - ) - if chunks: - da = da.chunk(chunks) - dataarrays.append(da) - - if len(dataarrays) > 1: - da = merge_arrays( - dataarrays, - bounds=[xmin, ymin, xmax, ymax], - res=res, - method=method, - ) - else: - da = dataarrays[0] - if res is not None: - da = da.rio.reproject( - da.rio.crs, + rasterio_env = { + "GDAL_DISABLE_READDIR_ON_OPEN": "YES", + } + with Env(**rasterio_env): + for _, row in tqdm( + gdf.iterrows(), desc="Downloading bathymetry", total=gdf.index.size + ): + url = row["geotiff"] + ds = xr.open_dataset(url, engine="rasterio") + ds = ds.assign_coords({"y": ds["y"].round(0), "x": ds["x"].round(0)}) + da = ( + ds["band_data"] + .sel(band=1, x=slice(xmin, xmax), y=slice(ymax, ymin)) + .drop_vars("band") + ) + if chunks: + da = da.chunk(chunks) + dataarrays.append(da) + + if len(dataarrays) > 1: + da = merge_arrays( + dataarrays, + bounds=[xmin, ymin, xmax, ymax], res=res, - resampling=method, + method=method, ) + else: + da = dataarrays[0] + if res is not None: + da = da.rio.reproject( + da.rio.crs, + res=res, + resampling=method, + ) return da diff --git a/nlmod/read/waterboard.py b/nlmod/read/waterboard.py index 63aadc97..5cebb856 100644 --- a/nlmod/read/waterboard.py +++ b/nlmod/read/waterboard.py @@ -18,11 +18,11 @@ def get_polygons(**kwargs): https://github.com/gwmod/nlmod/issues/47 """ - warnings.warn( "get_polygons is deprecated and will eventually be removed, " "please use nlmod.read.waterboard.download_polygons() in the future.", DeprecationWarning, + stacklevel=2, ) return download_polygons(**kwargs) @@ -42,8 +42,16 @@ def download_polygons(**kwargs): ws["waterschap"] = ws["waterschap"].str.replace("HH van ", "") ws["waterschap"] = ws["waterschap"].str.replace("HHS van ", "") ws["waterschap"] = ws["waterschap"].str.replace("HH ", "") + ws["waterschap"] = ws["waterschap"].str.replace("Hoogheemraadschap van ", "") + ws["waterschap"] = ws["waterschap"].str.replace("Hoogheemraadschap ", "") + ws["waterschap"] = ws["waterschap"].str.replace("Administratieve grens ", "") + ws["waterschap"] = ws["waterschap"].str.replace(" 2018", "") + ws["waterschap"] = ws["waterschap"].str.replace(" (INSPIRE-grens)", "") + ws["waterschap"] = ws["waterschap"].str.replace("Beheergebied waterschap ", "") + ws["waterschap"] = ws["waterschap"].str.replace("WS ", "") ws["waterschap"] = ws["waterschap"].str.replace("Waterschap ", "") ws["waterschap"] = ws["waterschap"].str.replace("Wetterskip ", "") + ws = ws.set_index("waterschap") return ws @@ -97,19 +105,23 @@ def get_configuration(): "index": "OVKIDENT", }, "level_areas": { - "url": "https://maps.waternet.nl/arcgis/rest/services/Publiek/GW_GPG/MapServer", - "layer": 5, # Vigerende peilgebieden - "index": "GPGIDENT", - "summer_stage": [ - "GPGZMRPL", - "IWS_BOVENGRENS_ZOMERPEIL", - "IWS_GPGVASTP", - ], - "winter_stage": [ - "GPGWNTPL", - "OPVAFWWP", - "IWS_GPGVASTP", - ], + "url": "https://services-eu1.arcgis.com/nN7Fv7YeT1qPGzBX/arcgis/rest/services/DAMO_BEHEERREGISTER_PEILGEBIEDEN_VIGEREND/FeatureServer", + "layer": 0, # Peilgebied vigerend + "index": "code", + # "summer_stage": [ + # "GPGZMRPL", + # "IWS_BOVENGRENS_ZOMERPEIL", + # "IWS_GPGVASTP", + # ], + # "winter_stage": [ + # "GPGWNTPL", + # "OPVAFWWP", + # "IWS_GPGVASTP", + # ], + }, + "culverts": { + "url": "https://services-eu1.arcgis.com/nN7Fv7YeT1qPGzBX/ArcGIS/rest/services/agol_legger_duikersifonhevel_actueel/FeatureServer", + "layer": 2, }, } @@ -170,16 +182,20 @@ def get_configuration(): "index": "Nieuw_Dommel_ID", }, "culverts": { - "url": "https://services8.arcgis.com/dmR647kStmcYa6EN/arcgis/rest/services/LWW_2023_Duiker_V/FeatureServer" + "url": "https://services8.arcgis.com/dmR647kStmcYa6EN/arcgis/rest/services/LWW_2023_Duiker_V/FeatureServer", + "index": "Lokaal_ID", }, "weirs": { - "url": "https://services8.arcgis.com/dmR647kStmcYa6EN/arcgis/rest/services/LWW_2023_Stuw_V/FeatureServer" + "url": "https://services8.arcgis.com/dmR647kStmcYa6EN/arcgis/rest/services/LWW_2023_Stuw_V/FeatureServer", + "index": "Lokaal_ID", }, "profile_lines": { - "url": "https://services8.arcgis.com/dmR647kStmcYa6EN/ArcGIS/rest/services/LWW_2023_A_water_profiellijn_V/FeatureServer" + "url": "https://services8.arcgis.com/dmR647kStmcYa6EN/ArcGIS/rest/services/LWW_2023_A_water_profiellijn_V/FeatureServer", + "index": "Dommel_ID", }, "profile_points": { "url": "https://services8.arcgis.com/dmR647kStmcYa6EN/ArcGIS/rest/services/LWW_2023_A_water_profielpunt_V/FeatureServer", + "index": "Dommel_ID", }, } @@ -234,27 +250,27 @@ def get_configuration(): config["Fryslân"] = { "bgt_code": "W0653", "watercourses": { - "url": "https://gis.wetterskipfryslan.nl/arcgis/rest/services/BeheerregisterWaterlopen/MapServer", - "layer": 0, # # Wateren (primair, secundair) - "index": "OVKIDENT", - "bottom_height": "AVVBODH", - "water_depth": "AVVDIEPT", - # "url": "https://gis.wetterskipfryslan.nl/arcgis/rest/services/Legger_vastgesteld__2019/MapServer", - # "layer": 604, # Wateren legger - # "index": "BLAEU.LEG_VL_GW_OVK.OVKIDENT", - # "bottom_height": "BLAEU.LEG_VL_GW_OVK.AVVBODH", + "url": "https://services1.arcgis.com/I0drYBaRsGebHGKb/ArcGIS/rest/services/Watersysteem_Wetterskip_Frysl%C3%A2n/FeatureServer", + "layer": 11, # Waterloop + "index": "Code", + "bottom_height": "Bodemhoogte_legger_mNAP", }, "level_areas": { - # "url": "https://gis.wetterskipfryslan.nl/arcgis/rest/services/Peilbelsuit_Friese_boezem/MapServer", - # "index": "BLAEU_WFG_GPG_BEHEER_PBHIDENT", - "url": "https://gis.wetterskipfryslan.nl/arcgis/rest/services/Peilen/MapServer", - # "layer": 1, # PeilenPeilenbeheerkaart - Peilen - "layer": 4, # Peilgebied praktijk - "index": "PBHIDENT", - # "layer": 4, # Peilbesluitenkaart - # "index": "GPGIDENT", - "summer_stage": "HOOGPEIL", - "winter_stage": "LAAGPEIL", + "url": "https://services1.arcgis.com/I0drYBaRsGebHGKb/ArcGIS/rest/services/Watersysteem_Wetterskip_Frysl%C3%A2n/FeatureServer", + "layer": 5, # Peilen_actueel + "index": "Code", + "summer_stage": "Hoogpeil_mNAP", + "winter_stage": "Laagpeil_mNAP", + }, + "weirs": { + "url": "https://services1.arcgis.com/I0drYBaRsGebHGKb/ArcGIS/rest/services/Watersysteem_Wetterskip_Frysl%C3%A2n/FeatureServer", + "layer": 7, + "index": "Code", + }, + "culverts": { + "url": "https://services1.arcgis.com/I0drYBaRsGebHGKb/ArcGIS/rest/services/Watersysteem_Wetterskip_Frysl%C3%A2n/FeatureServer", + "layer": 2, + "index": "Code", }, } @@ -301,17 +317,21 @@ def get_configuration(): "bgt_code": "W0655", "watercourses": { # "url": "https://geoportaal.wshd.nl/arcgis/rest/services/Geoportaal/Legger2014waterbeheersing_F_transparant/FeatureServer", - "url": "https://geoportaal.wshd.nl/arcgis/rest/services/Watersysteem/Watersysteem/MapServer", - "layer": 15, # Waterlopen + # "url": "https://geoportaal.wshd.nl/arcgis/rest/services/Watersysteem/Watersysteem/MapServer", + # "layer": 15, # Waterlopen + "url": "https://services1.arcgis.com/kuOZuNSbAumsw8wc/arcgis/rest/services/WSHD_HydroObject_(obv_type)_(extern)/FeatureServer", + "layer": 0, "bottom_height": [ ["BODEMHOOGTE_BOVENSTROOMS", "BODEMHOOGTE_BENEDENSTROOMS"] ], "bottom_width": "BODEMBREEDTE", }, "level_areas": { - "url": "https://geoportaal.wshd.nl/arcgis/rest/services/Watersysteem/Peilgebieden/MapServer", - "layer": 31, - "f": "json", + # "url": "https://geoportaal.wshd.nl/arcgis/rest/services/Watersysteem/Peilgebieden/MapServer", + # "layer": 31, + # "f": "json", + "url": "https://services1.arcgis.com/kuOZuNSbAumsw8wc/arcgis/rest/services/WSHD_Peilgebied_vigerend_(extern)/FeatureServer", + "layer": 3, "summer_stage": [ "REKENPEIL_ZOMER", "BOVENGRENS_BEHEERMARGE_ZOMER", @@ -323,6 +343,14 @@ def get_configuration(): "ONDERGRENS_BEHEERMARGE_WINTER", ], }, + "weirs": { + "url": "https://services1.arcgis.com/kuOZuNSbAumsw8wc/arcgis/rest/services/WSHD_Stuw_(extern)_/FeatureServer", + "layer": 1, + }, + "culverts": { + "url": "https://services1.arcgis.com/kuOZuNSbAumsw8wc/arcgis/rest/services/WSHD_Duiker_Sifon_Hevel__Persleiding__lijn_extern/FeatureServer", + "layer": 3, + }, } config["Hunze en Aa's"] = { @@ -361,18 +389,15 @@ def get_configuration(): config["Noorderzijlvest"] = { "bgt_code": "W0647", "watercourses": { - # "url": "https://arcgis.noorderzijlvest.nl/server/rest/services/Waterkwantiteit/Watergang/MapServer", - # "layer": 0, # Primair + "url": "https://geo.noorderzijlvest.nl/server/rest/services/Waterkwantiteit/Watergang/MapServer", + "layer": 0, # Primair # "layer": 1, # Secundair # "layer": 2, # Tertiair - "url": "https://services2.arcgis.com/ppcrd5oBNPEvV1T3/ArcGIS/rest/services/Normatieve_afmetingen/FeatureServer", - "layer": 1, # Normatieve afmetingen - "bottom_height": [["IWS_AVVHOBOS_L", "IWS_AVVHOBES_L"]], - "bottom_width": "AVVBODDR", - "index": "OVKIDENT", + "bottom_width": "DAMO_W.HydroObject.BREEDTE", + "index": "OBJECTID", }, "level_areas": { - "url": "https://arcgis.noorderzijlvest.nl/server/rest/services/Peilbeheer/Peilgebied/MapServer/", + "url": "https://geo.noorderzijlvest.nl/server/rest/services/Peilbeheer/Peilgebied/MapServer", "layer": 3, "index": "WS_CODE", "summer_stage": [ @@ -452,12 +477,12 @@ def get_configuration(): # "layer": 70, # Profiellijnen }, "weirs": { - "url": "https://kaarten.wsrl.nl/arcgis/rest/services/Kaarten/Watersysteem_beheerregister/MapServer", - "layer": 8, # Stuw beheer (9) + "url": "https://portal.wsrl.nl/kaarten/rest/services/Watersysteem/Legger_en_Werkingsgebieden_Wateren_Vastgesteld/MapServer", + "layer": 11, # Stuw }, "culverts": { - "url": "https://kaarten.wsrl.nl/arcgis/rest/services/Kaarten/Watersysteem_beheerregister/MapServer", - "layer": 5, # Duiker sifon hevel beheer (5) + "url": "https://portal.wsrl.nl/kaarten/rest/services/Watersysteem/Legger_en_Werkingsgebieden_Wateren_Vastgesteld/MapServer", + "layer": 7, # Duiker, sifon, hevel }, "level_areas": { # "url": "https://kaarten.wsrl.nl/arcgis/rest/services/Kaarten/Peilgebieden_praktijk/FeatureServer", @@ -645,11 +670,11 @@ def get_data(wb, data_kind, extent=None, max_record_count=None, config=None, **k A GeoDataFrame containing data from the waterboard (polygons for level_areas/level_deviations and lines for watercourses). """ - warnings.warn( "'get_data' is deprecated and will eventually be removed, " "please use 'nlmod.read.waterboard.download_data()' in the future.", DeprecationWarning, + stacklevel=2, ) return download_data(wb, data_kind, extent, max_record_count, config, **kwargs) @@ -768,11 +793,41 @@ def download_data( def _set_column_from_columns(gdf, set_column, from_columns, nan_values=None): - """Retrieve values from one or more Geo)DataFrame-columns and set these values as - another column. + """Populate a target column from one or more source columns. + + The function creates ``set_column`` and fills missing values by iterating over + ``from_columns`` in order. Each source can be either a single column name or a + list of column names. For a list input, the row-wise mean of available values is + used. Existing values in ``set_column`` are preserved once filled. + + Parameters + ---------- + gdf : pandas.DataFrame or geopandas.GeoDataFrame + Input table that contains the source columns. + set_column : str + Name of the column that will be created and filled. + from_columns : str or list, optional + Source definition used to populate ``set_column``. Supported entries are: + - ``str``: copy values from this column. + - ``list[str]``: use row-wise mean of these columns. + - ``list`` containing a mix of the above: processed in order until all + possible values are filled. + nan_values : float, int or list, optional + Value or values that should be interpreted as missing and replaced by + ``np.nan`` in the resulting ``set_column``. + + Returns + ------- + pandas.DataFrame or geopandas.GeoDataFrame + ``gdf`` with the newly created ``set_column``. + + Raises + ------ + ValueError + If ``set_column`` already exists in ``gdf``. """ if set_column in gdf.columns: - raise (Exception(f"Column {set_column} allready exists")) + raise (ValueError(f"Column {set_column} allready exists")) gdf[set_column] = np.nan if from_columns is None: return gdf diff --git a/nlmod/read/webservices.py b/nlmod/read/webservices.py index 19085ea0..d27a1c21 100644 --- a/nlmod/read/webservices.py +++ b/nlmod/read/webservices.py @@ -198,7 +198,7 @@ def _get_data(url, params, timeout=120, **kwargs): if "error" in data: code = data["error"]["code"] message = data["error"]["message"] - raise (ValueError(f"Error code {code}: {message}")) + raise (ValueError(f"Error code {code}: '{message}' from {url}")) return data @@ -338,7 +338,7 @@ def wcs( crs="EPSG:28992", maxsize=2000, ): - """Download data from a web coverage service (WCS) + """Download data from a web coverage service (WCS). Parameters ---------- @@ -423,8 +423,10 @@ def _split_wcs_extent( fmt, crs, ): - """There is a max height and width limit for the wcs server. This function splits - your extent in chunks smaller than the limit. It returns a list of Memory files. + """Splits your extent in chunks smaller than the limit. + + There is a max height and width limit for the wcs server. It returns a list of + Memory files. Parameters ---------- @@ -476,7 +478,8 @@ def _split_wcs_extent( end_y = start_y + maxsize * res subextent = [start_x, end_x, start_y, end_y] logger.debug( - f"segment x {tx + 1} of {x_segments}, segment y {ty + 1} of {y_segments}" + f"segment x {tx + 1} of {x_segments}, " + f"segment y {ty + 1} of {y_segments}" ) memfile = _download_wcs(subextent, res, url, identifier, version, fmt, crs) @@ -563,6 +566,7 @@ def _download_wcs(extent, res, url, identifier, version, fmt, crs): def get_temporary_certificate(cert_url): + """Download an intermediate certificate and combine it with the certifi bundle.""" import tempfile import certifi diff --git a/nlmod/sim/sim.py b/nlmod/sim/sim.py index a4c8fc2e..2eb599d6 100644 --- a/nlmod/sim/sim.py +++ b/nlmod/sim/sim.py @@ -7,17 +7,20 @@ import flopy import numpy as np import pandas as pd +import xarray as xr from .. import util +from ..dims.grid import get_idomain, modelgrid_from_ds from ..dims.time import get_perlen logger = logging.getLogger(__name__) def write_and_run(sim, ds, write_ds=True, script_path=None, silent=False): - """Write modflow files and run the model. Extra options include writing the model - dataset to a netcdf file in the model workspace and copying the modelscript to the - model workspace. + """Write modflow files and run the model. + + Extra options include writing the model dataset to a netcdf file in the model + workspace and copying the modelscript to the model workspace. Parameters ---------- @@ -50,10 +53,14 @@ def write_and_run(sim, ds, write_ds=True, script_path=None, silent=False): if write_ds: logger.info("write model dataset to cache") + for attr, value in ds.attrs.items(): + if isinstance(value, pathlib.Path): + ds.attrs[attr] = str(value) + ds.attrs["model_dataset_written_to_disk_on"] = dt.datetime.now().strftime( "%Y%m%d_%H:%M:%S" ) - if isinstance(ds.attrs["model_ws"], pathlib.PurePath): + if isinstance(ds.attrs["model_ws"], pathlib.Path): ds.to_netcdf(ds.attrs["model_ws"] / f"{ds.model_name}.nc") else: ds.to_netcdf(os.path.join(ds.attrs["model_ws"], f"{ds.model_name}.nc")) @@ -70,7 +77,7 @@ def write_and_run(sim, ds, write_ds=True, script_path=None, silent=False): def get_tdis_perioddata(ds, nstp="nstp", tsmult="tsmult"): - """Get tdis_perioddata from ds. + r"""Get tdis_perioddata from ds. Parameters ---------- @@ -101,7 +108,7 @@ def get_tdis_perioddata(ds, nstp="nstp", tsmult="tsmult"): if isinstance(tsmult, float): tsmult = [tsmult] * len(perlen) - tdis_perioddata = list(zip(perlen, nstp, tsmult)) + tdis_perioddata = list(zip(perlen, nstp, tsmult, strict=False)) return tdis_perioddata @@ -250,8 +257,191 @@ def ems(sim, pname="ems", model=None, **kwargs): def register_ims_package(sim, model, ims): + """Register an ims package with the simulation.""" sim.register_ims_package(ims, [model.name]) def register_solution_package(sim, model, solver): + """Register a solution package with the simulation.""" sim.register_solution_package(solver, [model.name]) + + +def get_parent_child_exchange_gdf(ds_parent, ds_child, boundnames="angldegx"): + """Get geodataframe with shared faces between parent and child model grids. + + Only considers horizontal connections between the parent and child model grids. The + shared faces are determined by finding the intersection of the boundaries of the + parent and child model grids. + + Parameters + ---------- + ds_parent : xarray.Dataset + dataset with model data for the parent model. + ds_child : xarray.Dataset + dataset with model data for the child model. + boundnames : str, optional + column to use as a boundname for modflow, default is 'angledegx', which will + convert the angle between the parent and child cell centroids to a compass + direction. If None, no boundnames will be used. + + Returns + ------- + shared_faces : geopandas.GeoDataFrame + geodataframe with shared faces between parent and child model grids, with + columns for parent and child cell ids, shared face geometry, and exchange + variables (cl1, cl2, hwva, angldegx, boundnames if applicable). + """ + gdf_parent = modelgrid_from_ds(ds_parent).to_geodataframe() + gdf_child = modelgrid_from_ds(ds_child).to_geodataframe() + + outer_ring = gdf_parent.loc[gdf_parent.touches(gdf_child.union_all())] + candidates = outer_ring.sjoin(gdf_child, how="inner", predicate="intersects") + candidates["shared_face_geom"] = candidates.apply( + lambda row: row["geometry"].boundary.intersection( + gdf_child.loc[row["index_right"], "geometry"] + ), + axis=1, + ) + shared_faces = candidates.loc[ + candidates["shared_face_geom"].geom_type.isin(["LineString", "MultiLineString"]) + ].copy() + shared_faces["child_geom"] = gdf_child.loc[ + shared_faces["index_right"], "geometry" + ].values + + # compute exchange variables + shared_faces["hwva"] = shared_faces.shared_face_geom.length + shared_faces["cl1"] = shared_faces.apply( + lambda row: row["geometry"].centroid.distance(row["shared_face_geom"]), axis=1 + ) + shared_faces["cl2"] = shared_faces.apply( + lambda row: row["shared_face_geom"].distance(row["child_geom"].centroid), axis=1 + ) + + shared_faces = shared_faces.reset_index(names="parent_cellid").rename( + columns={"index_right": "child_cellid", "geometry": "parent_geom"} + ) + dx = shared_faces["parent_geom"].centroid.x - shared_faces["child_geom"].centroid.x + dy = shared_faces["parent_geom"].centroid.y - shared_faces["child_geom"].centroid.y + + shared_faces["angldegx"] = np.degrees(np.atan2(dy, dx)) % 360 + shared_faces["cdist"] = np.sqrt(dx**2 + dy**2) + shared_faces = shared_faces.loc[ + :, + [ + "parent_cellid", + "parent_geom", + "shared_face_geom", + "child_cellid", + "child_geom", + "cl1", + "cl2", + "hwva", + "angldegx", + "cdist", + ], + ] + + if boundnames == "angldegx": + + def angle_to_compass(angle): + if (315 <= angle < 360) or (0 <= angle < 45): + return "E" + elif 45 <= angle < 135: + return "N" + elif 135 <= angle < 225: + return "W" + elif 225 <= angle < 315: + return "S" + + shared_faces["boundnames"] = shared_faces["angldegx"].apply(angle_to_compass) + shared_faces = shared_faces.sort_values(by="angldegx") + elif boundnames in shared_faces.columns: + shared_faces["boundnames"] = shared_faces[boundnames] + shared_faces = shared_faces.sort_values(by="boundnames") + else: + # not sure how this would work, since you don't really know the number of + # shared faces a priori and the order they would appear in... But maybe if + # you build multiple exchanges? + shared_faces["boundnames"] = boundnames + + return shared_faces + + +def gwfgwf( + sim, ds_parent, ds_child, exgtype="GWF6-GWF6", boundnames="angldegx", **kwargs +): + """Create GWF-GWF exchange package from the model datasets. + + Parameters + ---------- + sim : flopy MFSimulation + simulation object. + ds_parent : xarray.Dataset + dataset with model data for the parent model. + ds_child : xarray.Dataset + dataset with model data for the child model. + exgtype : str, optional + exchange type. The default is "GWF6-GWF6". + boundnames : str, optional + name of the boundary condition. The default is "angldegx". If None, no + boundname is added to the exchangedata. + **kwargs + passed on to flopy.mf6.ModflowGwfgwf + + Returns + ------- + gwfgwf : flopy ModflowGwfgwf + gwfgwf exchange object. + """ + # get single layer + exch_gdf = get_parent_child_exchange_gdf( + ds_parent, + ds_child, + boundnames=boundnames, + ) + if exch_gdf.empty: + raise ValueError("No shared faces found between parent and child model grids.") + + exch_gdf = exch_gdf.rename( + columns={ + "parent_cellid": "cellidm1", + "child_cellid": "cellidm2", + } + ) + exch_gdf["ihc"] = 1 # exchange is always horizontal + # ensure layers are equal + xr.testing.assert_equal(ds_parent.layer, ds_child.layer) + + idomain_parent = get_idomain(ds_parent) + idomain_child = get_idomain(ds_child) + exchangedata = [] + usecols = ["cellidm1", "cellidm2", "ihc", "cl1", "cl2", "hwva", "angldegx", "cdist"] + if boundnames is not None: + usecols.append("boundnames") + for ilay in range(ds_parent.sizes["layer"]): + idf = exch_gdf.loc[:, usecols].copy() + mask_active_parent = ( + idomain_parent.isel(layer=ilay, icell2d=idf["cellidm1"]) > 0 + ).values + mask_active_child = ( + idomain_child.isel(layer=ilay, icell2d=idf["cellidm2"]) > 0 + ).values + idf = idf.loc[mask_active_parent & mask_active_child].copy() + idf["cellidm1"] = [(ilay, cid) for cid in idf["cellidm1"]] + idf["cellidm2"] = [(ilay, cid) for cid in idf["cellidm2"]] + exchangedata.append(idf) + + exchangedata = pd.concat(exchangedata, axis=0) + + return flopy.mf6.ModflowGwfgwf( + sim, + exgmnamea=ds_parent.model_name, + exgmnameb=ds_child.model_name, + exgtype=exgtype, + nexg=len(exchangedata), + exchangedata=exchangedata.to_records(index=False), + auxiliary=["ANGLDEGX", "CDIST"], + boundnames=True if boundnames is not None else None, + **kwargs, + ) diff --git a/nlmod/util.py b/nlmod/util.py index d0334051..ab4dde56 100644 --- a/nlmod/util.py +++ b/nlmod/util.py @@ -22,6 +22,7 @@ except ImportError: # fallback: generate a dummy method with the same interface def tqdm(iterable=None, **_kwargs): + """Dummy tqdm.""" return iterable if iterable is not None else [] @@ -38,6 +39,21 @@ class MissingValueError(Exception): """Generic error when an expected value is not defined.""" +def _get_dim_coord_for_comparison(da_ds, dim): + """Get a normalized 1D coordinate for dimension comparison. + + Bare xarray dimensions behave like ``range(size)`` when indexed, but they are not + registered as coordinates or indexes. Normalize both implicit and explicit + dimension coordinates to the same explicit representation before comparing. + """ + if dim in da_ds.coords: + values = da_ds[dim].values + else: + values = np.arange(da_ds.sizes[dim]) + + return xr.DataArray(values, dims=(dim,), coords={dim: values}, name=dim) + + def check_da_dims_coords(da, ds): """Check if DataArray dimensions and coordinates match those in Dataset. @@ -63,7 +79,10 @@ def check_da_dims_coords(da, ds): shared_dims = set(da.dims) & set(ds.dims) for dim in shared_dims: try: - xr.testing.assert_identical(da[dim], ds[dim]) + xr.testing.assert_identical( + _get_dim_coord_for_comparison(da, dim), + _get_dim_coord_for_comparison(ds, dim), + ) except AssertionError as e: logger.error(f"da '{da.name}' coordinates do not match ds!") raise e @@ -71,9 +90,10 @@ def check_da_dims_coords(da, ds): def get_model_dirs(model_ws): - """Creates a new model workspace directory, if it does not exists yet. Within the - model workspace directory a few subdirectories are created (if they don't exist - yet): + """Creates a new model workspace directory, if it does not exists yet. + + Within the model workspace directory a few subdirectories are created + (if they don't exist yet): - figure - cache @@ -397,7 +417,7 @@ def get_flopy_bin_directories(version_tag=None, repo="executables"): return path_list -def download_mfbinaries(bindir=None, version_tag="latest", repo="executables"): +def download_mfbinaries(bindir=None, version_tag="latest", repo="modflow6"): """Download and unpack platform-specific modflow binaries. Source: USGS @@ -603,7 +623,7 @@ def compare_model_extents(extent1, extent2): def extent_to_polygon(extent): - """Generate a shapely Polygon from an extent ([xmin, xmax, ymin, ymax]) + """Generate a shapely Polygon from an extent ([xmin, xmax, ymin, ymax]). Parameters ---------- @@ -883,13 +903,15 @@ def format(self, record) -> str: return super().format(record) -def get_color_logger(level="INFO", logger_name=None): +def get_color_logger(level="INFO", logger_name="nlmod"): """Get a logger with colored output. Parameters ---------- level : str, optional The logging level to set for the logger. Default is "INFO". + logger_name : str, optional + The name of the logger. Default is "nlmod". Returns ------- @@ -1239,7 +1261,7 @@ def zonal_statistics( gc["values"] = da gc = gc.set_coords("index") groups = gc.groupby("index") - for stat, column in zip(statistics, columns): + for stat, column in zip(statistics, columns, strict=False): if stat == "min": values = groups.min() elif stat == "max": @@ -1282,7 +1304,7 @@ def zonal_statistics( all_touched=all_touched, progress=progressbar, ) - for stat, column in zip(statistics, columns): + for stat, column in zip(statistics, columns, strict=False): geometry[column] = [x[stat] for x in stats] else: diff --git a/nlmod/version.py b/nlmod/version.py index 0918bce5..3cb79ce3 100644 --- a/nlmod/version.py +++ b/nlmod/version.py @@ -1,7 +1,7 @@ from importlib import metadata from platform import python_version -__version__ = "0.11.2" +__version__ = "0.12.0" def show_versions() -> None: diff --git a/pyproject.toml b/pyproject.toml index 80f62af3..86b5024f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,14 +18,13 @@ maintainers = [ { name = "R.J. Calje", email = "r.calje@artesia-water.nl" }, { name = "D.A. Brakenhoff", email = "d.brakenhoff@artesia-water.nl" }, ] -requires-python = ">= 3.10" +requires-python = ">= 3.11" dependencies = [ "flopy>=3.10.0", "xarray>=0.16.1", "netcdf4>=1.7.2", "rasterio>=1.1.0", "rioxarray", - "affine>=0.3.1", "geopandas", "shapely>=2.0.0", "matplotlib", @@ -41,7 +40,6 @@ classifiers = [ "Intended Audience :: Other Audience", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", @@ -61,17 +59,17 @@ full = [ "rasterstats", "contextily", "scikit-image", - "py7zr", "joblib", "tqdm", "hydropandas>=0.9.2", "owslib>=0.24.1", "pyshp>=2.1.3", + "numba", ] knmi = ["h5netcdf", "h5py", "nlmod[grib]"] -grib = ["cfgrib", "ecmwflibs"] +grib = ["cfgrib"] test = ["pytest>=7", "pytest-cov", "lxml"] -nbtest = ["nbformat", "nbconvert>6.4.5"] +nbtest = ["nbformat", "nbconvert>6.4.5", "openpyxl"] lint = ["ruff"] ci = ["nlmod[full,lint,test,nbtest]"] rtd = [ @@ -83,7 +81,10 @@ rtd = [ "sphinx_rtd_theme", "nbconvert", "openpyxl", + "numpy<2.5.0", # until there is a new version of netCDF4 (now 1.7.4) this solves "Setting the shape on a NumPy array has been deprecated" + "affine<3.0.1", # until there are newer versions of rasterio (now 1.5.1) and rioxarray (now 0.23.0), this solves "Use @ matmul instead of * mul operator for matrix multiplication" ] +lgn = ["xarray-spatial", "exactextract"] [tool.setuptools.dynamic] version = { attr = "nlmod.version.__version__" } @@ -129,9 +130,10 @@ ignore = [ extend-select = ["I"] pydocstyle.convention = "numpy" +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["D103", "D205", "E501", "E722", "PT011", "PT028", "PT030"] +"docs/**" = ["D103", "E402", "E501"] + [tool.pytest.ini_options] addopts = "--strict-markers --durations=0 --cov-report xml:coverage.xml --cov nlmod -v" markers = ["notebooks: run notebooks", "slow: slow tests", "skip: skip tests"] - -[tool.coverage.run] -dynamic_context = "test_function" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..32baf846 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,46 @@ +import gc +import os +import shutil +from pathlib import Path + +import matplotlib.pyplot as plt +import pytest + +try: + from xarray.backends.file_manager import FILE_CACHE +except Exception: # pragma: no cover - defensive for xarray internals + FILE_CACHE = None + +MODEL_DATA_ENV_VAR = "NLMOD_TEST_MODEL_DATA_DIR" + + +@pytest.fixture(scope="session", autouse=True) +def session_model_data_dir(tmp_path_factory): + """Use one temp folder for model-data generated and shared during a test session.""" + source_dir = Path(__file__).parent / "data" + model_data_dir = tmp_path_factory.mktemp("model_data") + + # Seed the shared temp directory with checked-in fixture files. + for pattern in ("*.nc", "*.zip", "*.tar"): + for src in source_dir.glob(pattern): + shutil.copy2(src, model_data_dir / src.name) + + old_value = os.environ.get(MODEL_DATA_ENV_VAR) + os.environ[MODEL_DATA_ENV_VAR] = str(model_data_dir) + try: + yield model_data_dir + finally: + if old_value is None: + os.environ.pop(MODEL_DATA_ENV_VAR, None) + else: + os.environ[MODEL_DATA_ENV_VAR] = old_value + + +@pytest.fixture(autouse=True) +def close_all_matplotlib_figures(): + """Close figures and backend file handles created during a test.""" + yield + plt.close("all") + gc.collect() + if FILE_CACHE is not None: + FILE_CACHE.clear() diff --git a/tests/data/mf6output/structured/test.cbc b/tests/data/mf6output/structured/test.cbc index a87abdbc..7b98e7f0 100644 Binary files a/tests/data/mf6output/structured/test.cbc and b/tests/data/mf6output/structured/test.cbc differ diff --git a/tests/data/mf6output/structured/test.hds b/tests/data/mf6output/structured/test.hds index 09f051ed..3c2cbc31 100644 Binary files a/tests/data/mf6output/structured/test.hds and b/tests/data/mf6output/structured/test.hds differ diff --git a/tests/data/mf6output/unstructured/test.cbc b/tests/data/mf6output/unstructured/test.cbc index 6c804c35..71740a47 100644 Binary files a/tests/data/mf6output/unstructured/test.cbc and b/tests/data/mf6output/unstructured/test.cbc differ diff --git a/tests/data/mf6output/unstructured/test.disu.grb b/tests/data/mf6output/unstructured/test.disu.grb index 901aab50..3ed89c42 100644 Binary files a/tests/data/mf6output/unstructured/test.disu.grb and b/tests/data/mf6output/unstructured/test.disu.grb differ diff --git a/tests/data/mf6output/unstructured/test.hds b/tests/data/mf6output/unstructured/test.hds index a5e88618..c3953984 100644 Binary files a/tests/data/mf6output/unstructured/test.hds and b/tests/data/mf6output/unstructured/test.hds differ diff --git a/tests/test_001_model.py b/tests/test_001_model.py index 29150f28..809d1fb2 100644 --- a/tests/test_001_model.py +++ b/tests/test_001_model.py @@ -1,19 +1,49 @@ import os -import tempfile import numpy as np import pandas as pd import pytest +import util import xarray as xr import nlmod -tmpdir = tempfile.gettempdir() -tst_model_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") +def _get_model_data_path(name): + return os.path.join(util.get_model_data_dir(), name) -def test_model_directories(tmpdir): - model_ws = os.path.join(tmpdir, "test_model") + +def _get_model_ws(name): + return os.path.join(util.get_model_data_dir(), name) + + +def _ensure_cached_model(name): + model_data_path = _get_model_data_path(name + ".nc") + if os.path.exists(model_data_path): + return model_data_path + + if name == "small_model": + _build_small_model_grid_only(model_name="small_model") + elif name == "sea_model_grid_only": + _build_sea_model_grid_only(model_name="sea_model") + elif name == "sea_model": + _ensure_cached_model("sea_model_grid_only") + _build_sea_model() + else: + raise FileNotFoundError( + f"No cached model generator is configured for '{name}'." + ) + + if not os.path.exists(model_data_path): + raise FileNotFoundError( + f"Cached model '{name}' was not created at expected path: {model_data_path}" + ) + + return model_data_path + + +def test_model_directories(tmp_path): + model_ws = os.path.join(tmp_path, "test_model") figdir, cachedir = nlmod.util.get_model_dirs(model_ws) @@ -27,25 +57,8 @@ def test_snap_extent(): assert new_extent == [1000.0, 2000.0, 7975.0, 10010.0] -def get_ds_time_steady(tmpdir, modelname="test"): - model_ws = os.path.join(tmpdir, "test_model") - ds = nlmod.base.set_ds_attrs(xr.Dataset(), modelname, model_ws) - ds = nlmod.time.set_ds_time(ds, time=["2015-1-2"], start="2015-1-1", steady=True) - return ds - - -def get_ds_time_transient(tmpdir, modelname="test"): - model_ws = os.path.join(tmpdir, "test_model") - ds = nlmod.base.set_ds_attrs(xr.Dataset(), modelname, model_ws) - nper = 11 - time = pd.date_range(start="2015-1-2", periods=nper, freq="D") - steady = np.zeros(nper) - ds = nlmod.time.set_ds_time(ds, time=time, start="2015-1-1", steady=steady) - return ds - - -def test_get_ds(): - model_ws = os.path.join(tmpdir, "test_model_ds") +def test_get_ds(tmp_path): + model_ws = os.path.join(tmp_path, "test_model_ds") nlmod.get_ds( [-500, 500, -500, 500], delr=10.0, @@ -59,8 +72,8 @@ def test_get_ds(): ) -def test_get_ds_variable_delrc(): - model_ws = os.path.join(tmpdir, "test_model_ds") +def test_get_ds_variable_delrc(tmp_path): + model_ws = os.path.join(tmp_path, "test_model_ds") nlmod.get_ds( extent=[-500, 500, -500, 500], delr=[100] * 5 + [20] * 5 + [100] * 4, @@ -75,14 +88,13 @@ def test_get_ds_variable_delrc(): ) -@pytest.mark.slow -def test_create_small_model_grid_only(tmpdir, model_name="test"): +def _build_small_model_grid_only(model_name="small_model"): extent = [98700.0, 99000.0, 489500.0, 489700.0] # extent, nrow, ncol = nlmod.read.regis.fit_extent_to_regis(extent, 100, 100) regis_geotop_ds = nlmod.read.regis.get_combined_layer_models( extent, regis_botm_layer="KRz5", use_regis=True, use_geotop=True ) - model_ws = os.path.join(tmpdir, model_name) + model_ws = _get_model_ws(model_name) ds = nlmod.base.to_model_ds( regis_geotop_ds, model_name, model_ws, delr=100.0, delc=100.0 ) @@ -114,17 +126,21 @@ def test_create_small_model_grid_only(tmpdir, model_name="test"): _ = nlmod.gwf.dis(ds, gwf) # save ds - ds.to_netcdf(os.path.join(tst_model_dir, "small_model.nc")) + ds.to_netcdf(_get_model_data_path("small_model.nc")) @pytest.mark.slow -def test_create_sea_model_grid_only(tmpdir, model_name="test"): +def test_create_small_model_grid_only(): + _build_small_model_grid_only(model_name="small_model") + + +def _build_sea_model_grid_only(model_name="sea_model"): extent = [95000.0, 105000.0, 494000.0, 500000.0] # extent, nrow, ncol = nlmod.read.regis.fit_extent_to_regis(extent, 100, 100) regis_geotop_ds = nlmod.read.regis.get_combined_layer_models( extent, use_regis=True, use_geotop=True ) - model_ws = os.path.join(tmpdir, model_name) + model_ws = _get_model_ws("sea_model_grid_only") ds = nlmod.base.to_model_ds( regis_geotop_ds, model_name, model_ws, delr=100.0, delc=100.0 ) @@ -139,31 +155,24 @@ def test_create_sea_model_grid_only(tmpdir, model_name="test"): steady=steady, ) + ds.attrs["model_name"] = model_name + ds.attrs["model_ws"] = model_ws + # save ds - ds.to_netcdf(os.path.join(tst_model_dir, "basic_sea_model.nc")) + ds.to_netcdf(_get_model_data_path("sea_model_grid_only.nc")) @pytest.mark.slow -def test_create_sea_model_grid_only_delr_delc_50(tmpdir, model_name="test"): - ds = get_ds_time_transient(tmpdir) - extent = [95000.0, 105000.0, 494000.0, 500000.0] - # extent, nrow, ncol = nlmod.read.regis.fit_extent_to_regis(extent, 50.0, 50.0) - regis_geotop_ds = nlmod.read.regis.get_combined_layer_models( - extent, use_regis=True, use_geotop=True - ) - model_ws = os.path.join(tmpdir, model_name) - ds = nlmod.base.to_model_ds( - regis_geotop_ds, model_name, model_ws, delr=50.0, delc=50.0 - ) - - # save ds - ds.to_netcdf(os.path.join(tst_model_dir, "sea_model_grid_50.nc")) +def test_create_sea_model_grid_only(): + _build_sea_model_grid_only(model_name="sea_model") -@pytest.mark.slow -def test_create_sea_model(tmpdir): +def _build_sea_model(): ds = xr.open_dataset( - os.path.join(tst_model_dir, "basic_sea_model.nc"), mask_and_scale=False + _get_model_data_path("sea_model_grid_only.nc"), mask_and_scale=False + ) + ds = nlmod.base.set_ds_attrs( + ds, model_name="sea_model", model_ws=_get_model_ws("sea_model") ) # create simulation sim = nlmod.sim.sim(ds) @@ -208,175 +217,23 @@ def test_create_sea_model(tmpdir): _ = nlmod.gwf.chd(ds, gwf, mask="edge_mask", head="starting_head") # add knmi recharge to the model datasets - ds.update(nlmod.read.knmi.get_recharge(ds)) + ds.update(nlmod.read.knmi.get_recharge(ds, hourly_precision=True)) # create recharge package _ = nlmod.gwf.rch(ds, gwf) - _ = nlmod.sim.write_and_run(sim, ds) - + ds.to_netcdf(_get_model_data_path("sea_model.nc")) -@pytest.mark.slow -def test_create_sea_model_perlen_list(tmpdir): - ds = xr.open_dataset(os.path.join(tst_model_dir, "basic_sea_model.nc")) - - # update model_ws - model_ws = os.path.join(tmpdir, "test_model_perlen_list") - ds = nlmod.base.set_ds_attrs(ds, ds.model_name, model_ws) - - # create transient with perlen list - start = ds.time.start - perlen = [3650, 14, 10, 11] # length of the time steps - steady = np.zeros(len(perlen), dtype=int) - steady[0] = 1 - - # drop time dimension before setting time - ds = ds.drop_dims("time") - - # update current ds with new time dicretisation - ds = nlmod.time.set_ds_time( - ds, - time=np.cumsum(perlen), - start=start, - steady=steady, - ) - - # create simulation - sim = nlmod.sim.sim(ds) - - # create time discretisation - _ = nlmod.sim.tdis(ds, sim) - - # create groundwater flow model - gwf = nlmod.gwf.gwf(ds, sim) - - # create ims - _ = nlmod.sim.ims(sim) - - # Create discretization - nlmod.gwf.dis(ds, gwf) - - # create node property flow - nlmod.gwf.npf(ds, gwf) - - # Create the initial conditions package - nlmod.gwf.ic(ds, gwf, starting_head=1.0) - - # Create the output control package - nlmod.gwf.oc(ds, gwf) - - # voeg grote oppervlaktewaterlichamen toe - da_name = "surface_water" - gdf_surface_water = nlmod.read.rws.get_gdf_surface_water(ds=ds) - ds.update( - nlmod.read.rws.get_surface_water(ds, gdf=gdf_surface_water, da_basename=da_name) - ) - _ = nlmod.gwf.ghb(ds, gwf, bhead=f"{da_name}_stage", cond=f"{da_name}_cond") - - # surface level drain - ds.update(nlmod.read.ahn.get_ahn(ds)) - nlmod.gwf.surface_drain_from_ds(ds, gwf, 1.0) - - # add constant head cells at model boundaries - ds.update(nlmod.grid.mask_model_edge(ds)) - nlmod.gwf.chd(ds, gwf, mask="edge_mask", head="starting_head") - - # add knmi recharge to the model datasets - ds.update(nlmod.read.knmi.get_recharge(ds)) - # create recharge package - nlmod.gwf.rch(ds, gwf) - - nlmod.sim.write_and_run(sim, ds) + _ = nlmod.sim.write_and_run(sim, ds) @pytest.mark.slow -def test_create_sea_model_perlen_14(tmpdir): - ds = xr.open_dataset(os.path.join(tst_model_dir, "basic_sea_model.nc")) - - # update model_ws - model_ws = os.path.join(tmpdir, "test_model_perlen_14") - ds = nlmod.base.set_ds_attrs(ds, ds.model_name, model_ws) - - # create transient with perlen list - perlen = 14 # length of the transient time steps - nper = 4 - start = ds.time.start - perlen = perlen * np.ones(nper) - perlen[0] = 3652.0 # length of the steady state step - steady = np.zeros(nper, dtype=int) - steady[0] = 1 - time = nlmod.time.ds_time_idx_from_tdis_settings(start, perlen=perlen) - - # drop time dimension before setting time - ds = ds.drop_dims("time") - - # update current ds with new time discretization - ds = nlmod.time.set_ds_time( - ds, - time=time, - start=start, - steady=steady, - ) - - # create simulation - sim = nlmod.sim.sim(ds) - - # create time discretisation - _ = nlmod.sim.tdis(ds, sim) - - # create ims - _ = nlmod.sim.ims(sim) - - # create groundwater flow model - gwf = nlmod.gwf.gwf(ds, sim) - - # Create discretization - nlmod.gwf.dis(ds, gwf) - - # create node property flow - nlmod.gwf.npf(ds, gwf) - - # Create the initial conditions package - nlmod.gwf.ic(ds, gwf, starting_head=1.0) - - # Create the output control package - nlmod.gwf.oc(ds, gwf) - - # voeg grote oppervlaktewaterlichamen toe - da_name = "surface_water" - gdf_surface_water = nlmod.read.rws.get_gdf_surface_water(ds=ds) - ds.update( - nlmod.read.rws.get_surface_water(ds, gdf=gdf_surface_water, da_basename=da_name) - ) - _ = nlmod.gwf.ghb(ds, gwf, bhead=f"{da_name}_stage", cond=f"{da_name}_cond") - - # surface level drain - ds.update(nlmod.read.ahn.get_ahn(ds)) - nlmod.gwf.surface_drain_from_ds(ds, gwf, 1.0) - - # add constant head cells at model boundaries - ds.update(nlmod.grid.mask_model_edge(ds)) - nlmod.gwf.chd(ds, gwf, mask="edge_mask", head="starting_head") - - # add knmi recharge to the model datasets - ds.update(nlmod.read.knmi.get_recharge(ds)) - # create recharge package - nlmod.gwf.rch(ds, gwf) - - nlmod.sim.write_and_run(sim, ds) +def test_create_sea_model(): + _build_sea_model() # obtaining the test models def get_ds_from_cache(name="small_model"): - ds = xr.open_dataset(os.path.join(tst_model_dir, name + ".nc")) - return ds - - -# other functions -def _check_tmpdir(tmpdir): - # pytest uses a LocalPath object for the tmpdir argument when testing - # this function convert a LocalPath object to a string - - if isinstance(tmpdir, str): - return tmpdir - else: - return str(tmpdir) + model_data_path = _ensure_cached_model(name) + # Load into memory so the source file handle can be closed immediately. + with xr.open_dataset(model_data_path) as ds: + return ds.load() diff --git a/tests/test_002_regis_geotop.py b/tests/test_002_regis_geotop.py index eab162b2..291b7f19 100644 --- a/tests/test_002_regis_geotop.py +++ b/tests/test_002_regis_geotop.py @@ -1,37 +1,49 @@ import matplotlib.pyplot as plt import numpy as np +import xarray as xr import nlmod +from nlmod.read.geotop import split_layers_on_geul -def test_get_regis(extent=[98600.0, 99000.0, 489400.0, 489700.0]): +def test_get_regis(extent=None): + if extent is None: + extent = [98600.0, 99000.0, 489400.0, 489700.0] regis_ds = nlmod.read.regis.download_regis(extent) assert regis_ds.sizes["layer"] == 20 def test_get_regis_botm_layer_BEk1( - extent=[98700.0, 99000.0, 489500.0, 489700.0], + extent=None, botm_layer="MSc", ): + if extent is None: + extent = [98700.0, 99000.0, 489500.0, 489700.0] regis_ds = nlmod.read.regis.download_regis(extent, botm_layer) assert regis_ds.sizes["layer"] == 15 assert regis_ds.layer.values[-1] == botm_layer -def test_get_regis_only_c(extent=[98700.0, 99000.0, 489500.0, 489700.0]): +def test_get_regis_only_c(extent=None): + if extent is None: + extent = [98700.0, 99000.0, 489500.0, 489700.0] regis_ds = nlmod.read.regis.download_regis(extent, variables="c") assert np.all([x == "c" for x in regis_ds.data_vars]) assert regis_ds.sizes["layer"] == 8 -def test_get_regis_only_c_and_kd(extent=[98700.0, 99000.0, 489500.0, 489700.0]): +def test_get_regis_only_c_and_kd(extent=None): + if extent is None: + extent = [98700.0, 99000.0, 489500.0, 489700.0] regis_ds = nlmod.read.regis.download_regis(extent, variables=["c", "kD"]) assert np.all([x in ["c", "kD"] for x in regis_ds.data_vars]) assert regis_ds.sizes["layer"] == 18 -def test_get_geotop(extent=[98600.0, 99000.0, 489400.0, 489700.0]): +def test_get_geotop(extent=None): + if extent is None: + extent = [98600.0, 99000.0, 489400.0, 489700.0] geotop_ds = nlmod.read.geotop.download_geotop(extent) line = [(extent[0], extent[2]), (extent[1], extent[3])] @@ -43,7 +55,9 @@ def test_get_geotop(extent=[98600.0, 99000.0, 489400.0, 489700.0]): nlmod.plot.geotop_lithok_on_map(geotop_ds, z=-20.2, ax=ax) -def test_get_regis_geotop(extent=[98600.0, 99000.0, 489400.0, 489700.0]): +def test_get_regis_geotop(extent=None): + if extent is None: + extent = [98600.0, 99000.0, 489400.0, 489700.0] regis_geotop_ds = nlmod.read.regis.get_combined_layer_models( extent, use_regis=True, use_geotop=True ) @@ -52,8 +66,10 @@ def test_get_regis_geotop(extent=[98600.0, 99000.0, 489400.0, 489700.0]): def test_get_regis_geotop_keep_all_layers( - extent=[98600.0, 99000.0, 489400.0, 489700.0], + extent=None, ): + if extent is None: + extent = [98600.0, 99000.0, 489400.0, 489700.0] regis_geotop_ds = nlmod.read.regis.get_combined_layer_models( extent, use_regis=True, use_geotop=True, remove_nan_layers=False ) @@ -75,3 +91,89 @@ def test_add_kh_and_kv(): gt = nlmod.read.geotop.add_kh_and_kv(gt, df) # again, but using the stochastic method gt = nlmod.read.geotop.add_kh_and_kv(gt, df, stochastic=True) + + +def test_geulen_geotop(): + + # example geul at top + strat = xr.DataArray( + np.array([[0, 10, 10], [1, 1, 1], [2, 2, 2]])[:, None, :], + coords={"z": [0, -0.5, -1.0], "y": [100], "x": [100, 200, 300]}, + dims=("z", "y", "x"), + ) + units_no_geul = [0, 1, 2] + geulen = [10] + + strat_modified, new_unit_order = split_layers_on_geul(strat, units_no_geul, geulen) + assert np.array_equal(np.unique(strat_modified), np.unique(new_unit_order)) + + # example geul at bottom + strat = xr.DataArray( + np.array([[0, 0, 0], [1, 1, 1], [2, 10, 10]])[:, None, :], + coords={"z": [0, -0.5, -1.0], "y": [100], "x": [100, 200, 300]}, + dims=("z", "y", "x"), + ) + units_no_geul = [0, 1, 2] + geulen = [10] + + strat_modified, new_unit_order = split_layers_on_geul(strat, units_no_geul, geulen) + assert np.array_equal(np.unique(strat_modified), np.unique(new_unit_order)) + + # example split single layer + strat = xr.DataArray( + np.array([[0, 0, 0], [0, 10, 10], [0, 0, 0]])[:, None, :], + coords={"z": [0, -0.5, -1.0], "y": [100], "x": [100, 200, 300]}, + dims=("z", "y", "x"), + ) + units_no_geul = [0] + geulen = [10] + + strat_modified, new_unit_order = split_layers_on_geul(strat, units_no_geul, geulen) + assert np.array_equal(np.unique(strat_modified), np.unique(new_unit_order)) + + # example simple add between layers + strat = xr.DataArray( + np.array([[0, 0, 0], [0, 10, 10], [1, 1, 1]])[:, None, :], + coords={"z": [0, -0.5, -1.0], "y": [100], "x": [100, 200, 300]}, + dims=("z", "y", "x"), + ) + units_no_geul = [0, 1] + geulen = [10] + + strat_modified, new_unit_order = split_layers_on_geul(strat, units_no_geul, geulen) + assert np.array_equal(np.unique(strat_modified), np.unique(new_unit_order)) + + # example add between layers with geul at top and bottom + strat = xr.DataArray( + np.array([[0, 0, 10], [10, 10, 10], [10, 1, 1]])[:, None, :], + coords={"z": [0, -0.5, -1.0], "y": [100], "x": [100, 200, 300]}, + dims=("z", "y", "x"), + ) + units_no_geul = [0, 1] + geulen = [10] + + strat_modified, new_unit_order = split_layers_on_geul(strat, units_no_geul, geulen) + + # example geul between and below + strat = xr.DataArray( + np.array([[0, 0, 0], [10, 1, 1], [0, 10, 10]])[:, None, :], + coords={"z": [0, -0.5, -1.0], "y": [100], "x": [100, 200, 300]}, + dims=("z", "y", "x"), + ) + units_no_geul = [0, 1] + geulen = [10] + + strat_modified, new_unit_order = split_layers_on_geul(strat, units_no_geul, geulen) + assert np.array_equal(np.unique(strat_modified), np.unique(new_unit_order)) + + # example 2 geulen + strat = xr.DataArray( + np.array([[0, 10, 0], [1, 20, 1], [2, 2, 2]])[:, None, :], + coords={"z": [0, -0.5, -1.0], "y": [100], "x": [100, 200, 300]}, + dims=("z", "y", "x"), + ) + units_no_geul = [0, 1, 2] + geulen = [10, 20] + + strat_modified, new_unit_order = split_layers_on_geul(strat, units_no_geul, geulen) + assert np.array_equal(np.unique(strat_modified), np.unique(new_unit_order)) diff --git a/tests/test_003_mfpackages.py b/tests/test_003_mfpackages.py index 6857d9d2..0874031d 100644 --- a/tests/test_003_mfpackages.py +++ b/tests/test_003_mfpackages.py @@ -1,13 +1,12 @@ import numpy as np +import pytest import test_001_model import xarray as xr import nlmod -def sim_tdis_gwf_ims_from_ds(tmpdir): - ds = test_001_model.get_ds_from_cache("basic_sea_model") - +def sim_tdis_gwf_ims_from_ds(ds): # create simulation sim = nlmod.sim.sim(ds) @@ -23,59 +22,59 @@ def sim_tdis_gwf_ims_from_ds(tmpdir): return sim, gwf -def dis_from_ds(tmpdir): +def test_dis_from_ds(): ds = test_001_model.get_ds_from_cache("small_model") - _, gwf = sim_tdis_gwf_ims_from_ds(tmpdir) + _, gwf = sim_tdis_gwf_ims_from_ds(ds) nlmod.gwf.dis(ds, gwf) -def npf_from_ds(tmpdir): +def test_npf_from_ds(): ds = test_001_model.get_ds_from_cache("small_model") - _, gwf = sim_tdis_gwf_ims_from_ds(tmpdir) - nlmod.gwf.dis(ds) + _, gwf = sim_tdis_gwf_ims_from_ds(ds) + nlmod.gwf.dis(ds, gwf) nlmod.gwf.npf(ds, gwf) -def oc_from_ds(tmpdir): +def test_oc_from_ds(): ds = test_001_model.get_ds_from_cache("small_model") - _, gwf = sim_tdis_gwf_ims_from_ds(tmpdir) + _, gwf = sim_tdis_gwf_ims_from_ds(ds) nlmod.gwf.oc(ds, gwf) -def sto_from_ds(tmpdir): +def test_sto_from_ds(): ds = test_001_model.get_ds_from_cache("small_model") - _, gwf = sim_tdis_gwf_ims_from_ds(tmpdir) + _, gwf = sim_tdis_gwf_ims_from_ds(ds) nlmod.gwf.sto(ds, gwf) -def ghb_from_ds(tmpdir): - ds = test_001_model.get_ds_from_cache("full_sea_model") - _, gwf = sim_tdis_gwf_ims_from_ds(tmpdir) +def test_ghb_from_ds(): + ds = test_001_model.get_ds_from_cache("sea_model") + _, gwf = sim_tdis_gwf_ims_from_ds(ds) _ = nlmod.gwf.dis(ds, gwf) nlmod.gwf.ghb(ds, gwf, bhead="surface_water_stage", cond="surface_water_cond") -def rch_from_ds(tmpdir): - ds = test_001_model.get_ds_from_cache("full_sea_model") - _, gwf = sim_tdis_gwf_ims_from_ds(tmpdir) +def test_rch_from_ds(): + ds = test_001_model.get_ds_from_cache("sea_model") + _, gwf = sim_tdis_gwf_ims_from_ds(ds) _ = nlmod.gwf.dis(ds, gwf) nlmod.gwf.rch(ds, gwf) -def drn_from_ds(tmpdir): - ds = test_001_model.get_ds_from_cache("full_sea_model") - _, gwf = sim_tdis_gwf_ims_from_ds(tmpdir) +def test_drn_from_ds(): + ds = test_001_model.get_ds_from_cache("sea_model") + _, gwf = sim_tdis_gwf_ims_from_ds(ds) _ = nlmod.gwf.dis(ds, gwf) nlmod.gwf.surface_drain_from_ds(ds, gwf, 1.0) -def chd_from_ds(tmpdir): +def test_chd_from_ds(): ds = test_001_model.get_ds_from_cache("small_model") - _, gwf = sim_tdis_gwf_ims_from_ds(tmpdir) + _, gwf = sim_tdis_gwf_ims_from_ds(ds) _ = nlmod.gwf.dis(ds, gwf) _ = nlmod.gwf.ic(ds, gwf, starting_head=1.0) @@ -85,7 +84,7 @@ def chd_from_ds(tmpdir): nlmod.gwf.chd(ds, gwf, mask="edge_mask", head="starting_head") -def get_value_from_ds_datavar(): +def test_get_value_from_ds_datavar(): ds = xr.Dataset( coords={ "layer": [0, 1], @@ -104,7 +103,7 @@ def get_value_from_ds_datavar(): # get value from ds, variable and stored name are different v1 = nlmod.util._get_value_from_ds_datavar(ds, "test", "test_var") - xr.testing.assert_equal(ds["test_var"].values, v1) + (ds["test_var"].values == v1).all() # do not get value from ds, value is Data Array, should log info msg v2 = nlmod.util._get_value_from_ds_datavar(ds, "test", v0, return_da=True) @@ -124,7 +123,18 @@ def get_value_from_ds_datavar(): assert v5 is None, "should be None." -def get_value_from_ds_attr(): +def test_check_da_dims_coords_with_implicit_dim_coordinate(): + ds = xr.Dataset(coords={"icell2d": np.arange(3)}) + da = xr.DataArray(np.ones(3), dims=("icell2d",), name="test") + + assert nlmod.util.check_da_dims_coords(da, ds) + + ds_shifted = xr.Dataset(coords={"icell2d": np.arange(1, 4)}) + with pytest.raises(AssertionError): + nlmod.util.check_da_dims_coords(da, ds_shifted) + + +def test_get_value_from_ds_attr(): ds = xr.Dataset( coords={ "layer": [0, 1], @@ -159,7 +169,7 @@ def get_value_from_ds_attr(): v5 = nlmod.util._get_value_from_ds_attr(ds, "test", "test", value=3.0) assert v5 == 3.0 - # user user-provided str value, no msg, since "test" is not in attrs + # use user-provided str value, no msg, since "test" is not in attrs v6 = nlmod.util._get_value_from_ds_attr(ds, "test", "test", value="test") assert v6 == "test" diff --git a/tests/test_004_northsea.py b/tests/test_004_northsea.py index 9a6086bd..4cac1c4b 100644 --- a/tests/test_004_northsea.py +++ b/tests/test_004_northsea.py @@ -10,15 +10,16 @@ def test_get_gdf_opp_water(): def test_surface_water_to_dataset(): # model with sea - ds = test_001_model.get_ds_from_cache("basic_sea_model") + ds = test_001_model.get_ds_from_cache("sea_model_grid_only") name = "surface_water" - nlmod.read.rws.get_surface_water(ds, da_basename=name) + gdf_surface_water = nlmod.read.rws.get_gdf_surface_water(ds) + nlmod.read.rws.discretize_surface_water(ds, gdf_surface_water, da_basename=name) def test_get_northsea_seamodel(): # model with sea - ds = test_001_model.get_ds_from_cache("basic_sea_model") - ds_sea = nlmod.read.rws.get_northsea(ds) + ds = test_001_model.get_ds_from_cache("sea_model_grid_only") + ds_sea = nlmod.read.rws.discretize_northsea(ds) assert (ds_sea.northsea == 1).sum() > 0 @@ -26,15 +27,15 @@ def test_get_northsea_seamodel(): def test_get_northsea_nosea(): # model without sea ds = test_001_model.get_ds_from_cache("small_model") - ds_sea = nlmod.read.rws.get_northsea(ds) + ds_sea = nlmod.read.rws.discretize_northsea(ds) assert (ds_sea.northsea == 1).sum() == 0 def test_fill_top_bot_kh_kv_seamodel(): # model with sea - ds = test_001_model.get_ds_from_cache("basic_sea_model") - ds.update(nlmod.read.rws.get_northsea(ds)) + ds = test_001_model.get_ds_from_cache("sea_model_grid_only") + ds.update(nlmod.read.rws.discretize_northsea(ds)) fal = nlmod.layers.get_first_active_layer(ds) fill_mask = (fal == fal.nodata) * ds["northsea"] @@ -44,7 +45,7 @@ def test_fill_top_bot_kh_kv_seamodel(): def test_fill_top_bot_kh_kv_nosea(): # model with sea ds = test_001_model.get_ds_from_cache("small_model") - ds.update(nlmod.read.rws.get_northsea(ds)) + ds.update(nlmod.read.rws.discretize_northsea(ds)) fal = nlmod.layers.get_first_active_layer(ds) fill_mask = (fal == fal.nodata) * ds["northsea"] @@ -53,8 +54,8 @@ def test_fill_top_bot_kh_kv_nosea(): def test_get_bathymetry_seamodel(): # model with sea - ds = test_001_model.get_ds_from_cache("basic_sea_model") - ds.update(nlmod.read.rws.get_northsea(ds)) + ds = test_001_model.get_ds_from_cache("sea_model_grid_only") + ds.update(nlmod.read.rws.discretize_northsea(ds)) ds_bathymetry = nlmod.read.jarkus.get_bathymetry(ds, datavar_sea="northsea") assert (~ds_bathymetry.bathymetry.isnull()).sum() > 0 @@ -63,7 +64,7 @@ def test_get_bathymetry_seamodel(): def test_get_bathymetry_nosea(): # model without sea ds = test_001_model.get_ds_from_cache("small_model") - ds.update(nlmod.read.rws.get_northsea(ds)) + ds.update(nlmod.read.rws.discretize_northsea(ds)) ds_bathymetry = nlmod.read.jarkus.get_bathymetry(ds, datavar_sea="northsea") assert (~ds_bathymetry.bathymetry.isnull()).sum() == 0 @@ -71,7 +72,7 @@ def test_get_bathymetry_nosea(): def test_add_bathymetry_to_top_bot_kh_kv_seamodel(): # model with sea - ds = test_001_model.get_ds_from_cache("basic_sea_model") + ds = test_001_model.get_ds_from_cache("sea_model_grid_only") ds.update(nlmod.read.rws.discretize_northsea(ds)) bathymetry = nlmod.read.jarkus.get_bathymetry(ds, datavar_sea="northsea") ds.update(bathymetry.drop_vars("time")) diff --git a/tests/test_005_external_data.py b/tests/test_005_external_data.py index dd1d2527..69892bce 100644 --- a/tests/test_005_external_data.py +++ b/tests/test_005_external_data.py @@ -12,10 +12,10 @@ def test_get_recharge(): # model with sea - ds = test_001_model.get_ds_from_cache("basic_sea_model") + ds = test_001_model.get_ds_from_cache("sea_model_grid_only") # add knmi recharge to the model dataset - ds.update(nlmod.read.knmi.get_recharge(ds)) + ds.update(nlmod.read.knmi.get_recharge(ds, hourly_precision=True)) def test_get_recharge_most_common(): @@ -24,19 +24,23 @@ def test_get_recharge_most_common(): ds = nlmod.time.set_ds_time(ds, start="2021", time=pd.date_range("2022", "2023")) # add knmi recharge to the model dataset - ds.update(nlmod.read.knmi.get_recharge(ds, most_common_station=True)) + ds.update( + nlmod.read.knmi.get_recharge( + ds, hourly_precision=True, most_common_station=True + ) + ) def test_get_recharge_steady_state(): # model with sea - ds = test_001_model.get_ds_from_cache("basic_sea_model") + ds = test_001_model.get_ds_from_cache("sea_model_grid_only") # modify mtime ds = ds.drop_dims("time") ds = nlmod.time.set_ds_time(ds, time=[3650], start="2000-1-1") # add knmi recharge to the model dataset - ds.update(nlmod.read.knmi.get_recharge(ds)) + ds.update(nlmod.read.knmi.get_recharge(ds, hourly_precision=True)) def test_get_recharge_not_available(): @@ -44,7 +48,7 @@ def test_get_recharge_not_available(): time = [pd.Timestamp.now().normalize()] ds = nlmod.time.set_ds_time(ds, start=time[0] - pd.Timedelta(days=21), time=time) with pytest.raises((ValueError)): - ds.update(nlmod.read.knmi.get_recharge(ds)) + ds.update(nlmod.read.knmi.get_recharge(ds, hourly_precision=True)) def test_get_recharge_add_stn_dimensions(): @@ -58,7 +62,9 @@ def test_get_recharge_add_stn_dimensions(): ds["active_domain"].data[0, 0] = False time = pd.date_range("2024", "2025") ds = nlmod.time.set_ds_time(ds, start="2023", time=time) - ds.update(nlmod.read.knmi.get_recharge(ds, add_stn_dimensions=True)) + ds.update( + nlmod.read.knmi.get_recharge(ds, hourly_precision=True, add_stn_dimensions=True) + ) # create simulation sim = nlmod.sim.sim(ds) @@ -100,6 +106,8 @@ def test_add_recharge_as_float(): gwf = nlmod.gwf.gwf(ds, sim) _ = nlmod.sim.ims(sim) _ = nlmod.gwf.dis(ds, gwf) + + # test with single recharge value _ = nlmod.gwf.rch(ds, gwf, recharge=0.1) spd = gwf.rch.stress_period_data.data @@ -108,6 +116,34 @@ def test_add_recharge_as_float(): assert (spd[0]["recharge"] == 0.1).all() +def test_add_recharge_as_ts(): + ds = nlmod.get_ds( + [100000, 110000, 420000, 430000], + model_ws=os.path.join("models", "test_add_recharge_as_ts"), + model_name="test", + ) + time = pd.date_range("2024", "2025") + ds = nlmod.time.set_ds_time(ds, start="2023", time=time) + + sim = nlmod.sim.sim(ds) + _ = nlmod.sim.tdis(ds, sim) + gwf = nlmod.gwf.gwf(ds, sim) + _ = nlmod.sim.ims(sim) + _ = nlmod.gwf.dis(ds, gwf) + + # test with recharge as time series + ds["precipitation"] = xr.full_like(ds["time"], 0.7e-3, dtype=float) + _ = nlmod.gwf.rch(ds, gwf, recharge="precipitation") + + spd = gwf.rch.stress_period_data.data + assert len(spd) == 1 + assert len(spd[0]) == 10000 + assert spd[0]["recharge"].dtype == object + + ts = gwf.rch.ts[0] + assert spd[0]["recharge"][0] in pd.DataFrame(ts.time_series_namerecord.array).values + + def test_ahn_within_extent(): extent = [104000.0, 105000.0, 494000.0, 494600.0] da = nlmod.read.ahn.download_latest_ahn_from_wcs(extent) @@ -170,7 +206,7 @@ def test_download_ahn6(): def test_get_ahn(): # model with sea - ds = test_001_model.get_ds_from_cache("basic_sea_model") + ds = test_001_model.get_ds_from_cache("sea_model_grid_only") # add ahn data to the model dataset ahn_ds = nlmod.read.ahn.get_ahn(ds) @@ -197,7 +233,7 @@ def test_check_ahn_files_up_to_date(): def test_get_surface_water_ghb(): # model with sea - ds = test_001_model.get_ds_from_cache("basic_sea_model") + ds = test_001_model.get_ds_from_cache("sea_model_grid_only") # create simulation sim = nlmod.sim.sim(ds) @@ -214,7 +250,14 @@ def test_get_surface_water_ghb(): nlmod.gwf.dis(ds, gwf) # add surface water levels to the model dataset - ds.update(nlmod.read.rws.get_surface_water(ds, da_basename="surface_water")) + gdf_surface_water = nlmod.read.rws.get_gdf_surface_water(ds) + ds.update( + nlmod.read.rws.discretize_surface_water( + ds, + gdf_surface_water, + da_basename="surface_water", + ) + ) def test_get_brp(): @@ -224,11 +267,11 @@ def test_get_brp(): # disable because slow (~35 seconds depending on internet connection) @pytest.mark.skip(reason="slow") -def test_get_bofek(): +def test_get_bofek(tmp_path): # model with sea - ds = test_001_model.get_ds_from_cache("basic_sea_model") + ds = test_001_model.get_ds_from_cache("sea_model_grid_only") # add knmi recharge to the model dataset - gdf_bofek = nlmod.read.bofek.download_bofek_gdf(ds) + gdf_bofek = nlmod.read.bofek.download_bofek_gdf(nlmod.grid.get_extent(ds), tmp_path) assert not gdf_bofek.empty, "Bofek geodataframe is empty" diff --git a/tests/test_006_caching.py b/tests/test_006_caching.py index c035cbf0..a2ab588e 100644 --- a/tests/test_006_caching.py +++ b/tests/test_006_caching.py @@ -9,49 +9,49 @@ def test_cache_ahn_data_array(): extent = [119_900, 120_000, 441_900, 442_000] cache_name = "ahn4.nc" - with tempfile.TemporaryDirectory() as tmpdir: - assert not os.path.exists( - os.path.join(tmpdir, cache_name) - ), "Cache should not exist yet1" + with tempfile.TemporaryDirectory() as tmp_path: + assert not os.path.exists(os.path.join(tmp_path, cache_name)), ( + "Cache should not exist yet1" + ) ahn_no_cache = nlmod.read.ahn.download_ahn4(extent) - assert not os.path.exists( - os.path.join(tmpdir, cache_name) - ), "Cache should not exist yet2" + assert not os.path.exists(os.path.join(tmp_path, cache_name)), ( + "Cache should not exist yet2" + ) ahn_cached = nlmod.read.ahn.download_ahn4( - extent, cachedir=tmpdir, cachename=cache_name + extent, cachedir=tmp_path, cachename=cache_name + ) + assert os.path.exists(os.path.join(tmp_path, cache_name)), ( + "Cache should have existed by now" ) - assert os.path.exists( - os.path.join(tmpdir, cache_name) - ), "Cache should have existed by now" assert ahn_cached.equals(ahn_no_cache) - modification_time1 = os.path.getmtime(os.path.join(tmpdir, cache_name)) + modification_time1 = os.path.getmtime(os.path.join(tmp_path, cache_name)) # Check if the cache is used. If not, cache is rewritten and modification time changes ahn_cache = nlmod.read.ahn.download_ahn4( - extent, cachedir=tmpdir, cachename=cache_name + extent, cachedir=tmp_path, cachename=cache_name ) assert ahn_cache.equals(ahn_no_cache) - modification_time2 = os.path.getmtime(os.path.join(tmpdir, cache_name)) + modification_time2 = os.path.getmtime(os.path.join(tmp_path, cache_name)) assert modification_time1 == modification_time2, "Cache should not be rewritten" # Different extent should not lead to using the cache extent = [119_800, 120_000, 441_900, 442_000] ahn_cache = nlmod.read.ahn.download_ahn4( - extent, cachedir=tmpdir, cachename=cache_name + extent, cachedir=tmp_path, cachename=cache_name + ) + modification_time3 = os.path.getmtime(os.path.join(tmp_path, cache_name)) + assert modification_time1 != modification_time3, ( + "Cache should have been rewritten" ) - modification_time3 = os.path.getmtime(os.path.join(tmpdir, cache_name)) - assert ( - modification_time1 != modification_time3 - ), "Cache should have been rewritten" # clear cache again - nlmod.cache.clear_cache(tmpdir, prompt=False) + nlmod.cache.clear_cache(tmp_path, prompt=False) def test_cache_northsea_data_array(): """Test caching of AHN data array. Does have dataset as argument.""" - from nlmod.read.rws import get_northsea + from nlmod.read.rws import discretize_northsea ds1 = nlmod.get_ds( [119_700, 120_000, 441_900, 442_000], @@ -74,39 +74,39 @@ def test_cache_northsea_data_array(): cache_name = "northsea.nc" - with tempfile.TemporaryDirectory() as tmpdir: - assert not os.path.exists( - os.path.join(tmpdir, cache_name) - ), "Cache should not exist yet1" - out1_no_cache = get_northsea(ds1) - assert not os.path.exists( - os.path.join(tmpdir, cache_name) - ), "Cache should not exist yet2" - - out1_cached = get_northsea(ds1, cachedir=tmpdir, cachename=cache_name) - assert os.path.exists( - os.path.join(tmpdir, cache_name) - ), "Cache should exist by now" + with tempfile.TemporaryDirectory() as tmp_path: + assert not os.path.exists(os.path.join(tmp_path, cache_name)), ( + "Cache should not exist yet1" + ) + out1_no_cache = discretize_northsea(ds1) + assert not os.path.exists(os.path.join(tmp_path, cache_name)), ( + "Cache should not exist yet2" + ) + + out1_cached = discretize_northsea(ds1, cachedir=tmp_path, cachename=cache_name) + assert os.path.exists(os.path.join(tmp_path, cache_name)), ( + "Cache should exist by now" + ) assert out1_cached.equals(out1_no_cache) - modification_time1 = os.path.getmtime(os.path.join(tmpdir, cache_name)) + modification_time1 = os.path.getmtime(os.path.join(tmp_path, cache_name)) # Check if the cache is used. If not, cache is rewritten and modification time changes - out1_cache = get_northsea(ds1, cachedir=tmpdir, cachename=cache_name) + out1_cache = discretize_northsea(ds1, cachedir=tmp_path, cachename=cache_name) assert out1_cache.equals(out1_no_cache) - modification_time2 = os.path.getmtime(os.path.join(tmpdir, cache_name)) + modification_time2 = os.path.getmtime(os.path.join(tmp_path, cache_name)) assert modification_time1 == modification_time2, "Cache should not be rewritten" # Only properties of `coords_2d` determine if the cache is used. Cache should still be used. ds1["toppertje"] = ds1.top + 1 - out1_cache = get_northsea(ds1, cachedir=tmpdir, cachename=cache_name) + out1_cache = discretize_northsea(ds1, cachedir=tmp_path, cachename=cache_name) assert out1_cache.equals(out1_no_cache) - modification_time2 = os.path.getmtime(os.path.join(tmpdir, cache_name)) + modification_time2 = os.path.getmtime(os.path.join(tmp_path, cache_name)) assert modification_time1 == modification_time2, "Cache should not be rewritten" # Different extent should not lead to using the cache - out2_cache = get_northsea(ds2, cachedir=tmpdir, cachename=cache_name) - modification_time3 = os.path.getmtime(os.path.join(tmpdir, cache_name)) - assert ( - modification_time1 != modification_time3 - ), "Cache should have been rewritten" + out2_cache = discretize_northsea(ds2, cachedir=tmp_path, cachename=cache_name) + modification_time3 = os.path.getmtime(os.path.join(tmp_path, cache_name)) + assert modification_time1 != modification_time3, ( + "Cache should have been rewritten" + ) assert not out2_cache.equals(out1_no_cache) diff --git a/tests/test_007_run_notebooks.py b/tests/test_007_run_notebooks.py index 93e1dab1..9f0104cd 100644 --- a/tests/test_007_run_notebooks.py +++ b/tests/test_007_run_notebooks.py @@ -1,111 +1,84 @@ -"""run notebooks in the examples directory.""" +"""Run all notebooks in the docs directory recursively.""" # ruff: noqa: D103 -import os +import re +from pathlib import Path import nbformat import pytest from nbconvert.preprocessors import ExecutePreprocessor -tst_dir = os.path.dirname(os.path.realpath(__file__)) -nbdir = os.path.join(tst_dir, "..", "docs", "examples") - - -def _run_notebook(nbdir, fname): - fname_nb = os.path.join(nbdir, fname) - with open(fname_nb) as f: +tst_dir = Path(__file__).resolve().parent +docs_dir = (tst_dir / ".." / "docs").resolve() + + +def _is_dated_notebook_copy(path): + return re.match(r"^\d{8,10}_", path.name) is not None + + +def _iter_notebooks(base_dir): + for path in sorted(base_dir.rglob("*.ipynb")): + rel_parts = path.relative_to(base_dir).parts + if rel_parts and rel_parts[0] == "build": + continue + if ".ipynb_checkpoints" in rel_parts: + continue + if _is_dated_notebook_copy(path): + continue + yield path + + +NOTEBOOKS = list(_iter_notebooks(docs_dir)) +NOTEBOOKS_BY_REL = {path.relative_to(docs_dir).as_posix(): path for path in NOTEBOOKS} + +# Some notebooks consume artifacts produced by other notebooks. +NOTEBOOK_DEPENDENCIES = { + "examples/09_schoonhoven.ipynb": ["data_sources/02_surface_water.ipynb"], + "examples/14_stromingen_example.ipynb": ["data_sources/02_surface_water.ipynb"], + "utilities/08_gis.ipynb": [ + "examples/01_basic_model.ipynb", + "examples/03_local_grid_refinement.ipynb", + ], + "utilities/13_plot_methods.ipynb": ["examples/09_schoonhoven.ipynb"], + "workflows/03_aggregating_surface_water.ipynb": [ + "data_sources/02_surface_water.ipynb" + ], + "workflows/10_modpath.ipynb": ["examples/03_local_grid_refinement.ipynb"], + "workflows/11_particle_tracking_prt.ipynb": [ + "examples/00_model_from_scratch.ipynb" + ], + "workflows/18_observations.ipynb": ["examples/03_local_grid_refinement.ipynb"], +} + +_EXECUTED_NOTEBOOKS = set() + + +def _run_notebook(path): + with path.open(encoding="utf-8") as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=6000) - out = ep.preprocess(nb, {"metadata": {"path": nbdir}}) - - return out - - -@pytest.mark.notebooks -def test_run_notebook_00_model_from_scratch(): - _run_notebook(nbdir, "00_model_from_scratch.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_01_basic_model(): - _run_notebook(nbdir, "01_basic_model.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_02_surface_water(): - _run_notebook(nbdir, "02_surface_water.ipynb") - + ep.preprocess(nb, {"metadata": {"path": str(path.parent)}}) -@pytest.mark.notebooks -def test_run_notebook_03_local_grid_refinement(): - _run_notebook(nbdir, "03_local_grid_refinement.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_04_modifying_layermodels(): - _run_notebook(nbdir, "04_modifying_layermodels.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_05_caching(): - _run_notebook(nbdir, "05_caching.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_06_gridding_vector_data(): - _run_notebook(nbdir, "06_gridding_vector_data.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_07_resampling(): - _run_notebook(nbdir, "07_resampling.ipynb") +def _run_with_dependencies(path): + rel = path.relative_to(docs_dir).as_posix() + for dep_rel in NOTEBOOK_DEPENDENCIES.get(rel, []): + dep_path = NOTEBOOKS_BY_REL.get(dep_rel) + if dep_path is None: + raise RuntimeError(f"Notebook dependency not found: {dep_rel}") + if dep_path not in _EXECUTED_NOTEBOOKS: + _run_with_dependencies(dep_path) -@pytest.mark.notebooks -def test_run_notebook_08_gis(): - _run_notebook(nbdir, "08_gis.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_09_schoonhoven(): - _run_notebook(nbdir, "09_schoonhoven.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_10_modpath(): - _run_notebook(nbdir, "10_modpath.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_11_grid_rotation(): - _run_notebook(nbdir, "11_grid_rotation.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_12_layer_generation(): - _run_notebook(nbdir, "12_layer_generation.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_13_plot_methods(): - _run_notebook(nbdir, "13_plot_methods.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_14_stromingen_example(): - _run_notebook(nbdir, "14_stromingen_example.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_15_geotop(): - _run_notebook(nbdir, "15_geotop.ipynb") - - -@pytest.mark.notebooks -def test_run_notebook_16_groundwater_transport(): - _run_notebook(nbdir, "16_groundwater_transport.ipynb") + if path not in _EXECUTED_NOTEBOOKS: + _run_notebook(path) + _EXECUTED_NOTEBOOKS.add(path) @pytest.mark.notebooks -def test_run_notebook_17_unsaturated_zone_flow(): - _run_notebook(nbdir, "17_unsaturated_zone_flow.ipynb") +@pytest.mark.parametrize( + "notebook_path", + NOTEBOOKS, + ids=[str(path.relative_to(docs_dir)) for path in NOTEBOOKS], +) +def test_run_notebook(notebook_path): + _run_with_dependencies(notebook_path) diff --git a/tests/test_008_waterboard.py b/tests/test_008_waterboard.py index c8c973a6..6eae67bf 100644 --- a/tests/test_008_waterboard.py +++ b/tests/test_008_waterboard.py @@ -1,9 +1,9 @@ import os -import tempfile import matplotlib import numpy as np import pytest +import util import nlmod @@ -25,29 +25,11 @@ def test_bgt_waterboards(): bgt = nlmod.gwf.surface_water.add_stages_from_waterboards(bgt, la=la) -def get_ahn_colormap(name="ahn", N=256): - colors = np.array( - [ - [0, 98, 177], # dark blue - [0, 156, 224], # medium blue - [115, 216, 255], # light blue - [128, 137, 0], # dark green - [164, 221, 0], # light green: center - [252, 220, 0], # yellow - [251, 158, 0], # orange - [211, 49, 21], # light red - [159, 5, 0], # dark red - ] - ) - cmap = matplotlib.colors.LinearSegmentedColormap.from_list(name, colors / 255, N=N) - return cmap - - @pytest.mark.skip("too slow") def test_download_level_areas( data_kind="level_areas", plot=True, save=True, figdir=r"..\docs\_static" ): - cachedir = os.path.join(tempfile.tempdir, "test_download_level_areas") + cachedir = os.path.join(util.get_model_data_dir(), "test_download_level_areas") if not os.path.isdir(cachedir): os.makedirs(cachedir) @@ -60,7 +42,12 @@ def test_download_level_areas( # xmin, ymin, xmax, ymax = waterboards.at[wb, "geometry"].bounds # extent = [xmin, xmax, ymin, ymax] gdf[wb] = nlmod.read.waterboard.download_data( - wb, data_kind, max_record_count=1000, cachedir=cachedir, cachename=wb + wb, + data_kind, + max_record_count=1000, + cachedir=cachedir, + cachename=wb, + # verify=False, ) except Exception as e: if str(e) == f"{data_kind} not available for {wb}": @@ -79,11 +66,11 @@ def test_download_level_areas( else: raise (Exception(f"Unknown data_kind: {data_kind}")) # plot the winter_stage and summer_stage - for column, label in zip(columns, labels): + for column, label in zip(columns, labels, strict=False): f, ax = nlmod.plot.get_map([9000, 279000, 304000, 623000], base=100000) waterboards.plot(edgecolor="k", facecolor="none", ax=ax) norm = matplotlib.colors.Normalize(-10.0, 20.0) - cmap = get_ahn_colormap() + cmap = nlmod.plot.get_ahn_colormap() for wb in waterboards.index: if wb in gdf: try: @@ -123,7 +110,7 @@ def get_extent(waterboards, wb, buffer=1000.0): # extent = [57000, 58000, 378000, 379000] return extent - cachedir = os.path.join(tempfile.tempdir, "test_download_watercourses") + cachedir = os.path.join(util.get_model_data_dir(), "test_download_watercourses") if not os.path.isdir(cachedir): os.makedirs(cachedir) diff --git a/tests/test_009_layers.py b/tests/test_009_layers.py index 98e94c82..09d86584 100644 --- a/tests/test_009_layers.py +++ b/tests/test_009_layers.py @@ -3,18 +3,23 @@ import matplotlib.pyplot as plt import numpy as np +import pytest import test_001_model +import util +import xarray as xr from pandas import DataFrame from shapely.geometry import LineString import nlmod from nlmod.plot import DatasetCrossSection +MODEL_DATA_ENV_VAR = "NLMOD_TEST_MODEL_DATA_DIR" + def get_regis_horstermeer(cachedir=None, cachename="regis_horstermeer"): extent = [131000, 136800, 471500, 475700] if cachedir is None: - cachedir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") + cachedir = os.path.join(util.get_model_data_dir(), "regis_cache") if not os.path.isdir(cachedir): os.makedirs(cachedir) regis = nlmod.read.download_regis(extent, cachedir=cachedir, cachename=cachename) @@ -23,7 +28,7 @@ def get_regis_horstermeer(cachedir=None, cachename="regis_horstermeer"): def get_regis_unstructured(): regis = get_regis_horstermeer() - cachedir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") + cachedir = os.path.join(util.get_model_data_dir(), "regis_cache") if not os.path.isdir(cachedir): os.makedirs(cachedir) return nlmod.grid.refine(regis, cachedir) @@ -398,3 +403,83 @@ def test_get_modellayers_indexer(): idxfull = nlmod.layers.get_modellayers_indexer(ds, df, full_output=True) assert (idxfull["modellayer_top"] == np.array([1, 4, 0])).all() assert (idxfull["modellayer_bot"] == np.array([2, 4, 4])).all() + + +@pytest.mark.parametrize( + "func", + [ + nlmod.layers.get_isosurface_1d, + nlmod.layers._get_isosurface_1d_numpy, + nlmod.layers._get_isosurface_1d_numba, + ], +) +def test_isosurface_1d(func): + left = np.nan + right = np.nan + # monotonic increasing with duplicates + da = np.array([10, 20, 30, 30, 40]) + z = np.array([-1, -2, -3, -4, -5]) + value = 30 + elev = func(da, z, value, left=left, right=right) + assert elev == -3.0 + + # non-monotonic with duplicates + da = np.array([10, 20, 30, 30, 10]) + z = np.array([-1, -2, -3, -4, -5]) + value = 25 + elev = func(da, z, value, left=left, right=right) + assert elev == -2.5 + + # negative of previous + elev = func(-da, -z, -value, left=left, right=right) + assert elev == 2.5 + + # check left and right limits + da = -np.array([10, 11, 10]) + z = -np.array([-1, -2, -3]) + value = 25 + left = 999 + right = -999 + elev = func(da, z, value, left=left, right=right) + assert elev == right + + da = -np.array([10, 11, 10]) + z = -np.array([-1, -2, -3]) + value = -25 + elev = func(da, z, value, left=left, right=right) + assert elev == left + + +def test_get_isosurface_numpy_with_broadcast_z_dims(): + layer = [0, 1, 2] + y = [0] + x = [0] + time = [0, 1] + + # z has no time dimension + z = xr.DataArray( + np.array([[[-1.0, -2.0, -3.0]]]), + dims=("y", "x", "layer"), + coords={"y": y, "x": x, "layer": layer}, + ) + + # da has time dimension and crosses value=15 between first two layers + da = xr.DataArray( + np.array( + [ + [[[10.0, 20.0, 30.0]]], + [[[10.0, 20.0, 30.0]]], + ] + ), + dims=("time", "y", "x", "layer"), + coords={"time": time, "y": y, "x": x, "layer": layer}, + ) + + iso = nlmod.layers.get_isosurface(da, z, 15.0, method="numpy") + + expected = xr.DataArray( + np.array([[[-1.5]], [[-1.5]]]), + dims=("time", "y", "x"), + coords={"time": time, "y": y, "x": x}, + ) + xr.testing.assert_allclose(iso, expected) diff --git a/tests/test_011_dcs.py b/tests/test_011_dcs.py index 1cdf9c63..c03848b7 100644 --- a/tests/test_011_dcs.py +++ b/tests/test_011_dcs.py @@ -13,6 +13,7 @@ def test_dcs_structured(): dcs.label_layers() dcs.plot_array(ds["kh"], alpha=0.5) dcs.plot_grid() + dcs.hatch_layers(ds.layer[::2]) def test_dcs_cs_model(): @@ -35,6 +36,7 @@ def test_dcs_vertex(): dcs.label_layers() dcs.plot_array(ds["kh"], alpha=0.5) dcs.plot_grid(vertical=False) + dcs.hatch_layers(ds.layer[::2]) def test_cross_section_utils(): diff --git a/tests/test_013_surface_water.py b/tests/test_013_surface_water.py index fc6d97e3..f8577d64 100644 --- a/tests/test_013_surface_water.py +++ b/tests/test_013_surface_water.py @@ -4,13 +4,16 @@ import geopandas as gpd import numpy as np import pandas as pd +import pytest +import util +from shapely.geometry import box import nlmod def get_ds_and_gdf(): model_name = "sw" - model_ws = os.path.join("data", model_name) + model_ws = os.path.join(util.get_model_data_dir(), model_name) extent = [119000, 120000, 523000, 524000] ds = nlmod.get_ds(extent, model_ws=model_ws, model_name=model_name) ds = nlmod.time.set_ds_time(ds, time=[365.0], start=pd.Timestamp.today()) @@ -50,7 +53,7 @@ def test_get_seaonal_timeseries(): def test_gdf_lake(): model_name = "la" - model_ws = os.path.join("data", model_name) + model_ws = os.path.join(util.get_model_data_dir(), model_name) ds = nlmod.get_ds( [170000, 171000, 550000, 551000], model_ws=model_ws, model_name=model_name ) @@ -132,3 +135,82 @@ def test_aggregate(): assert not celldata.isna().any(axis=None) riv_spd = nlmod.gwf.surface_water.build_spd(celldata, "RIV", ds) flopy.mf6.ModflowGwfriv(gwf, stress_period_data=riv_spd) + + +def _get_lake_model(model_name): + model_ws = os.path.join(util.get_model_data_dir(), model_name) + ds = nlmod.get_ds( + [170000, 171000, 550000, 551000], model_ws=model_ws, model_name=model_name + ) + ds = nlmod.time.set_ds_time(ds, time=[1], start=pd.Timestamp.today()) + ds = nlmod.dims.refine(ds) + + sim = nlmod.sim.sim(ds) + nlmod.sim.tdis(ds, sim) + nlmod.sim.ims(sim) + gwf = nlmod.gwf.gwf(ds, sim) + nlmod.gwf.dis(ds, gwf) + return ds, gwf + + +def test_lake_from_gdf_aggregates_pieces_per_cell(): + """Multiple pieces of a lake in one cell merge into a single connection. + + A lake commonly intersects a grid cell in multiple polygon pieces (e.g. after + nlmod.grid.gdf_to_grid). Each piece should contribute conductance for its own + area, and the pieces should collapse to a single lake-GWF connection per cell, + like nlmod.gwf.surface_water.aggregate already does for RIV/DRN celldata. + """ + ds, gwf = _get_lake_model("lp") + + # lake_0 covers cell 14 with two pieces (6000 m2 at clake=10 and 2000 m2 at + # clake=20) and cell 15 with one piece (10000 m2, the full cell, at clake=10) + gdf_lake = gpd.GeoDataFrame( + { + "name": ["lake_0", "lake_0", "lake_0"], + "strt": [1.0, 1.0, 1.0], + "clake": [10.0, 20.0, 10.0], + }, + geometry=[ + box(0, 0, 100, 60), + box(0, 60, 50, 100), + box(100, 0, 200, 100), + ], + index=[14, 14, 15], + ) + + lak = nlmod.gwf.lake_from_gdf(gwf, gdf_lake, ds, boundname_column="name") + + conns = lak.connectiondata.array + assert len(conns) == 2, "pieces within a cell should merge to one connection" + bedleak = { + cellid[-1]: bl + for (_, _, cellid, *_), bl in zip( + conns.tolist(), conns["bedleak"], strict=False + ) + } + # bedleak * cell_area == sum(piece_area / clake), cell area is 10000 m2 + assert bedleak[14] == pytest.approx((6000.0 / 10.0 + 2000.0 / 20.0) / 10000.0) + assert bedleak[15] == pytest.approx((10000.0 / 10.0) / 10000.0) + assert lak.packagedata.array[0][2] == 2 # nlakeconn counts cells, not pieces + + +def test_lake_from_gdf_accepts_floating_point_strt_noise(): + """Float-level noise in per-cell strt values is accepted as a single value. + + Aggregated inputs (e.g. area-weighted stages per cell) carry float-level noise; + a single-value check with exact equality rejects them for no physical reason. + """ + ds, gwf = _get_lake_model("ln") + + gdf_lake = gpd.GeoDataFrame( + { + "name": ["lake_0", "lake_0"], + "strt": [1.0, 1.0 + 1e-12], + "clake": [10.0, 10.0], + }, + index=[14, 15], + ) + + lak = nlmod.gwf.lake_from_gdf(gwf, gdf_lake, ds, boundname_column="name") + assert lak.packagedata.array[0][1] == pytest.approx(1.0) diff --git a/tests/test_014_gis.py b/tests/test_014_gis.py index 0ae4231f..5ab543d6 100644 --- a/tests/test_014_gis.py +++ b/tests/test_014_gis.py @@ -17,8 +17,8 @@ def test_vertex_da_to_gdf(): def test_ds_to_ugrid_nc_file(): ds = util.get_ds_vertex() - fname = os.path.join("data", "ugrid_test.nc") + fname = os.path.join(util.get_model_data_dir(), "ugrid_test.nc") nlmod.gis.ds_to_ugrid_nc_file(ds, fname) - fname = os.path.join("data", "ugrid_test_qgis.nc") + fname = os.path.join(util.get_model_data_dir(), "ugrid_test_qgis.nc") nlmod.gis.ds_to_ugrid_nc_file(ds, fname, for_imod_qgis_plugin=True) diff --git a/tests/test_015_gwf_output.py b/tests/test_015_gwf_output.py index 6e15cc09..c0d0474d 100644 --- a/tests/test_015_gwf_output.py +++ b/tests/test_015_gwf_output.py @@ -1,28 +1,200 @@ import os -import tempfile +import flopy as fp import numpy as np +import pandas as pd import pytest import test_001_model +import xarray as xr import nlmod from nlmod.dims.grid import refine from nlmod.gwf import get_budget_da, get_heads_da -tmpdir = tempfile.gettempdir() -tst_model_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") - grberror = "Cannot create budget data-array without grid information." -def test_create_small_model_grid_only(tmpdir, model_name="test"): +def get_ds(): + model_name = "test" + extent = [98700.0, 99000.0, 489500.0, 489700.0] + # extent, nrow, ncol = nlmod.read.regis.fit_extent_to_regis(extent, 100, 100) + regis_geotop_ds = nlmod.read.regis.get_combined_layer_models( + extent, regis_botm_layer="KRz5", use_regis=True, use_geotop=True + ) + model_ws = os.path.join("./data/mf6output/structured") + ds = nlmod.to_model_ds( + regis_geotop_ds, model_name, model_ws, delr=100.0, delc=100.0 + ) + return ds + + +def structured_model(): + ds = get_ds() + assert ds.sizes["layer"] == 5 + + ds = nlmod.time.set_ds_time(ds, time=[1, 2, 3], start="2015-1-1", steady=[1, 0, 0]) + + # create simulation + sim = nlmod.sim.sim(ds) + + # create time discretisation + _ = nlmod.sim.tdis(ds, sim) + + # create ims + nlmod.sim.ims(sim) + + # create groundwater flow model + gwf = nlmod.gwf.gwf(ds, sim) + + # Create discretization + nlmod.gwf.dis(ds, gwf) + + # create node property flow + nlmod.gwf.npf(ds, gwf, save_flows=True, save_specific_discharge=True) + + # Create the initial conditions package + nlmod.gwf.ic(ds, gwf, starting_head=1.0) + nlmod.gwf.oc(ds, gwf) + + ds.update(nlmod.grid.mask_model_edge(ds)) + nlmod.gwf.chd(ds, gwf, mask="edge_mask", head="starting_head") + + nlmod.sim.write_and_run(sim, ds) + + # delete all files except .cbc, .hds and .dis.grb + for file in os.listdir(ds.model_ws): + if not file.endswith((".cbc", ".hds", ".dis.grb")): + if os.path.isdir(os.path.join(ds.model_ws, file)): + os.rmdir(os.path.join(ds.model_ws, file)) + else: + os.remove(os.path.join(ds.model_ws, file)) + + +def vertex_model(): + ds = get_ds() + model_ws = os.path.join("./data/mf6output/vertex") + ds.attrs["model_ws"] = model_ws + # unstructured + dsv = refine( + ds, + model_ws=model_ws, + refinement_features=None, + exe_name=None, + remove_nan_layers=True, + model_coordinates=False, + ) + + dsv = nlmod.time.set_ds_time( + dsv, time=[1, 2, 3], start="2015-1-1", steady=[1, 0, 0] + ) + + # create simulation + sim = nlmod.sim.sim(dsv) + + # create time discretisation + _ = nlmod.sim.tdis(dsv, sim) + + # create ims + nlmod.sim.ims(sim) + + # create groundwater flow model + gwf = nlmod.gwf.gwf(dsv, sim) + + # Create discretization + nlmod.gwf.disv(dsv, gwf) + + # create node property flow + nlmod.gwf.npf(dsv, gwf, save_flows=True, save_specific_discharge=True) + + # Create the initial conditions package + nlmod.gwf.ic(dsv, gwf, starting_head=1.0) + nlmod.gwf.oc(dsv, gwf) + + dsv.update(nlmod.grid.mask_model_edge(dsv)) + nlmod.gwf.chd(dsv, gwf, mask="edge_mask", head="starting_head") + + nlmod.sim.write_and_run(sim, dsv) + + # delete all files except .cbc, .hds and .dis.grb + for file in os.listdir(dsv.model_ws): + if not file.endswith((".cbc", ".hds", ".disv.grb")): + if os.path.isdir(os.path.join(dsv.model_ws, file)): + os.rmdir(os.path.join(dsv.model_ws, file)) + else: + os.remove(os.path.join(dsv.model_ws, file)) + + +def model_unstructured(): + ds = get_ds() + model_ws = os.path.join("./data/mf6output/unstructured") + modelgrid = nlmod.grid.modelgrid_from_ds(ds, rotated=False, nlay=1) + g = fp.utils.gridgen.Gridgen(modelgrid, model_ws=model_ws, exe_name="gridgen") + g.build() + gridprops = g.get_gridprops_disu6() + sim = fp.mf6.MFSimulation(version="mf6", exe_name="mf6", sim_ws=model_ws) + fp.mf6.ModflowTdis(sim, time_units="DAYS", nper=3, perioddata=[[1.0, 1, 1.0]] * 3) + fp.mf6.ModflowIms(sim) + gwf = fp.mf6.ModflowGwf(sim, modelname=ds.model_name, save_flows=True) + fp.mf6.ModflowGwfdisu(gwf, **gridprops) + fp.mf6.ModflowGwfnpf(gwf, save_specific_discharge=True) + fp.mf6.ModflowGwfic(gwf, strt=1.0) + fp.mf6.ModflowGwfoc( + gwf, + head_filerecord=f"{ds.model_name}.hds", + budget_filerecord=f"{ds.model_name}.cbc", + saverecord=[("HEAD", "ALL"), ("BUDGET", "ALL")], + ) + sim.write_simulation() + sim.run_simulation() + + # delete all files except .cbc, .hds and .dis.grb + for file in os.listdir(model_ws): + if not file.endswith((".cbc", ".hds", ".disu.grb")): + if os.path.isdir(os.path.join(model_ws, file)): + os.rmdir(os.path.join(model_ws, file)) + else: + os.remove(os.path.join(model_ws, file)) + + +def model_voronoi(): + model_ws = "./data/mf6output/voronoi" + ds = nlmod.grid.modelgrid_to_ds(grbfile=os.path.join(model_ws, "voronoi.disv.grb")) + ds.attrs["exe_name"] = "mf6" + ds.attrs["model_name"] = "voronoi" + ds.attrs["model_ws"] = str(model_ws) + ds.attrs["mfversion"] = "mf6" + edge_mask = nlmod.grid.mask_model_edge(ds)["edge_mask"] + ds["kh"] = 10.0 * xr.ones_like(ds["botm"]) + ds["kv"] = 10.0 * xr.ones_like(ds["botm"]) + ds = nlmod.time.set_ds_time(ds, start="2020-01-01", perlen=[1.0]) + sim = nlmod.sim.sim(ds) + nlmod.sim.tdis(ds, sim) + nlmod.sim.ims(sim) + gwf = nlmod.gwf.gwf(ds, sim) + nlmod.gwf.disv(ds, gwf) + nlmod.gwf.npf(ds, gwf) + nlmod.gwf.ghb(ds, gwf, bhead=10 * edge_mask, cond=1e8 * edge_mask) + nlmod.gwf.ic(ds, gwf, starting_head=10.0) + df = pd.DataFrame(index=["well1"], columns=["x", "y", "top", "botm", "Q"]) + df.loc["well1"] = 1500, 500, 0.0, -10.0, -200.0 + nlmod.gwf.wells.wel_from_df(df, ds=ds, gwf=gwf) + nlmod.gwf.oc(ds, gwf) + nlmod.sim.write_and_run(sim, ds) + + # delete all files except .cbc, .hds and .disv.grb + for file in os.listdir(model_ws): + if not file.endswith((".cbc", ".hds", ".disv.grb")): + os.remove(os.path.join(model_ws, file)) + + +def test_create_small_model_grid_only(tmp_path, model_name="test"): model_name = "test" extent = [98700.0, 99000.0, 489500.0, 489700.0] # extent, nrow, ncol = nlmod.read.regis.fit_extent_to_regis(extent, 100, 100) regis_geotop_ds = nlmod.read.regis.get_combined_layer_models( extent, regis_botm_layer="KRz5", use_regis=True, use_geotop=True ) - model_ws = os.path.join(tmpdir, model_name) + model_ws = os.path.join(tmp_path, model_name) ds = nlmod.to_model_ds( regis_geotop_ds, model_name, model_ws, delr=100.0, delc=100.0 ) @@ -211,11 +383,11 @@ def test_get_budget_da_from_file_unstructured_no_grb(): def test_get_budget_da_from_file_unstructured_with_grb(): fname_cbc = "./tests/data/mf6output/unstructured/test.cbc" grb_file = "./tests/data/mf6output/unstructured/test.disu.grb" - nlmod.gwf.output.get_budget_da("GHB", fname=fname_cbc, grb_file=grb_file) + nlmod.gwf.output.get_budget_da("SPDIS", fname=fname_cbc, grb_file=grb_file) def test_postprocess_head(): - ds = test_001_model.get_ds_from_cache("basic_sea_model") + ds = test_001_model.get_ds_from_cache("sea_model") head = nlmod.gwf.get_heads_da(ds) nlmod.gwf.calculate_gxg(head) @@ -226,7 +398,7 @@ def test_postprocess_head(): def test_get_flow_residuals(): - ds = test_001_model.get_ds_from_cache("basic_sea_model") + ds = test_001_model.get_ds_from_cache("sea_model") da = nlmod.gwf.output.get_flow_residuals(ds) assert "time" in da.dims da = nlmod.gwf.output.get_flow_residuals(ds, kstpkper=(0, 0)) @@ -234,7 +406,7 @@ def test_get_flow_residuals(): def test_get_flow_lower_face(): - ds = test_001_model.get_ds_from_cache("basic_sea_model") + ds = test_001_model.get_ds_from_cache("sea_model") da = nlmod.gwf.output.get_flow_lower_face(ds) assert "time" in da.dims da = nlmod.gwf.output.get_flow_lower_face(ds, kstpkper=(0, 0)) diff --git a/tests/test_016_time.py b/tests/test_016_time.py index 9e780aed..c9766d33 100644 --- a/tests/test_016_time.py +++ b/tests/test_016_time.py @@ -41,7 +41,7 @@ def test_get_time_step_length(): def test_time_options(): """Attempt to list all the variations of start, time and perlen - caling the nlmod.dims.set_ds_time functions + caling the nlmod.dims.set_ds_time functions. """ ds = nlmod.get_ds([0, 1000, 2000, 3000]) @@ -91,7 +91,7 @@ def test_time_options(): def test_time_out_of_bounds(): - """Related to this issue: https://github.com/gwmod/nlmod/issues/374 + """Related to this issue: https://github.com/gwmod/nlmod/issues/374. pandas timestamps can only do computations with dates between the years 1678 and 2262. """ diff --git a/tests/test_018_knmi_data_platform.py b/tests/test_018_knmi_data_platform.py index c9d82a50..56569598 100644 --- a/tests/test_018_knmi_data_platform.py +++ b/tests/test_018_knmi_data_platform.py @@ -1,14 +1,13 @@ # ruff: noqa: D103 +import shutil from pathlib import Path from xarray import Dataset from nlmod.read import knmi_data_platform -data_path = Path(__file__).parent / "data" - -def test_download_multiple_nc_files() -> None: +def test_download_multiple_nc_files(tmp_path) -> None: dataset_name = "EV24" dataset_version = "2" @@ -24,11 +23,10 @@ def test_download_multiple_nc_files() -> None: # download the first file fnames = files[0:1] - dirname = "download" knmi_data_platform.download_files( - dataset_name, dataset_version, fnames, dirname=dirname + dataset_name, dataset_version, fnames, dirname=tmp_path ) - file = Path(dirname) / fnames[0] + file = tmp_path / fnames[0] assert file.exists(), f"File {file} was not downloaded properly" ds = knmi_data_platform.read_nc(file) @@ -37,7 +35,7 @@ def test_download_multiple_nc_files() -> None: print(f"Error in knmi_data_platform test: {e}") -def test_download_read_zip_file() -> None: +def test_download_read_zip_file(tmp_path) -> None: dataset_name = "rad_nl25_rac_mfbs_24h_netcdf4" dataset_version = "2.0" try: @@ -46,35 +44,40 @@ def test_download_read_zip_file() -> None: assert len(files) > 0, "No files found" # download the last file - dirname = "download" fname = files[1] knmi_data_platform.download_file( - dataset_name, dataset_version, fname=fname, dirname=dirname + dataset_name, dataset_version, fname=fname, dirname=tmp_path ) - file = Path(dirname) / fname + file = tmp_path / fname assert file.exists(), f"File {file} was not downloaded properly" except knmi_data_platform.KNMIDataPlatformError as e: print(f"Error in knmi_data_platform test: {e}") -def test_read_zip_file() -> None: - fname = data_path / "KNMI_Data_Platform_NETCDF.zip" +def test_read_zip_file(tmp_path) -> None: + src = Path(__file__).resolve().parent / "data" / "KNMI_Data_Platform_NETCDF.zip" + fname = tmp_path / src.name + shutil.copy2(src, fname) try: - _ = knmi_data_platform.read_dataset_from_zip(str(fname), hour=24) + _ = knmi_data_platform.read_dataset_from_zip(fname, hour=24) except RuntimeError: # allow RuntimeError for this test for now (2025-11-19) # locally this fail does not happen, and we cannot recreate it pass -def test_read_h5() -> None: - fname = data_path / "KNMI_Data_Platform_H5.zip" - _ = knmi_data_platform.read_dataset_from_zip(str(fname)) +def test_read_h5(tmp_path) -> None: + src = Path(__file__).resolve().parent / "data" / "KNMI_Data_Platform_H5.zip" + fname = tmp_path / src.name + shutil.copy2(src, fname) + _ = knmi_data_platform.read_dataset_from_zip(fname) -def test_read_grib() -> None: - fname = data_path / "KNMI_Data_Platform_GRIB.tar" +def test_read_grib(tmp_path) -> None: + src = Path(__file__).resolve().parent / "data" / "KNMI_Data_Platform_GRIB.tar" + fname = tmp_path / src.name + shutil.copy2(src, fname) _ = knmi_data_platform.read_dataset_from_zip( - str(fname), + fname, filter_by_keys={"stepType": "instant", "typeOfLevel": "heightAboveGround"}, ) diff --git a/tests/test_019_attributes_encodings.py b/tests/test_019_attributes_encodings.py index 03333f9d..aac2c5fb 100644 --- a/tests/test_019_attributes_encodings.py +++ b/tests/test_019_attributes_encodings.py @@ -17,10 +17,10 @@ def test_encodings_float_as_int16(): por_data[1] = np.nan ds = xr.Dataset( - data_vars=dict( - heads=xr.DataArray(data=heads_data), - porosity=xr.DataArray(data=por_data), - ) + data_vars={ + "heads": xr.DataArray(data=heads_data), + "porosity": xr.DataArray(data=por_data), + } ) encodings = get_encodings( ds, set_encoding_inplace=False, allowed_to_read_data_vars_for_minmax=True @@ -30,8 +30,8 @@ def test_encodings_float_as_int16(): assert encodings["porosity"]["dtype"] == "int16", "dtype should be int16" # test writing to temporary netcdf file - with TemporaryDirectory() as tmpdir: - fp_test = os.path.join(tmpdir, "test2.nc") + with TemporaryDirectory() as tmp_path: + fp_test = os.path.join(tmp_path, "test2.nc") ds.to_netcdf(fp_test, encoding=encodings) with xr.open_dataset(fp_test, mask_and_scale=True) as ds2: @@ -48,7 +48,7 @@ def test_encodings_float_as_int16(): data = np.arange(1.0, 1e6) data[1] = np.nan - ds = xr.Dataset(data_vars=dict(heads=xr.DataArray(data=data))) + ds = xr.Dataset(data_vars={"heads": xr.DataArray(data=data)}) encodings = get_encodings( ds, set_encoding_inplace=False, allowed_to_read_data_vars_for_minmax=True ) @@ -62,7 +62,7 @@ def test_encondings_inplace(): data = np.arange(1.0, 5.0) data[1] = np.nan - ds = xr.Dataset(data_vars=dict(heads=xr.DataArray(data=data))) + ds = xr.Dataset(data_vars={"heads": xr.DataArray(data=data)}) ds_inplace = ds.copy(deep=True) encodings = get_encodings( @@ -75,14 +75,14 @@ def test_encondings_inplace(): ) # test writing to temporary netcdf file - with TemporaryDirectory() as tmpdir: - fp_test = os.path.join(tmpdir, "test2.nc") + with TemporaryDirectory() as tmp_path: + fp_test = os.path.join(tmp_path, "test2.nc") ds.to_netcdf(fp_test, encoding=encodings) with xr.open_dataset(fp_test, mask_and_scale=True) as ds2: ds2.load() - fp_test_inplace = os.path.join(tmpdir, "test_inplace.nc") + fp_test_inplace = os.path.join(tmp_path, "test_inplace.nc") ds_inplace.to_netcdf(fp_test_inplace) with xr.open_dataset(fp_test_inplace, mask_and_scale=True) as ds_inplace2: diff --git a/tests/test_020_uzf.py b/tests/test_020_uzf.py index 89f67b21..aea4718f 100644 --- a/tests/test_020_uzf.py +++ b/tests/test_020_uzf.py @@ -13,7 +13,9 @@ def test_uzf_structured(): time = pd.date_range("2022", "2023", freq="D") ds = nlmod.time.set_ds_time(ds, start="2021", time=time) - ds.update(nlmod.read.knmi.get_recharge(ds, method="separate")) + ds.update( + nlmod.read.knmi.get_recharge(ds, hourly_precision=True, method="separate") + ) # generate sim and gwf # create simulation diff --git a/tests/test_021_nhi.py b/tests/test_021_nhi.py index 2ea38a10..a0723c35 100644 --- a/tests/test_021_nhi.py +++ b/tests/test_021_nhi.py @@ -1,46 +1,48 @@ # ruff: noqa: D103 import os -import tempfile import geopandas as gpd import matplotlib.pyplot as plt import numpy as np -import requests import pytest +import requests import nlmod -tmpdir = tempfile.gettempdir() @pytest.mark.slow -def test_buisdrainage(): - model_ws = os.path.join(tmpdir, "buidrain") - ds = nlmod.get_ds([110_000, 130_000, 435_000, 445_000], model_ws=model_ws) - ds = nlmod.read.nhi.add_buisdrainage(ds) +def test_buisdrainage(tmp_path): + try: + model_ws = os.path.join(tmp_path, "buisdrain") + ds = nlmod.get_ds([110_000, 130_000, 435_000, 445_000], model_ws=model_ws) + ds.update(nlmod.read.nhi.discretize_buisdrainage(ds)) - # assert that all locations with a specified depth also have a positive conductance - mask = ~ds["buisdrain_depth"].isnull() - assert np.all(ds["buisdrain_cond"].data[mask] > 0) + # assert that all locations with a specified depth also have a positive conductance + mask = ~ds["buisdrain_depth"].isnull() + assert np.all(ds["buisdrain_cond"].data[mask] > 0) - # assert that all locations with a positive conductance also have a specified depth - mask = ds["buisdrain_cond"] > 0 - assert np.all(~np.isnan(ds["buisdrain_depth"].data[mask])) + # assert that all locations with a positive conductance also have a specified depth + mask = ds["buisdrain_cond"] > 0 + assert np.all(~np.isnan(ds["buisdrain_depth"].data[mask])) + except requests.exceptions.RequestException as e: + pytest.skip(f"Buisdrainage URL unavailable: {e}") +@pytest.mark.skip("NHI GWO Database is down until further notice.") def test_gwo(): username = os.environ["NHI_GWO_USERNAME"] password = os.environ["NHI_GWO_PASSWORD"] # download all wells from Brabant Water try: - wells = nlmod.read.nhi.get_gwo_wells( + wells = nlmod.read.nhi.download_gwo_wells( username=username, password=password, organisation="Brabant Water" ) assert isinstance(wells, gpd.GeoDataFrame) # download extractions from well "13-PP016" of pomping station Veghel - measurements, gdf = nlmod.read.nhi.get_gwo_measurements( + measurements, gdf = nlmod.read.nhi.download_gwo_measurements( username, password, well_site="veghel", filter__well__name="13-PP016" ) assert measurements.reset_index()["Name"].isin(gdf.index).all() @@ -59,7 +61,7 @@ def allow_network_fail(e): def test_gwo_entire_pumping_station(): username = os.environ["NHI_GWO_USERNAME"] password = os.environ["NHI_GWO_PASSWORD"] - measurements, gdf = nlmod.read.nhi.get_gwo_measurements( + measurements, gdf = nlmod.read.nhi.download_gwo_measurements( username, password, well_site="veghel", @@ -69,11 +71,15 @@ def test_gwo_entire_pumping_station(): ncols = 3 nrows = int(np.ceil(len(gdf.index) / ncols)) f, axes = plt.subplots( - nrows=nrows, ncols=ncols, figsize=(10, 10), sharex=True, sharey=True + nrows=nrows, + ncols=ncols, + figsize=(10, 10), + sharex=True, + sharey=True, + layout="constrained", ) axes = axes.ravel() - for name, ax in zip(gdf.index, axes): + for name, ax in zip(gdf.index, axes, strict=False): measurements.loc[name, "Volume"].plot(ax=ax) ax.set_xlabel("") ax.set_title(name) - f.tight_layout(pad=0.0) diff --git a/tests/test_022_gwt.py b/tests/test_022_gwt.py index 09551bab..1647b90d 100644 --- a/tests/test_022_gwt.py +++ b/tests/test_022_gwt.py @@ -1,7 +1,7 @@ import os -import tempfile import pandas as pd +import util import xarray as xr import nlmod @@ -10,9 +10,8 @@ def test_gwt_model(): extent = [103700, 106700, 527500, 528500] - tmpdir = tempfile.gettempdir() model_name = "trnsprt_tst" - model_ws = os.path.join(tmpdir, model_name) + model_ws = os.path.join(util.get_model_data_dir(), model_name) layer_model = nlmod.read.download_regis(extent, botm_layer="MSz1") # create a model ds @@ -42,7 +41,9 @@ def test_gwt_model(): ds["sea"] = nlmod.read.rws.calculate_sea_coverage(ahn, ds=ds, method="average") # download knmi recharge data - knmi_ds = nlmod.read.knmi.get_recharge(ds, method="separate") + knmi_ds = nlmod.read.knmi.get_recharge( + ds, method="separate", hourly_precision=False + ) # update model dataset ds.update(knmi_ds) @@ -136,12 +137,8 @@ def test_gwt_model(): nlmod.gwt.get_concentration_at_gw_surface(c) # test isosurface: first elevation where 10_000 mg/l is reached - z = xr.DataArray( - gwf.modelgrid.zcellcenters, - coords={"layer": c.layer, "y": c.y, "x": c.x}, - dims=("layer", "y", "x"), - ) - nlmod.layers.get_isosurface(c, z, 10_000.0) + z = nlmod.dims.layers.get_zcellcenters(ds) + nlmod.layers.get_isosurface(c, z, 10_000.0, method="numpy") # Convert calculated heads to equivalent freshwater heads, and vice versa hf = nlmod.gwt.output.freshwater_head(ds, h, c) diff --git a/tests/test_023_hfb.py b/tests/test_023_hfb.py index 26754b57..11933322 100644 --- a/tests/test_023_hfb.py +++ b/tests/test_023_hfb.py @@ -1,20 +1,129 @@ # ruff: noqa: D103 +import matplotlib +import pytest + +matplotlib.use("Agg") + import flopy import geopandas as gpd +import numpy as np +import pandas as pd import util -from shapely.geometry import LineString, Polygon +import xarray as xr +from flopy.utils import make_hfb_array +from shapely.geometry import LineString, MultiLineString, Point, Polygon import nlmod +from nlmod.dims.grid import modelgrid_from_ds +from nlmod.dims.layers import get_idomain + + +def _normalize_spd(spd): + return [ + (tuple(int(i) for i in cellid1), tuple(int(i) for i in cellid2), float(hydchr)) + for cellid1, cellid2, hydchr in spd + ] + + +def _expected_partial_hydchr(ds, cellid1, cellid2, hydchr, frac): + # negative HYDCHR factor for a layer penetrated over frac + d1, d2 = nlmod.gwf.hfb._center_to_face_distances(ds, cellid1, cellid2) + kh1 = ds["kh"].values[cellid1] + kh2 = ds["kh"].values[cellid2] + conductance = 1.0 / (d1 / kh1 + d2 / kh2) + return -float((1.0 - frac) + frac * hydchr / (conductance + hydchr)) + + +def _structured_diagonal_expected_spd(ds, hydchr=1 / 100.0): + expected_spd = [] + for ilay in (0, 1, 2): + for idx in range(9): + for cellid1, cellid2 in ( + ((ilay, idx, idx), (ilay, idx + 1, idx)), + ((ilay, idx + 1, idx + 1), (ilay, idx + 1, idx)), + ): + if ilay == 2: + # barrier penetrates half of the bottom layer + layer_hydchr = _expected_partial_hydchr( + ds, cellid1, cellid2, hydchr, 0.5 + ) + else: + layer_hydchr = hydchr + expected_spd.append((cellid1, cellid2, layer_hydchr)) + return expected_spd + + +def _flopy_pair_set(ds, geometry, layer=0): + modelgrid = modelgrid_from_ds(ds, idomain=get_idomain(ds).values) + return { + frozenset( + ( + tuple(int(i) for i in row.cellid1), + tuple(int(i) for i in row.cellid2), + ) + ) + for row in make_hfb_array(modelgrid, geometry) + if int(row.cellid1[0]) == layer + } + + +def _flopy_depth5_top_layer_spd(ds, geometries, hydchr): + if not isinstance(geometries, (list, tuple)): + geometries = [geometries] + modelgrid = modelgrid_from_ds(ds, idomain=get_idomain(ds).values) + expected_spd = [] + for geometry in geometries: + for row in make_hfb_array(modelgrid, geometry): + cellid1 = tuple(int(i) for i in row.cellid1) + cellid2 = tuple(int(i) for i in row.cellid2) + if cellid1[0] == 0: + # depth 5.0 penetrates half of the 10-m top layer + expected_spd.append( + ( + cellid1, + cellid2, + _expected_partial_hydchr(ds, cellid1, cellid2, hydchr, 0.5), + ) + ) + return expected_spd + + +def test_flopy_make_hfb_array_available(): + assert callable(make_hfb_array) + + +def test_get_hfb_spd_uses_flopy_make_hfb_array(monkeypatch): + ds = util.get_ds_structured() + gdf = gpd.GeoDataFrame({"geometry": [LineString([(0, 1000), (1000, 0)])]}) + calls = [] + original_make_hfb_array = nlmod.gwf.hfb.make_hfb_array + + def spy_make_hfb_array(modelgrid, geometry): + calls.append((modelgrid, geometry)) + return original_make_hfb_array(modelgrid, geometry) + + monkeypatch.setattr(nlmod.gwf.hfb, "make_hfb_array", spy_make_hfb_array) + + nlmod.gwf.hfb.get_hfb_spd(ds, gdf, hydchr=1 / 100.0, depth=5.0) + + assert len(calls) == 1 def test_get_hfb_spd_vertex(): - # this test also tests line_to_hfb ds = util.get_ds_vertex() ds = nlmod.time.set_ds_time(ds, "2023", time="2024") gwf = util.get_gwf(ds) - coords = [(0, 1000), (1000, 0)] - gdf = gpd.GeoDataFrame({"geometry": [LineString(coords)]}) + geometry = LineString([(0, 1000), (1000, 0)]) + gdf = gpd.GeoDataFrame({"geometry": [geometry]}) spd = nlmod.gwf.hfb.get_hfb_spd(ds, gdf, hydchr=1 / 100.0, depth=5.0) + assert { + frozenset((cellid1, cellid2)) for cellid1, cellid2, _ in _normalize_spd(spd) + } == _flopy_pair_set(ds, geometry) + # depth 5.0 penetrates half of the 10-m top layer + for cellid1, cellid2, hydchr_value in _normalize_spd(spd): + assert hydchr_value == _expected_partial_hydchr( + ds, cellid1, cellid2, 1 / 100.0, 0.5 + ) hfb = flopy.mf6.ModflowGwfhfb(gwf, stress_period_data={0: spd}) # also test the plot method ax = gdf.plot() @@ -22,19 +131,362 @@ def test_get_hfb_spd_vertex(): def test_get_hfb_spd_structured(): - # this test also tests line_to_hfb ds = util.get_ds_structured() ds = nlmod.time.set_ds_time(ds, "2023", time="2024") gwf = util.get_gwf(ds) coords = [(0, 1000), (1000, 0)] gdf = gpd.GeoDataFrame({"geometry": [LineString(coords)]}) - spd = nlmod.gwf.hfb.get_hfb_spd(ds, gdf, hydchr=1 / 100.0, depth=5.0) + spd = nlmod.gwf.hfb.get_hfb_spd(ds, gdf, hydchr=1 / 100.0, depth=25.0) + assert _normalize_spd(spd) == _structured_diagonal_expected_spd(ds) + elevation_spd = nlmod.gwf.hfb.get_hfb_spd( + ds, gdf, hydchr=1 / 100.0, elevation=-25.0 + ) + assert _normalize_spd(elevation_spd) == _structured_diagonal_expected_spd(ds) hfb = flopy.mf6.ModflowGwfhfb(gwf, stress_period_data={0: spd}) # also test the plot method ax = gdf.plot() nlmod.gwf.hfb.plot_hfb(hfb, gwf, ax=ax) +def test_get_hfb_spd_skips_inactive_and_passthrough_cells(): + ds = util.get_ds_structured() + ds["botm"].values[1] = ds["botm"].values[0] + active_domain = ds["top"] == ds["top"] + active_domain.values[0, 0] = False + ds["active_domain"] = active_domain + idomain = get_idomain(ds) + geometry = LineString([(0, 1000), (1000, 0)]) + gdf = gpd.GeoDataFrame({"geometry": [geometry]}) + + spd = _normalize_spd( + nlmod.gwf.hfb.get_hfb_spd(ds, gdf, hydchr=1 / 100.0, depth=25.0) + ) + expected_spd = [] + for ilay in (0, 2): + for idx in range(9): + for cellid1, cellid2 in ( + ((ilay, idx, idx), (ilay, idx + 1, idx)), + ((ilay, idx + 1, idx + 1), (ilay, idx + 1, idx)), + ): + if ilay == 2: + # with layer 1 collapsed, depth 25.0 penetrates 15 of the + # 20 m of layer 2 + layer_hydchr = _expected_partial_hydchr( + ds, cellid1, cellid2, 1 / 100.0, 0.75 + ) + else: + layer_hydchr = 1 / 100.0 + expected_spd.append((cellid1, cellid2, layer_hydchr)) + expected_spd = [ + row for row in expected_spd if row[0][1:] != (0, 0) and row[1][1:] != (0, 0) + ] + + assert spd == expected_spd + assert all( + idomain.values[cellid1] > 0 and idomain.values[cellid2] > 0 + for cellid1, cellid2, _ in spd + ) + assert all(cellid1[0] != 1 and cellid2[0] != 1 for cellid1, cellid2, _ in spd) + assert all( + cellid1[1:] != (0, 0) and cellid2[1:] != (0, 0) for cellid1, cellid2, _ in spd + ) + + +def test_hfb_from_df_vertex(): + ds = util.get_ds_vertex() + ds = nlmod.time.set_ds_time(ds, "2023", time="2024") + gwf = util.get_gwf(ds) + gdf = gpd.GeoDataFrame( + { + "hydchr": [1 / 100.0, 1 / 200.0], + "depth": [5.0, None], + "elevation": [None, -2.0], + "geometry": [ + LineString([(0, 1000), (1000, 0)]), + LineString([(100, 1000), (1000, 100)]), + ], + }, + ) + + hfb = nlmod.gwf.hfb.hfb_from_df(gdf, gwf, ds, pname="hfb_test") + + assert hfb.package_name == "hfb_test" + expected_spd = [] + for row_number, row in enumerate(gdf.itertuples()): + if pd.notna(row.depth): + expected_spd += nlmod.gwf.hfb.get_hfb_spd( + ds, + gdf.iloc[[row_number]], + hydchr=row.hydchr, + depth=row.depth, + ) + else: + expected_spd += nlmod.gwf.hfb.get_hfb_spd( + ds, + gdf.iloc[[row_number]], + hydchr=row.hydchr, + elevation=row.elevation, + ) + expected_spd = [ + [cellid1, cellid2, float(hydchr)] + for cellid1, cellid2, hydchr in expected_spd + if float(hydchr) != 0 + ] + actual_spd = [ + [row[0], row[1], float(row[2])] for row in hfb.stress_period_data.data[0] + ] + assert actual_spd == expected_spd + + +def test_hfb_from_df_requires_depth_or_elevation(): + ds = util.get_ds_vertex() + ds = nlmod.time.set_ds_time(ds, "2023", time="2024") + gwf = util.get_gwf(ds) + gdf = gpd.GeoDataFrame( + { + "hydchr": [1 / 100.0], + "depth": [None], + "elevation": [None], + "geometry": [LineString([(0, 1000), (1000, 0)])], + }, + ) + + with pytest.raises(ValueError, match="Exactly one"): + nlmod.gwf.hfb.hfb_from_df(gdf, gwf, ds) + + gdf.loc[0, "depth"] = 5.0 + gdf.loc[0, "elevation"] = -2.0 + with pytest.raises(ValueError, match="Exactly one"): + nlmod.gwf.hfb.hfb_from_df(gdf, gwf, ds) + + +@pytest.mark.parametrize("hydchr", [0.0, -0.01]) +def test_hfb_from_df_rejects_nonpositive_hydchr(hydchr): + ds = util.get_ds_vertex() + ds = nlmod.time.set_ds_time(ds, "2023", time="2024") + gwf = util.get_gwf(ds) + gdf = gpd.GeoDataFrame( + { + "hydchr": [hydchr], + "depth": [5.0], + "geometry": [LineString([(0, 1000), (1000, 0)])], + }, + ) + + with pytest.raises(ValueError, match="hydchr must be positive"): + nlmod.gwf.hfb.hfb_from_df(gdf, gwf, ds, elevation=None) + + +def test_hfb_from_df_returns_none_when_all_hydchr_rows_are_zero(): + ds = util.get_ds_vertex() + ds = nlmod.time.set_ds_time(ds, "2023", time="2024") + gwf = util.get_gwf(ds) + gdf = gpd.GeoDataFrame( + { + "hydchr": [1 / 100.0], + "depth": [0.0], + "geometry": [LineString([(0, 1000), (1000, 0)])], + }, + ) + + assert nlmod.gwf.hfb.hfb_from_df(gdf, gwf, ds, elevation=None) is None + + +def test_hfb_from_df_rejects_non_line_geometries(): + ds = util.get_ds_vertex() + ds = nlmod.time.set_ds_time(ds, "2023", time="2024") + gwf = util.get_gwf(ds) + gdf = gpd.GeoDataFrame( + { + "hydchr": [1 / 100.0], + "depth": [5.0], + "geometry": [Point(0, 1000)], + }, + ) + + with pytest.raises(ValueError, match="LineString"): + nlmod.gwf.hfb.hfb_from_df(gdf, gwf, ds, elevation=None) + + +def test_hfb_from_df_accepts_scalar_values(): + ds = util.get_ds_vertex() + ds = nlmod.time.set_ds_time(ds, "2023", time="2024") + gwf = util.get_gwf(ds) + gdf = gpd.GeoDataFrame({"geometry": [LineString([(0, 1000), (1000, 0)])]}) + + hfb = nlmod.gwf.hfb.hfb_from_df( + gdf, + gwf, + ds, + hydchr=1 / 100.0, + depth=5.0, + elevation=None, + ) + + expected_spd = nlmod.gwf.hfb.get_hfb_spd( + ds, + gdf, + hydchr=1 / 100.0, + depth=5.0, + ) + expected_spd = [ + [cellid1, cellid2, float(hydchr)] + for cellid1, cellid2, hydchr in expected_spd + if float(hydchr) != 0 + ] + actual_spd = [ + [row[0], row[1], float(row[2])] for row in hfb.stress_period_data.data[0] + ] + assert actual_spd == expected_spd + + +def test_hfb_from_df_scalar_values_apply_to_all_features(): + ds = util.get_ds_vertex() + ds = nlmod.time.set_ds_time(ds, "2023", time="2024") + gwf = util.get_gwf(ds) + gdf = gpd.GeoDataFrame( + { + "geometry": [ + LineString([(0, 1000), (1000, 0)]), + LineString([(100, 1000), (1000, 100)]), + ], + }, + ) + + hfb = nlmod.gwf.hfb.hfb_from_df( + gdf, + gwf, + ds, + hydchr=1 / 100.0, + depth=5.0, + elevation=None, + ) + + expected_spd = [] + for row_number in range(len(gdf)): + expected_spd += nlmod.gwf.hfb.get_hfb_spd( + ds, + gdf.iloc[[row_number]], + hydchr=1 / 100.0, + depth=5.0, + ) + expected_spd = [ + [cellid1, cellid2, float(hydchr)] + for cellid1, cellid2, hydchr in expected_spd + if float(hydchr) != 0 + ] + actual_spd = [ + [row[0], row[1], float(row[2])] for row in hfb.stress_period_data.data[0] + ] + assert actual_spd == expected_spd + + +def test_hfb_from_df_accepts_custom_column_names(): + ds = util.get_ds_vertex() + ds = nlmod.time.set_ds_time(ds, "2023", time="2024") + gwf = util.get_gwf(ds) + gdf = gpd.GeoDataFrame( + { + "resistance_inverse": [1 / 100.0], + "barrier_depth": [5.0], + "geometry": [LineString([(0, 1000), (1000, 0)])], + }, + ) + + hfb = nlmod.gwf.hfb.hfb_from_df( + gdf, + gwf, + ds, + hydchr="resistance_inverse", + depth="barrier_depth", + elevation=None, + ) + + expected_spd = nlmod.gwf.hfb.get_hfb_spd( + ds, + gdf, + hydchr=1 / 100.0, + depth=5.0, + ) + expected_spd = [ + [cellid1, cellid2, float(hydchr)] + for cellid1, cellid2, hydchr in expected_spd + if float(hydchr) != 0 + ] + actual_spd = [ + [row[0], row[1], float(row[2])] for row in hfb.stress_period_data.data[0] + ] + assert actual_spd == expected_spd + + +def test_hfb_from_df_accepts_multilinestring(): + ds = util.get_ds_vertex() + ds = nlmod.time.set_ds_time(ds, "2023", time="2024") + gwf = util.get_gwf(ds) + gdf = gpd.GeoDataFrame( + { + "geometry": [ + MultiLineString( + [ + [(0, 1000), (500, 500)], + [(500, 500), (1000, 0)], + ], + ), + ], + }, + ) + + hfb = nlmod.gwf.hfb.hfb_from_df( + gdf, + gwf, + ds, + hydchr=1 / 100.0, + depth=5.0, + elevation=None, + ) + + expected_spd = _flopy_depth5_top_layer_spd( + ds, list(gdf.geometry.iloc[0].geoms), 1 / 100.0 + ) + actual_spd = [ + (tuple(row[0]), tuple(row[1]), float(row[2])) + for row in hfb.stress_period_data.data[0] + ] + assert actual_spd == expected_spd + + +def test_hfb_from_df_accepts_scalar_elevation(): + ds = util.get_ds_vertex() + ds = nlmod.time.set_ds_time(ds, "2023", time="2024") + gwf = util.get_gwf(ds) + gdf = gpd.GeoDataFrame({"geometry": [LineString([(0, 1000), (1000, 0)])]}) + + hfb = nlmod.gwf.hfb.hfb_from_df( + gdf, + gwf, + ds, + hydchr=1 / 200.0, + depth=None, + elevation=-2.0, + ) + + expected_spd = nlmod.gwf.hfb.get_hfb_spd( + ds, + gdf, + hydchr=1 / 200.0, + elevation=-2.0, + ) + expected_spd = [ + [cellid1, cellid2, float(hydchr)] + for cellid1, cellid2, hydchr in expected_spd + if float(hydchr) != 0 + ] + actual_spd = [ + [row[0], row[1], float(row[2])] for row in hfb.stress_period_data.data[0] + ] + assert actual_spd == expected_spd + + def test_polygon_to_hfb_vertex(): ds = util.get_ds_vertex() ds = nlmod.time.set_ds_time(ds, "2023", time="2024") @@ -59,8 +511,32 @@ def test_polygon_to_hfb_structured(): nlmod.gwf.hfb.plot_hfb(hfb, gwf, ax=ax) +def test_line_to_hfb_deprecated(): + ds = util.get_ds_structured() + gdf = gpd.GeoDataFrame({"geometry": [LineString([(100, 1000), (225.0, 425.1)])]}) + + with pytest.warns(DeprecationWarning, match="line_to_hfb.*deprecated"): + cellids = nlmod.gwf.hfb.line_to_hfb(gdf, ds) + + assert cellids + + +def test_line2hfb_deprecated_without_duplicate_line_to_hfb_warning(): + ds = util.get_ds_structured() + gdf = gpd.GeoDataFrame({"geometry": [LineString([(100, 1000), (225.0, 425.1)])]}) + + with pytest.warns( + DeprecationWarning, match="line2hfb.*deprecated" + ) as warning_records: + cellids = nlmod.gwf.hfb.line2hfb(gdf, ds) + + messages = [str(record.message) for record in warning_records] + assert any("line2hfb" in message for message in messages) + assert not any("line_to_hfb" in message for message in messages) + assert cellids + + def test_line_to_hfb_buffer_structured(): - # this test also tests line_to_hfb ds = util.get_ds_structured() ds = nlmod.time.set_ds_time(ds, "2023", time="2024") gwf = util.get_gwf(ds) @@ -71,9 +547,10 @@ def test_line_to_hfb_buffer_structured(): LineString([(100, 1000), (225.0, 425.1)]), LineString([(225.0, 425.1), (225.0, 0)]), ], - } + }, ) - cellids = nlmod.gwf.hfb.line_to_hfb_buffer(gdf, ds) + with pytest.warns(DeprecationWarning, match="line_to_hfb_buffer.*deprecated"): + cellids = nlmod.gwf.hfb.line_to_hfb_buffer(gdf, ds) # also test the plot method ax = gdf.plot(column="name") gwf.modelgrid.plot(ax=ax) @@ -81,7 +558,6 @@ def test_line_to_hfb_buffer_structured(): def test_line_to_hfb_buffer_vertex(): - # this test also tests line_to_hfb ds = util.get_ds_vertex() ds = nlmod.time.set_ds_time(ds, "2023", time="2024") gwf = util.get_gwf(ds) @@ -92,10 +568,170 @@ def test_line_to_hfb_buffer_vertex(): LineString([(100, 1000), (225.0, 425.1)]), LineString([(225.0, 425.1), (225.0, 0)]), ], - } + }, ) - cellids = nlmod.gwf.hfb.line_to_hfb_buffer(gdf, ds) + with pytest.warns(DeprecationWarning, match="line_to_hfb_buffer.*deprecated"): + cellids = nlmod.gwf.hfb.line_to_hfb_buffer(gdf, ds) # also test the plot method ax = gdf.plot(column="name") gwf.modelgrid.plot(ax=ax) nlmod.gwf.hfb.plot_hfb(cellids, gwf, ax=ax) + + +def _run_two_column_flow(ws, nlay, top, botm, hfb_spd): + """Steady two-column CHD model; returns the flow crossing the shared face.""" + sim = flopy.mf6.MFSimulation( + sim_name="t", sim_ws=str(ws), exe_name=nlmod.util.get_exe_path("mf6") + ) + flopy.mf6.ModflowTdis(sim) + flopy.mf6.ModflowIms(sim, outer_dvclose=1e-9, inner_dvclose=1e-10) + gwf = flopy.mf6.ModflowGwf(sim, modelname="t", save_flows=True) + flopy.mf6.ModflowGwfdis( + gwf, nlay=nlay, nrow=1, ncol=2, delr=100.0, delc=100.0, top=top, botm=botm + ) + flopy.mf6.ModflowGwfnpf(gwf, icelltype=0, k=10.0) + flopy.mf6.ModflowGwfic(gwf, strt=1.0) + chd_spd = [[(ilay, 0, 0), 1.0] for ilay in range(nlay)] + chd_spd += [[(ilay, 0, 1), 0.0] for ilay in range(nlay)] + flopy.mf6.ModflowGwfchd(gwf, stress_period_data=chd_spd) + flopy.mf6.ModflowGwfhfb(gwf, stress_period_data={0: hfb_spd}) + flopy.mf6.ModflowGwfoc( + gwf, budget_filerecord="t.cbc", saverecord=[("BUDGET", "ALL")] + ) + sim.write_simulation(silent=True) + success, _ = sim.run_simulation(silent=True) + assert success + cbc = flopy.utils.CellBudgetFile(str(ws / "t.cbc")) + chd_flows = cbc.get_data(text="CHD")[0] + return chd_flows["q"][chd_flows["q"] > 0].sum() + + +def test_get_hfb_spd_partial_penetration_matches_resolved_barrier(tmp_path): + # A barrier penetrating the upper half of a single 10-m layer. The reference run + # resolves that layer into a walled and an open sublayer; the equivalent + # single-layer HYDCHR produced by get_hfb_spd must reproduce the reference flow. + ds = util.get_ds_structured( + extent=[0, 200, 0, 100], + model_name="hfb_partial", + top=10.0, + botm=[0.0], + kh=10.0, + kv=1.0, + ) + gdf = gpd.GeoDataFrame({"geometry": [LineString([(100, 0), (100, 100)])]}) + spd = nlmod.gwf.hfb.get_hfb_spd(ds, gdf, hydchr=1 / 100.0, depth=5.0) + spd = [[cellid1, cellid2, float(hydchr)] for cellid1, cellid2, hydchr in spd] + assert len(spd) == 1 + q_equivalent = _run_two_column_flow( + tmp_path / "eq", nlay=1, top=10.0, botm=[0.0], hfb_spd=spd + ) + q_reference = _run_two_column_flow( + tmp_path / "ref", + nlay=2, + top=10.0, + botm=[5.0, 0.0], + hfb_spd=[[(0, 0, 0), (0, 0, 1), 1 / 100.0]], + ) + assert q_equivalent == pytest.approx(q_reference, rel=0.01) + + +def _partial_hfb_ds(grid): + """One 10-m layer with kh varying in space, so kh differs across every face.""" + if grid == "quadtree": + # gridgen quadtree: 20-m cells refined to 10 m around a short line, so the + # diagonal wall crosses faces between coarse and fine cells + ds = util.get_ds_vertex( + extent=[0, 100, 0, 100], + line=[(40, 40), (60, 60)], + model_name="hfb_face", + delr=20.0, + top=10.0, + botm=[0.0], + kh=10.0, + kv=1.0, + ) + wall = LineString([(0, 100), (100, 0)]) + else: + delr = np.array([40.0, 40.0]) if grid == "uniform" else np.array([40.0, 200.0]) + ds = util.get_ds_structured( + extent=[0, float(delr.sum()), 0, 10], + model_name="hfb_face", + delr=delr, + delc=np.array([10.0]), + top=10.0, + botm=[0.0], + kh=10.0, + kv=1.0, + ) + wall = LineString([(40, 0), (40, 10)]) + if grid != "uniform": + ds["kh"] = ds["kh"] * (0.1 + 0.02 * ds["x"] + 0.01 * ds["y"]) + return nlmod.time.set_ds_time(ds, "2023", time="2024"), wall + + +def _face_conductances(ds, ws, hfb_spd): + """MF6 face conductance |Q / dh| per connection, with every cell a CHD.""" + ds.attrs["model_ws"] = str(ws) + sim = nlmod.sim.sim(ds) + nlmod.sim.tdis(ds, sim) + nlmod.sim.ims(sim, complexity="simple") + gwf = nlmod.gwf.gwf(ds, sim) + nlmod.gwf.dis(ds, gwf) + nlmod.gwf.npf(ds, gwf, icelltype=0, save_flows=True) + nlmod.gwf.ic(ds, gwf, starting_head=0.0) + x, y = xr.broadcast(ds["x"], ds["y"]) + head = (x + 0.37 * y).transpose(*ds["kh"].dims[1:]).values.ravel() + shape = ds["kh"].shape[1:] + chd_spd = [ + [(0, *np.unravel_index(node, shape)), float(head[node])] + for node in range(head.size) + ] + flopy.mf6.ModflowGwfchd(gwf, stress_period_data=chd_spd) + if hfb_spd: + flopy.mf6.ModflowGwfhfb(gwf, stress_period_data={0: hfb_spd}) + nlmod.gwf.oc(ds, gwf) + nlmod.sim.write_and_run(sim, ds, write_ds=False, silent=True) + grb = flopy.mf6.utils.MfGrdFile(next(str(p) for p in ws.glob("*.grb"))) + flowja = gwf.output.budget().get_data(text="FLOW-JA-FACE")[0].ravel() + return { + (node, int(grb.ja[k])): abs(flowja[k] / (head[node] - head[grb.ja[k]])) + for node in range(grb.nodes) + for k in range(grb.ia[node] + 1, grb.ia[node + 1]) + } + + +@pytest.mark.parametrize("grid", ["uniform", "unequal", "quadtree"]) +@pytest.mark.parametrize("hydchr", [1e-6, 0.01, 1.0]) +@pytest.mark.parametrize("depth", [3.0, 8.0]) +def test_get_hfb_spd_partial_face_conductance(tmp_path, grid, hydchr, depth): + # The walled part of the face (depth / thickness) is the barrier in series with the + # aquifer; the open part below it keeps the full conductance. The face conductance + # MF6 uses must equal that parallel combination. + ds, wall = _partial_hfb_ds(grid) + spd = nlmod.gwf.hfb.get_hfb_spd( + ds, gpd.GeoDataFrame(geometry=[wall]), hydchr=hydchr, depth=depth + ) + spd = [[cellid1, cellid2, float(value)] for cellid1, cellid2, value in spd] + assert spd + + modelgrid = modelgrid_from_ds(ds) + shape = ds["kh"].shape[1:] + c_open = _face_conductances(ds, tmp_path / "open", []) + c_wall = _face_conductances(ds, tmp_path / "wall", spd) + frac = depth / 10.0 + area_ratios = [] + for cellid1, cellid2, _ in spd: + node1 = int(np.ravel_multi_index(cellid1[1:], shape)) + node2 = int(np.ravel_multi_index(cellid2[1:], shape)) + poly1 = Polygon(modelgrid.get_cell_vertices(*cellid1[1:])) + poly2 = Polygon(modelgrid.get_cell_vertices(*cellid2[1:])) + area_ratios.append(poly1.area / poly2.area) + face_area = poly1.intersection(poly2).length * 10.0 + conductance = c_open[node1, node2] / face_area + expected = c_open[node1, node2] * ( + (1.0 - frac) + frac * hydchr / (conductance + hydchr) + ) + assert c_wall[node1, node2] == pytest.approx(expected, rel=1e-6) + if grid == "quadtree": + # the wall crosses faces between cells of different refinement levels + assert any(not np.isclose(ratio, 1.0) for ratio in area_ratios) diff --git a/tests/test_024_administrative.py b/tests/test_024_administrative.py index 0bb89b63..be81b5e1 100644 --- a/tests/test_024_administrative.py +++ b/tests/test_024_administrative.py @@ -1,3 +1,6 @@ +import pytest +import requests + import nlmod @@ -9,7 +12,9 @@ def test_get_municipalities_cbs(): def test_get_municipalities_kadaster(): extent = [100000, 110000, 400000, 410000] - gdf = nlmod.read.administrative.download_municipalities_gdf(source="kadaster", extent=extent) + gdf = nlmod.read.administrative.download_municipalities_gdf( + source="kadaster", extent=extent + ) assert len(gdf) > 0 @@ -33,6 +38,15 @@ def test_get_netherlands_kadaster(): assert len(gdf) > 0 -def test_get_waterboards(): - gdf = nlmod.read.administrative.download_waterboards_gdf() +def test_download_kadaster_percelen(): + extent = [118_200, 118_300, 439_800, 439_900] + gdf = nlmod.read.administrative.download_kadaster_percelen(extent=extent) assert len(gdf) > 0 + + +def test_get_waterboards(): + try: + gdf = nlmod.read.administrative.download_waterboards_gdf() + assert len(gdf) > 0 + except requests.exceptions.RequestException as exc: + pytest.skip(f"Network unavailable: {exc}") diff --git a/tests/test_025_modpath.py b/tests/test_025_modpath.py index e18fb5f7..97cc55f2 100644 --- a/tests/test_025_modpath.py +++ b/tests/test_025_modpath.py @@ -1,15 +1,12 @@ -import os - import flopy -import xarray as xr +import test_001_model import nlmod def test_modpath(): # start with runned model from test_001_model.test_create_sea_model - model_ws = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") - ds = xr.open_dataset(os.path.join(model_ws, "basic_sea_model.nc")) + ds = test_001_model.get_ds_from_cache("sea_model") sim = flopy.mf6.MFSimulation.load("mfsim.nam", sim_ws=ds.model_ws) gwf = sim.get_model(ds.model_name) @@ -41,7 +38,7 @@ def test_modpath(): nlmod.modpath.load_pathline_data(mpf) # get the nodes from a package - nodes = nlmod.modpath.package_to_nodes(gwf, "GHB", mpf) + nodes = nlmod.modpath.package_to_nodes(gwf, "GHB", ibound=mpf.ib) # get nodes of all cells in the top modellayer nodes = nlmod.modpath.layer_to_nodes(mpf, 0) diff --git a/tests/test_026_grid.py b/tests/test_026_grid.py index beb2fc91..c4fa2db0 100644 --- a/tests/test_026_grid.py +++ b/tests/test_026_grid.py @@ -1,14 +1,16 @@ import os -import tempfile +import flopy import geopandas as gpd import matplotlib.pyplot as plt import numpy as np +import util import xarray as xr +from shapely.geometry import LineString import nlmod -model_ws = os.path.join(tempfile.gettempdir(), "test_grid") +model_ws = os.path.join(util.get_model_data_dir(), "test_grid") extent = [98000.0, 99000.0, 489000.0, 490000.0] @@ -33,7 +35,7 @@ def get_regis(): def get_structured_model_ds(): - model_ws = os.path.join(tempfile.gettempdir(), "test_grid_structured") + model_ws = os.path.join(util.get_model_data_dir(), "test_grid_structured") fname = os.path.join(model_ws, "ds.nc") if not os.path.isfile(fname): if not os.path.isdir(model_ws): @@ -44,7 +46,7 @@ def get_structured_model_ds(): def get_structured_model_ds_rotated(): - model_ws = os.path.join(tempfile.gettempdir(), "test_grid_structured_rotated") + model_ws = os.path.join(util.get_model_data_dir(), "test_grid_structured_rotated") fname = os.path.join(model_ws, "ds.nc") if not os.path.isfile(fname): if not os.path.isdir(model_ws): @@ -55,7 +57,7 @@ def get_structured_model_ds_rotated(): def get_vertex_model_ds(bgt=None): - model_ws = os.path.join(tempfile.gettempdir(), "test_grid_vertex") + model_ws = os.path.join(util.get_model_data_dir(), "test_grid_vertex") fname = os.path.join(model_ws, "ds.nc") if not os.path.isfile(fname): if not os.path.isdir(model_ws): @@ -69,7 +71,7 @@ def get_vertex_model_ds(bgt=None): def get_vertex_model_ds_rotated(bgt=None): - model_ws = os.path.join(tempfile.gettempdir(), "test_grid_vertex_rotated") + model_ws = os.path.join(util.get_model_data_dir(), "test_grid_vertex_rotated") fname = os.path.join(model_ws, "ds.nc") if not os.path.isfile(fname): if not os.path.isdir(model_ws): @@ -84,14 +86,18 @@ def get_vertex_model_ds_rotated(bgt=None): def test_get_ds_rotated(): ds0 = get_structured_model_ds_rotated() - assert ds0.extent[0] == 0 and ds0.extent[2] == 0 - assert ds0.xorigin == extent[0] and ds0.yorigin == extent[2] + assert ds0.extent[0] == 0 + assert ds0.extent[2] == 0 + assert ds0.xorigin == extent[0] + assert ds0.yorigin == extent[2] # test refine method, by refining in all cells that contain surface water polygons ds = get_vertex_model_ds_rotated() assert len(ds.area) > np.prod(ds0.area.shape) - assert ds.extent[0] == 0 and ds.extent[2] == 0 - assert ds.xorigin == extent[0] and ds.yorigin == extent[2] + assert ds.extent[0] == 0 + assert ds.extent[2] == 0 + assert ds.xorigin == extent[0] + assert ds.yorigin == extent[2] f0, ax0 = plt.subplots() nlmod.plot.modelgrid(ds0, ax=ax0) @@ -380,7 +386,7 @@ def test_update_ds_from_layer_ds(): assert len(np.unique(ds["top"])) > 1 # test for a vertex grid - model_ws = os.path.join(tempfile.gettempdir(), "test_grid_vertex_200") + model_ws = os.path.join(util.get_model_data_dir(), "test_grid_vertex_200") ds = nlmod.grid.refine(ds, model_ws=model_ws, refinement_features=[(bgt, 1)]) ds = nlmod.grid.update_ds_from_layer_ds(ds, regis, method="nearest") assert len(np.unique(ds["top"])) > 1 @@ -395,7 +401,7 @@ def test_update_ds_from_layer_ds(): assert len(np.unique(ds["top"])) > 1 # test for a rotated vertex grid - model_ws = os.path.join(tempfile.gettempdir(), "test_grid_vertex_200_rotated") + model_ws = os.path.join(util.get_model_data_dir(), "test_grid_vertex_200_rotated") ds = nlmod.grid.refine(ds, model_ws=model_ws, refinement_features=[(bgt, 2)]) ds = nlmod.grid.update_ds_from_layer_ds(ds, regis, method="nearest") assert len(np.unique(ds["top"])) > 1 @@ -412,3 +418,15 @@ def test_gdf_area_per_index_to_da(): ds = get_vertex_model_ds() ds["bgt"] = nlmod.grid.gdf_area_to_da(bgt, ds) assert (ds["bgt"].sum("index") > 0).any() + + +def test_gdf_to_grid_line_length(): + modelgrid = flopy.discretization.StructuredGrid( + delc=np.full(10, 100.0), delr=np.full(10, 100.0), xoff=98000.0, yoff=489000.0 + ) + ix = flopy.utils.GridIntersect(modelgrid) + line = LineString([(98050.0, 489550.0), (98950.0, 489550.0)]) + gdf = gpd.GeoDataFrame(geometry=[line], crs=28992) + gdfg = nlmod.grid.gdf_to_grid(gdf, ix=ix, silent=True) + assert "length" in gdfg.columns + assert np.isclose(gdfg["length"].sum(), line.length) diff --git a/tests/test_027_util.py b/tests/test_027_util.py index 1a5e5570..19c24a05 100644 --- a/tests/test_027_util.py +++ b/tests/test_027_util.py @@ -1,12 +1,12 @@ import os -import tempfile import geopandas as gpd import rioxarray +import util import nlmod -model_ws = os.path.join(tempfile.gettempdir(), "test_util") +model_ws = os.path.join(util.get_model_data_dir(), "test_util") extent = [98000.0, 99000.0, 489000.0, 490000.0] diff --git a/tests/test_028_rws_bathymetry.py b/tests/test_028_rws_bathymetry.py index 6715c661..f3a487f3 100644 --- a/tests/test_028_rws_bathymetry.py +++ b/tests/test_028_rws_bathymetry.py @@ -1,4 +1,5 @@ import pytest +import xarray as xr import nlmod @@ -7,7 +8,12 @@ def test_bathymetry(resolution): xmin = 25_000.0 ymin = 410_000.0 + if resolution == 1: + xmin = 102_200.0 + ymin = 497_700.0 xmax = xmin + 2 * resolution ymax = ymin + 2 * resolution extent = [xmin, xmax, ymin, ymax] - nlmod.read.rws.download_bathymetry(extent, resolution=f"{resolution}m") + da = nlmod.read.rws.download_bathymetry(extent, resolution=f"{resolution}m") + assert isinstance(da, xr.DataArray) + assert not da.isnull().all() diff --git a/tests/test_029_bgt.py b/tests/test_029_bgt.py index 8b9445be..345c048e 100644 --- a/tests/test_029_bgt.py +++ b/tests/test_029_bgt.py @@ -1,5 +1,3 @@ -import os - import requests from lxml import html from pandas import to_datetime @@ -29,11 +27,8 @@ def test_bgt_layers(): assert "waterdeel" in layers -def test_bgt_zipfile(): - pathname = "download" - if not os.path.isdir(pathname): - os.makedirs(pathname) - fname = os.path.join(pathname, "test_bgt_zipfile.zip") +def test_bgt_zipfile(tmp_path): + fname = tmp_path / "test_bgt_zipfile.zip" # download data from 2 layers within extent, and also save data to zipfile extent = [119900, 120000, 440000, 440100] diff --git a/tests/test_032_config_version.py b/tests/test_032_config_version.py new file mode 100644 index 00000000..7cf704c0 --- /dev/null +++ b/tests/test_032_config_version.py @@ -0,0 +1,62 @@ +import pytest + +from nlmod import config +from nlmod.version import __version__, show_versions + + +@pytest.fixture(autouse=True) +def reset_config_options_after_test(): + config.reset_options() + yield + config.reset_options() + + +def test_set_get_and_reset_options(): + config.set_options(nc_hash=False, dataset_coords_hash=False) + + assert config.get_options("nc_hash") == {"nc_hash": False} + assert config.get_options("dataset_coords_hash") == { + "dataset_coords_hash": False, + } + + config.reset_options(["nc_hash", "dataset_coords_hash"]) + assert config.get_options("nc_hash") == {"nc_hash": True} + assert config.get_options("dataset_coords_hash") == { + "dataset_coords_hash": True, + } + + +def test_set_options_unknown_key_raises(): + with pytest.raises(ValueError, match="Unknown option"): + config.set_options(does_not_exist=True) + + +def test_cache_options_context_restores_on_exit(): + original = config.get_options()["dataset_data_vars_hash"] + + with config.cache_options(dataset_data_vars_hash=not original) as opts: + assert opts["dataset_data_vars_hash"] is (not original) + + assert config.get_options()["dataset_data_vars_hash"] is original + + +def test_cache_options_restores_on_exception(): + original = config.get_options()["explicit_dataset_coordinate_comparison"] + + with pytest.raises(RuntimeError, match="boom"): + with config.cache_options(explicit_dataset_coordinate_comparison=not original): + raise RuntimeError("boom") + + assert config.get_options()["explicit_dataset_coordinate_comparison"] is original + + +def test_show_versions_prints_expected_lines(capsys): + show_versions() + out = capsys.readouterr().out + + assert "Python version" in out + assert "NumPy version" in out + assert "Xarray version" in out + assert "Matplotlib version" in out + assert "Flopy version" in out + assert f"nlmod version : {__version__}" in out diff --git a/tests/test_033_additional_cases.py b/tests/test_033_additional_cases.py new file mode 100644 index 00000000..0fd323b2 --- /dev/null +++ b/tests/test_033_additional_cases.py @@ -0,0 +1,270 @@ +import cftime +import geopandas as gpd +import numpy as np +import pytest +import xarray as xr +from requests.exceptions import HTTPError, RequestException +from shapely.geometry import Polygon + +import nlmod +from nlmod import config +from nlmod.dims.layers import kveq_combined_layers +from nlmod.modpath.modpath import package_to_nodes +from nlmod.read import administrative, knmi_data_platform, webservices + + +@pytest.fixture(autouse=True) +def _reset_config_options(): + config.reset_options() + yield + config.reset_options() + + +def test_config_reset_unknown_key_raises_keyerror(): + with pytest.raises(KeyError): + config.reset_options(["does_not_exist"]) + + +def test_config_get_options_shape_contract(): + all_opts = config.get_options() + one_opt = config.get_options("nc_hash") + + assert isinstance(all_opts, dict) + assert set(one_opt) == {"nc_hash"} + assert isinstance(one_opt["nc_hash"], bool) + + +def test_set_ds_time_mixed_string_formats(): + ds = nlmod.get_ds([0, 100, 0, 100]) + out = nlmod.time.set_ds_time( + ds, + start="2000-01-01", + time=["2000-02-01", "2000/03/01"], + ) + + assert out.sizes["time"] == 2 + assert out.time.values[1] > out.time.values[0] + + +def test_set_ds_time_mixed_formats_with_cftime_start(): + ds = nlmod.get_ds([0, 100, 0, 100]) + out = nlmod.time.set_ds_time( + ds, + start=cftime.datetime(1000, 1, 1), + time=["2000-02-01", "2000/03/01"], + ) + + assert isinstance(out.time.values[0], cftime.datetime) + assert out.time.values[1] > out.time.values[0] + + +def test_kveq_combined_layers_zero_denominator_returns_nan(): + kv = xr.DataArray( + np.array( + [ + [[1.0, np.inf]], + [[1.0, np.inf]], + ] + ), + dims=("layer", "y", "x"), + ) + thickness = xr.DataArray(np.ones_like(kv), dims=kv.dims) + + out = kveq_combined_layers(kv, thickness, {0: (0, 1)}) + + assert np.isfinite(out.data[0, 0, 0]) + assert np.isnan(out.data[0, 0, 1]) + + +def test_webservices_get_data_non_json_propagates_value_error(monkeypatch): + class DummyResponse: + ok = True + url = "https://example.test" + + def json(self): + raise ValueError("not json") + + monkeypatch.setattr( + webservices.requests, "get", lambda *args, **kwargs: DummyResponse() + ) + + with pytest.raises(ValueError, match="not json"): + webservices._get_data("https://example.test", {}) + + +def test_webservices_get_data_http_error_on_bad_status(monkeypatch): + class DummyResponse: + ok = False + url = "https://example.test" + + monkeypatch.setattr( + webservices.requests, "get", lambda *args, **kwargs: DummyResponse() + ) + + with pytest.raises(HTTPError, match="Request not successful"): + webservices._get_data("https://example.test", {}) + + +def test_webservices_get_data_error_payload_raises(monkeypatch): + class DummyResponse: + ok = True + url = "https://example.test" + + def json(self): + return {"error": {"code": 500, "message": "boom"}} + + monkeypatch.setattr( + webservices.requests, "get", lambda *args, **kwargs: DummyResponse() + ) + + with pytest.raises(ValueError, match="Error code 500"): + webservices._get_data("https://example.test", {}) + + +def test_administrative_wfs_request_exception_propagates(monkeypatch): + def _raise(*args, **kwargs): + raise RequestException("network down") + + monkeypatch.setattr(administrative.webservices, "wfs", _raise) + + with pytest.raises(RequestException, match="network down"): + administrative.download_municipalities_gdf(source="cbs", cachename="muni_exc") + with pytest.raises(RequestException, match="network down"): + administrative.download_provinces_gdf(source="cbs", cachename="prov_exc") + with pytest.raises(RequestException, match="network down"): + administrative.download_netherlands_gdf(source="cbs", cachename="nl_exc") + + +def test_administrative_waterboards_request_exception_propagates(monkeypatch): + def _raise(*args, **kwargs): + raise RequestException("network down") + + monkeypatch.setattr(administrative.waterboard, "download_polygons", _raise) + + with pytest.raises(RequestException, match="network down"): + administrative.download_waterboards_gdf(cachename="wb_exc") + + +def test_knmi_download_file_accepts_pathlike(monkeypatch, tmp_path): + class JsonResponse: + def __init__(self, payload): + self._payload = payload + + def json(self): + return self._payload + + class StreamResponse: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def raise_for_status(self): + return None + + def iter_content(self, chunk_size=8192): + yield b"abc" + + def fake_get(url, *args, **kwargs): + if str(url).endswith("/url"): + return JsonResponse({"temporaryDownloadUrl": "https://download.test/file"}) + return StreamResponse() + + monkeypatch.setattr(knmi_data_platform.requests, "get", fake_get) + + knmi_data_platform.download_file( + dataset_name="dummy", + dataset_version="1", + fname="file.bin", + dirname=tmp_path, + api_key="token", + ) + + assert (tmp_path / "file.bin").exists() + + +def test_knmi_download_files_forwards_pathlike(monkeypatch, tmp_path): + calls = [] + + def fake_download_file(*, dirname, **kwargs): + calls.append(dirname) + + monkeypatch.setattr(knmi_data_platform, "download_file", fake_download_file) + + knmi_data_platform.download_files( + dataset_name="dummy", + dataset_version="1", + fnames=["a.bin", "b.bin"], + dirname=tmp_path, + api_key="token", + ) + + assert calls == [tmp_path, tmp_path] + + +def test_surface_water_discretize_missing_columns_raises_keyerror(): + ds = nlmod.get_ds( + [0, 100, 0, 100], + delr=50.0, + delc=50.0, + top=1.0, + botm=[0.0], + kh=1.0, + kv=1.0, + ) + gdf = gpd.GeoDataFrame( + {"geometry": [Polygon([(0, 0), (100, 0), (100, 100), (0, 100)])]}, + geometry="geometry", + crs=28992, + ) + + with pytest.raises(KeyError): + nlmod.read.rws.discretize_surface_water(ds, gdf, da_basename="sw") + + +def test_modpath_package_to_nodes_without_ibound_structured_and_vertex(): + class DummyConnData: + def __init__(self, cellids): + self.array = {"cellid": cellids} + + class DummyPackage: + def __init__(self, cellids): + self.connectiondata = DummyConnData(cellids) + + class DummyGrid: + def __init__(self, grid_type, shape): + self.grid_type = grid_type + self.shape = shape + + class DummyGwf: + def __init__(self, grid_type, shape, cellids): + self.modelgrid = DummyGrid(grid_type, shape) + self._pkg = DummyPackage(cellids) + + def get_package(self, name): + return self._pkg + + gwf_struct = DummyGwf("structured", (1, 2, 3), [(0, 0, 0), (0, 1, 2)]) + nodes_struct = package_to_nodes(gwf_struct, "GHB") + assert nodes_struct == [0, 5] + + gwf_vertex = DummyGwf("vertex", (2, 4), [(0, 0), (1, 3)]) + nodes_vertex = package_to_nodes(gwf_vertex, "GHB") + assert nodes_vertex == [0, 7] + + +def test_show_versions_propagates_missing_dependency(monkeypatch): + from importlib import metadata + + from nlmod import version as version_module + + def fake_version(name): + if name == "numpy": + raise metadata.PackageNotFoundError("numpy") + return "1.0" + + monkeypatch.setattr(version_module.metadata, "version", fake_version) + + with pytest.raises(metadata.PackageNotFoundError): + version_module.show_versions() diff --git a/tests/test_034_interactive_env.py b/tests/test_034_interactive_env.py new file mode 100644 index 00000000..4e7cf127 --- /dev/null +++ b/tests/test_034_interactive_env.py @@ -0,0 +1,12 @@ +import os + +import util + + +def test_model_data_dir_fallback_for_interactive(monkeypatch): + monkeypatch.delenv("NLMOD_TEST_MODEL_DATA_DIR", raising=False) + + model_data_dir = util.get_model_data_dir() + + assert os.path.isdir(model_data_dir) + assert os.environ.get("NLMOD_TEST_MODEL_DATA_DIR") == model_data_dir diff --git a/tests/util.py b/tests/util.py index 3d2c3265..3f65ddf6 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,14 +1,30 @@ import os +import tempfile from shapely.geometry import LineString import nlmod +MODEL_DATA_ENV_VAR = "NLMOD_TEST_MODEL_DATA_DIR" + + +def get_model_data_dir(): + model_data_dir = os.environ.get(MODEL_DATA_ENV_VAR) + if model_data_dir is None: + # In interactive windows there is no pytest fixture lifecycle, so provide + # a deterministic temp fallback to make single-test debugging possible. + model_data_dir = os.path.join( + tempfile.gettempdir(), "nlmod_test_model_data_interactive" + ) + os.environ[MODEL_DATA_ENV_VAR] = model_data_dir + os.makedirs(model_data_dir, exist_ok=True) + return model_data_dir + def get_ds_structured(extent=None, model_name="test", **kwargs): if extent is None: extent = [0, 1000, 0, 1000] - model_ws = os.path.join("data", model_name) + model_ws = os.path.join(get_model_data_dir(), model_name) ds = nlmod.get_ds(extent, model_name=model_name, model_ws=model_ws, **kwargs) return ds @@ -17,7 +33,7 @@ def get_ds_vertex(extent=None, line=None, **kwargs): if line is None: line = [(0, 1000), (1000, 0)] ds = get_ds_structured(extent=extent, **kwargs) - model_ws = os.path.join("data", "gridgen") + model_ws = os.path.join(get_model_data_dir(), "gridgen") refinement_features = [([LineString(line)], "line", 1)] ds = nlmod.grid.refine(ds, model_ws, refinement_features=refinement_features) return ds