{
"cells": [
{
"cell_type": "markdown",
"id": "63eb22d7",
"metadata": {
"papermill": {
"duration": 0.002376,
"end_time": "2025-06-18T09:44:21.416475",
"exception": false,
"start_time": "2025-06-18T09:44:21.414099",
"status": "completed"
},
"tags": []
},
"source": [
"# Large scale flood hazard (multiple regions)"
]
},
{
"cell_type": "markdown",
"id": "f2d7a2d3",
"metadata": {
"papermill": {
"duration": 0.002014,
"end_time": "2025-06-18T09:44:21.420717",
"exception": false,
"start_time": "2025-06-18T09:44:21.418703",
"status": "completed"
},
"tags": []
},
"source": [
"This example shows a workflow to derive pluvial flood hazard using the SFINCS model. The goal of the example is to show **how to scale the flood hazard to multiple model domains**. For more details about each step we refer the user to the pluvial flood risk example.\n",
"\n",
"This example also show how to work with parsing to CWL."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "0b7f0897",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:21.425341Z",
"iopub.status.busy": "2025-06-18T09:44:21.425181Z",
"iopub.status.idle": "2025-06-18T09:44:23.646360Z",
"shell.execute_reply": "2025-06-18T09:44:23.645874Z"
},
"papermill": {
"duration": 2.224555,
"end_time": "2025-06-18T09:44:23.647274",
"exception": false,
"start_time": "2025-06-18T09:44:21.422719",
"status": "completed"
},
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO - log - hydroflows version: 0.1.0\n"
]
}
],
"source": [
"# Import modules\n",
"from pathlib import Path\n",
"\n",
"from hydroflows.log import setuplog\n",
"from hydroflows.methods import rainfall, sfincs\n",
"from hydroflows.methods.utils.example_data import fetch_data\n",
"from hydroflows.workflow import Workflow, WorkflowConfig\n",
"\n",
"logger = setuplog(level=\"INFO\")\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "14d3a86b",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:23.653176Z",
"iopub.status.busy": "2025-06-18T09:44:23.652703Z",
"iopub.status.idle": "2025-06-18T09:44:23.655725Z",
"shell.execute_reply": "2025-06-18T09:44:23.655261Z"
},
"papermill": {
"duration": 0.006772,
"end_time": "2025-06-18T09:44:23.656576",
"exception": false,
"start_time": "2025-06-18T09:44:23.649804",
"status": "completed"
},
"tags": []
},
"outputs": [],
"source": [
"# Define case name and root directory\n",
"name = \"pluvial_multiple_regions\"\n",
"pwd = Path().resolve() # Get the current file location\n",
"case_root = Path(pwd, \"cases\", name) # output directory\n",
"pwd_rel = \"../../\" # relative path from the case directory to the current file\n"
]
},
{
"cell_type": "markdown",
"id": "d254e499",
"metadata": {
"papermill": {
"duration": 0.002157,
"end_time": "2025-06-18T09:44:23.660954",
"exception": false,
"start_time": "2025-06-18T09:44:23.658797",
"status": "completed"
},
"tags": []
},
"source": [
"## Workflow inputs"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "f78e334c",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:23.666061Z",
"iopub.status.busy": "2025-06-18T09:44:23.665737Z",
"iopub.status.idle": "2025-06-18T09:44:24.451806Z",
"shell.execute_reply": "2025-06-18T09:44:24.451256Z"
},
"papermill": {
"duration": 0.789771,
"end_time": "2025-06-18T09:44:24.452902",
"exception": false,
"start_time": "2025-06-18T09:44:23.663131",
"status": "completed"
},
"tags": []
},
"outputs": [],
"source": [
"# Fetch the build data\n",
"cache_dir = fetch_data(data=\"global-data\")"
]
},
{
"cell_type": "markdown",
"id": "adf70348",
"metadata": {
"papermill": {
"duration": 0.002229,
"end_time": "2025-06-18T09:44:24.457699",
"exception": false,
"start_time": "2025-06-18T09:44:24.455470",
"status": "completed"
},
"tags": []
},
"source": [
"Since the CWL runner we will be using is not supported on Windows, we will opt to run SFINCS using a docker container."
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "892a35cc",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:24.462809Z",
"iopub.status.busy": "2025-06-18T09:44:24.462512Z",
"iopub.status.idle": "2025-06-18T09:44:24.465673Z",
"shell.execute_reply": "2025-06-18T09:44:24.465274Z"
},
"papermill": {
"duration": 0.006559,
"end_time": "2025-06-18T09:44:24.466364",
"exception": false,
"start_time": "2025-06-18T09:44:24.459805",
"status": "completed"
},
"tags": []
},
"outputs": [],
"source": [
"# Setup the configuration\n",
"config = WorkflowConfig(\n",
" config=Path(pwd_rel, \"hydromt_config/sfincs_config.yml\"),\n",
" catalog_path=Path(cache_dir, \"data_catalog.yml\"),\n",
" sfincs_run_method=\"docker\",\n",
" start_date=\"2014-01-01\",\n",
" end_date=\"2021-12-31\",\n",
" # sfincs settings\n",
" hydromt_sfincs_config=Path(pwd_rel, \"hydromt_config/sfincs_config.yml\"),\n",
" subgrid_output=True,\n",
" # design event settings\n",
" rps=[2, 5, 10],\n",
")"
]
},
{
"cell_type": "markdown",
"id": "ab7e4f42",
"metadata": {
"papermill": {
"duration": 0.00222,
"end_time": "2025-06-18T09:44:24.470771",
"exception": false,
"start_time": "2025-06-18T09:44:24.468551",
"status": "completed"
},
"tags": []
},
"source": [
"## Create the workflow\n",
"\n",
"Note that we initialize the workflow with a region wildcard to create flood hazard for multiple regions."
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "ddabcf7a",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:24.475957Z",
"iopub.status.busy": "2025-06-18T09:44:24.475548Z",
"iopub.status.idle": "2025-06-18T09:44:24.478316Z",
"shell.execute_reply": "2025-06-18T09:44:24.477848Z"
},
"papermill": {
"duration": 0.006035,
"end_time": "2025-06-18T09:44:24.478997",
"exception": false,
"start_time": "2025-06-18T09:44:24.472962",
"status": "completed"
},
"tags": []
},
"outputs": [],
"source": [
"# Setup the workflow\n",
"wf = Workflow(\n",
" config=config,\n",
" wildcards={\"region\": [\"region\", \"region2\"]},\n",
" name=name,\n",
" root=case_root,\n",
")"
]
},
{
"cell_type": "markdown",
"id": "2701d77c",
"metadata": {
"papermill": {
"duration": 0.002195,
"end_time": "2025-06-18T09:44:24.483394",
"exception": false,
"start_time": "2025-06-18T09:44:24.481199",
"status": "completed"
},
"tags": []
},
"source": [
"### Build models\n",
"\n",
"In this section we build SFINCS models for multiple regions. Note that we use the ``{region}`` wildcard on the in- and outputs of each method. The method will be executed for each input region."
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "9909002e",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:24.488621Z",
"iopub.status.busy": "2025-06-18T09:44:24.488337Z",
"iopub.status.idle": "2025-06-18T09:44:24.494861Z",
"shell.execute_reply": "2025-06-18T09:44:24.494370Z"
},
"papermill": {
"duration": 0.009881,
"end_time": "2025-06-18T09:44:24.495544",
"exception": false,
"start_time": "2025-06-18T09:44:24.485663",
"status": "completed"
},
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"Rule(id=sfincs_build, method=sfincs_build, runs=2, repeat=['region'])"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Build the SFINCS models\n",
"sfincs_build = sfincs.SfincsBuild(\n",
" region=Path(pwd_rel, \"data/build/{region}.geojson\"), # input region\n",
" sfincs_root=\"models/sfincs/{region}\", # output model directory\n",
" config=wf.get_ref(\"$config.hydromt_sfincs_config\"),\n",
" catalog_path=wf.get_ref(\"$config.catalog_path\"),\n",
" subgrid_output=wf.get_ref(\"$config.subgrid_output\"),\n",
" )\n",
"wf.create_rule(sfincs_build, \"sfincs_build\")\n"
]
},
{
"cell_type": "markdown",
"id": "290d1405",
"metadata": {
"papermill": {
"duration": 0.00224,
"end_time": "2025-06-18T09:44:24.500142",
"exception": false,
"start_time": "2025-06-18T09:44:24.497902",
"status": "completed"
},
"tags": []
},
"source": [
"### Derive pluvial design events\n",
"\n",
"In contrast to the pluvial risk workflow, we will name the the `event_root` param \"events/{region}_events\". This way we avoid the following conflict when parsing to CWL. The outputs of both the `pluvial_events` rule and the `sfincs_build` rule will be used as inputs to the `sfincs_update` rule. If we were to follow the naming as the pluvial risk example, the `pluvial_events` and `sfincs_build` rules will create output directory with \"{region}\" as stem. Since CWL will flatten the rest of those output directory paths, the `sfincs_update` rule will receive two input directories both named \"{region}\"."
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "feda0f3f",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:24.505543Z",
"iopub.status.busy": "2025-06-18T09:44:24.505192Z",
"iopub.status.idle": "2025-06-18T09:44:24.510879Z",
"shell.execute_reply": "2025-06-18T09:44:24.510457Z"
},
"papermill": {
"duration": 0.009128,
"end_time": "2025-06-18T09:44:24.511548",
"exception": false,
"start_time": "2025-06-18T09:44:24.502420",
"status": "completed"
},
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO - wildcards - Added wildcard 'events' with values: ['p_event_rp002', 'p_event_rp005', 'p_event_rp010']\n"
]
},
{
"data": {
"text/plain": [
"Rule(id=pluvial_events, method=pluvial_design_events_GPEX, runs=2, repeat=['region'], expand=['events'])"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pluvial_events = rainfall.PluvialDesignEventsGPEX(\n",
" gpex_nc=Path(cache_dir, \"gpex.nc\"), \n",
" region=sfincs_build.output.sfincs_region,\n",
" event_root=\"events/{region}_events\",\n",
" rps=wf.get_ref(\"$config.rps\"),\n",
" wildcard=\"events\", # wildcard to use for the pluvial events\n",
")\n",
"\n",
"# Note that a new \"events\" wildcard is created for the events\n",
"wf.create_rule(pluvial_events, rule_id=\"pluvial_events\")\n"
]
},
{
"cell_type": "markdown",
"id": "3e8b7077",
"metadata": {
"papermill": {
"duration": 0.002367,
"end_time": "2025-06-18T09:44:24.516379",
"exception": false,
"start_time": "2025-06-18T09:44:24.514012",
"status": "completed"
},
"tags": []
},
"source": [
"### Derive flood hazard\n",
"\n",
"Here we have to set the `copy_model` param of the `sfincs_update` rule to `True`. The flattening of input file paths by CWL we encountered earlier does not work well with the relative paths the model config files will be using when `copy_model` is set to `False`.\n",
"\n",
"Note also that in the `sfincs_run` rule we use the `run_method` param instead of the `sfincs_exe` param to indicate that we will be using the docker container."
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "2132eb5b",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:24.521910Z",
"iopub.status.busy": "2025-06-18T09:44:24.521604Z",
"iopub.status.idle": "2025-06-18T09:44:24.530308Z",
"shell.execute_reply": "2025-06-18T09:44:24.529804Z"
},
"papermill": {
"duration": 0.012329,
"end_time": "2025-06-18T09:44:24.531107",
"exception": false,
"start_time": "2025-06-18T09:44:24.518778",
"status": "completed"
},
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"Rule(id=sfincs_downscale, method=sfincs_downscale, runs=6, repeat=['region', 'events'])"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Update the SFINCS models\n",
"sfincs_update = sfincs.SfincsUpdateForcing(\n",
" sfincs_inp=sfincs_build.output.sfincs_inp,\n",
" event_yaml=pluvial_events.output.event_yaml,\n",
" output_dir=sfincs_build.output.sfincs_inp.parent/\"simulations\"/\"{events}\",\n",
" copy_model=True,\n",
")\n",
"wf.create_rule(sfincs_update, rule_id=\"sfincs_update\")\n",
"\n",
"# Run SFINCS model\n",
"sfincs_run = sfincs.SfincsRun(\n",
" sfincs_inp=sfincs_update.output.sfincs_out_inp,\n",
" run_method=wf.get_ref(\"$config.sfincs_run_method\"),\n",
")\n",
"wf.create_rule(sfincs_run, rule_id=\"sfincs_run\")\n",
"\n",
"# Downscale the SFINCS waterlevels to high-resolution water\n",
"sfincs_downscale = sfincs.SfincsDownscale(\n",
" sfincs_map=sfincs_run.output.sfincs_map,\n",
" sfincs_subgrid_dep=sfincs_build.output.sfincs_subgrid_dep,\n",
" output_root=\"output/hazard/{region}\",\n",
")\n",
"wf.create_rule(sfincs_downscale, \"sfincs_downscale\")"
]
},
{
"cell_type": "markdown",
"id": "f89b0cb1",
"metadata": {
"papermill": {
"duration": 0.002426,
"end_time": "2025-06-18T09:44:24.536241",
"exception": false,
"start_time": "2025-06-18T09:44:24.533815",
"status": "completed"
},
"tags": []
},
"source": [
"## Visualize and execute the workflow"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "68054957",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:24.541934Z",
"iopub.status.busy": "2025-06-18T09:44:24.541580Z",
"iopub.status.idle": "2025-06-18T09:44:24.573928Z",
"shell.execute_reply": "2025-06-18T09:44:24.573430Z"
},
"papermill": {
"duration": 0.035949,
"end_time": "2025-06-18T09:44:24.574660",
"exception": false,
"start_time": "2025-06-18T09:44:24.538711",
"status": "completed"
},
"tags": []
},
"outputs": [
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# plot the rulegraph using graphviz\n",
"wf.plot_rulegraph(filename=\"rulegraph.svg\", plot_rule_attrs=True)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "81934205",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:24.581157Z",
"iopub.status.busy": "2025-06-18T09:44:24.580816Z",
"iopub.status.idle": "2025-06-18T09:44:24.586520Z",
"shell.execute_reply": "2025-06-18T09:44:24.586071Z"
},
"papermill": {
"duration": 0.009753,
"end_time": "2025-06-18T09:44:24.587269",
"exception": false,
"start_time": "2025-06-18T09:44:24.577516",
"status": "completed"
},
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO - workflow - Dryrun rule 1/5: sfincs_build (2 runs)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO - workflow - Dryrun rule 2/5: pluvial_events (2 runs)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO - workflow - Dryrun rule 3/5: sfincs_update (6 runs)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO - workflow - Dryrun rule 4/5: sfincs_run (6 runs)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO - workflow - Dryrun rule 5/5: sfincs_downscale (6 runs)\n"
]
}
],
"source": [
"# Do a dry run of the workflow\n",
"wf.dryrun()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "f0add97a",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:24.593634Z",
"iopub.status.busy": "2025-06-18T09:44:24.593486Z",
"iopub.status.idle": "2025-06-18T09:44:24.607916Z",
"shell.execute_reply": "2025-06-18T09:44:24.607475Z"
},
"papermill": {
"duration": 0.018431,
"end_time": "2025-06-18T09:44:24.608609",
"exception": false,
"start_time": "2025-06-18T09:44:24.590178",
"status": "completed"
},
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"cases/pluvial_multiple_regions:\n",
"- Snakefile\n",
"- rulegraph.svg\n",
"- Snakefile.config.yml\n"
]
}
],
"source": [
"# Write the workflow to a Snakefile and snakefile.config.yml\n",
"wf.to_snakemake()\n",
"\n",
"# show the files in the case directory\n",
"print(f\"{wf.root.relative_to(pwd)}:\")\n",
"for f in wf.root.iterdir():\n",
" print(f\"- {f.name}\")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "b24c4055",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:24.615221Z",
"iopub.status.busy": "2025-06-18T09:44:24.615024Z",
"iopub.status.idle": "2025-06-18T09:44:24.617573Z",
"shell.execute_reply": "2025-06-18T09:44:24.617068Z"
},
"papermill": {
"duration": 0.00677,
"end_time": "2025-06-18T09:44:24.618321",
"exception": false,
"start_time": "2025-06-18T09:44:24.611551",
"status": "completed"
},
"tags": []
},
"outputs": [],
"source": [
"# uncomment to run the workflow with snakemake\n",
"# import subprocess\n",
"# subprocess.run([\"snakemake\", \"-c\", \"1\"], cwd=wf.root)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "6366801c",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:24.625463Z",
"iopub.status.busy": "2025-06-18T09:44:24.625029Z",
"iopub.status.idle": "2025-06-18T09:44:24.664177Z",
"shell.execute_reply": "2025-06-18T09:44:24.663748Z"
},
"papermill": {
"duration": 0.043683,
"end_time": "2025-06-18T09:44:24.665060",
"exception": false,
"start_time": "2025-06-18T09:44:24.621377",
"status": "completed"
},
"tags": []
},
"outputs": [],
"source": [
"# Write the workflow to a cwl file and cwl config file\n",
"wf.to_cwl()"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "8e30371a",
"metadata": {
"execution": {
"iopub.execute_input": "2025-06-18T09:44:24.672100Z",
"iopub.status.busy": "2025-06-18T09:44:24.671730Z",
"iopub.status.idle": "2025-06-18T09:44:24.674165Z",
"shell.execute_reply": "2025-06-18T09:44:24.673754Z"
},
"papermill": {
"duration": 0.006624,
"end_time": "2025-06-18T09:44:24.674846",
"exception": false,
"start_time": "2025-06-18T09:44:24.668222",
"status": "completed"
},
"tags": []
},
"outputs": [],
"source": [
"# uncomment to run the workflow with cwll\n",
"# import subprocess\n",
"# subprocess.run([\"cwltool\", \"pluvial_multiple_regions.cwl\", \"pluvial_multiple_regions.config.yml\"], cwd=wf.root)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7b9967c7",
"metadata": {
"papermill": {
"duration": 0.002986,
"end_time": "2025-06-18T09:44:24.681038",
"exception": false,
"start_time": "2025-06-18T09:44:24.678052",
"status": "completed"
},
"tags": []
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "full",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.12"
},
"papermill": {
"default_parameters": {},
"duration": 4.458504,
"end_time": "2025-06-18T09:44:25.299497",
"environment_variables": {},
"exception": null,
"input_path": "/home/runner/work/HydroFlows/HydroFlows/docs/../examples/pluvial_multiple_regions.ipynb",
"output_path": "/home/runner/work/HydroFlows/HydroFlows/docs/_examples/pluvial_multiple_regions.ipynb",
"parameters": {},
"start_time": "2025-06-18T09:44:20.840993",
"version": "2.6.0"
}
},
"nbformat": 4,
"nbformat_minor": 5
}