{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Test notebook for Pastas parameter sensitivity analysis with PEST++ SEN Solver" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "DeprecationWarning: As of Pastas 1.5, no noisemodel is added to the pastas Model class by default anymore. To solve your model using a noisemodel, you have to explicitly add a noisemodel to your model before solving. For more information, and how to adapt your code, please see this issue on GitHub: https://github.com/pastas/pastas/issues/735/home/martin/repos/pyemu/pyemu/__init__.py:29: UserWarning: Failed to import legacy module. May impact ability to access older methods.ModuleNotFoundError No module named 'flopy'\n" ] } ], "source": [ "from pathlib import Path\n", "\n", "import matplotlib.pyplot as plt\n", "import pandas as pd\n", "import pastas as ps\n", "\n", "import pastas_plugins.pest as psp" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "head = (\n", " pd.read_csv(\n", " \"https://raw.githubusercontent.com/pastas/pastas/master/doc/examples/data/head_nb1.csv\",\n", " index_col=\"date\",\n", " parse_dates=True,\n", " ).squeeze()\n", ").iloc[-300:]\n", "prec = pd.read_csv(\n", " \"https://raw.githubusercontent.com/pastas/pastas/master/doc/examples/data/rain_nb1.csv\",\n", " index_col=\"date\",\n", " parse_dates=True,\n", ").squeeze()\n", "evap = pd.read_csv(\n", " \"https://raw.githubusercontent.com/pastas/pastas/master/doc/examples/data/evap_nb1.csv\",\n", " index_col=\"date\",\n", " parse_dates=True,\n", ").squeeze()\n", "pex = (prec - evap).dropna().rename(\"PrecipitationExcess\")" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "ml = ps.Model(head)\n", "sm = ps.StressModel(\n", " pex, ps.Exponential(), name=\"pex\", settings=ps.rcParams[\"timeseries\"][\"evap\"]\n", ")\n", "\n", "ml.add_stressmodel(sm)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "tags": [ "hide-output" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2025-05-22 10:43:54.231677 starting: opening PstFrom.log for logging\n", "2025-05-22 10:43:54.234961 starting PstFrom process\n", "2025-05-22 10:43:54.235822 starting: setting up dirs\n", "2025-05-22 10:43:54.236113 starting: removing existing new_d '/home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp'\n", "2025-05-22 10:43:54.239164 finished: removing existing new_d '/home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp' took: 0:00:00.003051\n", "2025-05-22 10:43:54.239406 starting: copying original_d '/home/martin/repos/pastas-plugins/docs/examples/pestf_sen/model' to new_d '/home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp'\n", "2025-05-22 10:43:54.243253 finished: copying original_d '/home/martin/repos/pastas-plugins/docs/examples/pestf_sen/model' to new_d '/home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp' took: 0:00:00.003847\n", "2025-05-22 10:43:54.246437 finished: setting up dirs took: 0:00:00.010615\n", "2025-05-22 10:43:54.280707 starting: adding grid type d style parameters for file(s) ['parameters_sel.csv']\n", "2025-05-22 10:43:54.282970 starting: loading list-style /home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp/parameters_sel.csv\n", "2025-05-22 10:43:54.284340 starting: reading list-style file: /home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp/parameters_sel.csv\n", "2025-05-22 10:43:54.288411 finished: reading list-style file: /home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp/parameters_sel.csv took: 0:00:00.004071\n", "2025-05-22 10:43:54.290689 loaded list-style '/home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp/parameters_sel.csv' of shape (3, 2)\n", "2025-05-22 10:43:54.293918 finished: loading list-style /home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp/parameters_sel.csv took: 0:00:00.010948\n", "2025-05-22 10:43:54.295155 starting: writing list-style template file '/home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp/parameters_sel.csv.tpl'\n", "2025-05-22 10:43:54.314358 finished: writing list-style template file '/home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp/parameters_sel.csv.tpl' took: 0:00:00.019203\n", "2025-05-22 10:43:54.323356 finished: adding grid type d style parameters for file(s) ['parameters_sel.csv'] took: 0:00:00.042649\n", "2025-05-22 10:43:54.324542 starting: adding observations from output file simulation.csv\n", "2025-05-22 10:43:54.324791 starting: adding observations from tabular output file '['simulation.csv']'\n", "2025-05-22 10:43:54.325015 starting: reading list-style file: /home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp/simulation.csv\n", "2025-05-22 10:43:54.328622 finished: reading list-style file: /home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp/simulation.csv took: 0:00:00.003607\n", "2025-05-22 10:43:54.338682 starting: building insfile for tabular output file simulation.csv\n", "2025-05-22 10:43:54.355475 finished: building insfile for tabular output file simulation.csv took: 0:00:00.016793\n", "2025-05-22 10:43:54.356444 starting: adding observation from instruction file '/home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp/simulation.csv.ins'\n", "2025-05-22 10:43:54.372230 finished: adding observation from instruction file '/home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp/simulation.csv.ins' took: 0:00:00.015786\n", "2025-05-22 10:43:54.373990 finished: adding observations from tabular output file '['simulation.csv']' took: 0:00:00.049199\n", "2025-05-22 10:43:54.377013 finished: adding observations from output file simulation.csv took: 0:00:00.052471\n", "2025-05-22 10:43:54.379744 WARNING: add_py_function() command: run() is not being called directly\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/home/martin/repos/pyemu/pyemu/logger.py:100: PyemuWarning: 2025-05-22 10:43:54.379744 WARNING: add_py_function() command: run() is not being called directly\n", "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "noptmax:0, npar_adj:3, nnz_obs:300\n", "noptmax:1, npar_adj:3, nnz_obs:300\n", "noptmax:1, npar_adj:3, nnz_obs:300\n", "\n", "\n", " pestpp-sen: a tool for global sensitivity analysis\n", "\n", " by The PEST++ Development Team\n", "\n", "\n", "version: 5.2.16\n", "binary compiled on Dec 1 2024 at 10:51:08\n", "\n", "started at 05/22/25 10:43:54\n", "...processing command line: ' ./pestpp-sen pest.pst /h :4005'\n", "...using panther run manager in master mode using port 4005\n", "\n", "using control file: \"pest.pst\"\n", "in directory: \"/home/martin/repos/pastas-plugins/docs/examples/pestf_sen/temp\"\n", "on host: \"VonkXPS15\"\n", "\n", "processing control file pest.pst\n", "\n", "\n", ":~-._ _.-~:\n", ": :.~^o._ ________---------________ _.o^~.:.:\n", " : ::.`?88booo~~~.::::::::...::::::::::::..~~oood88P'.::.:\n", " : ::: `?88P .:::.... ........:::::. ?88P' :::. :\n", " : :::. `? .::. . ...........:::. P' .:::. :\n", " : ::: ... .. ... .. .::::......::. :::. :\n", " ` :' .... .. .:::::. . ..:::::::....:::. `: .'\n", " :.. ____:::::::::. . . ....:::::::::____ ... :\n", " :... `:~ ^~-:::::.. .........:::::-~^ ~::.::::\n", " `.::. `\\ (8) \\b:::..::.:.:::::::d/ (8) /'.::::'\n", " ::::. ~-._v |b.::::::::::::::d| v_.-~..:::::\n", " `.:::::... ~~^?888b..:::::::::::d888P^~...::::::::'\n", " `.::::::::::....~~~ .:::::::::~~~:::::::::::::::'\n", " `..::::::::::: . ....:::: ::::::::::::,'\n", " `. .::::::: . .::::. ::::::::'.'\n", " `._ .::: . :::::. :::::_.'\n", " `-. : . ::::: :,-'\n", " :. :___ .:::___ .::\n", " ..--~~~~--:+::. ~~^?b..:::dP^~~.::++:--~~~~--..\n", " ___....--`+:::. `~8~' .:::+'--....___\n", " ~~ __..---`_=:: ___gd8bg___ :==_'---..__ ~~\n", " -~~~ _.--~~`-.~~~~~~~~~~~~~~~,-' ~~--._ ~~~-\n", "\n", "\n", " starting PANTHER master...\n", "\n", "IP addresses:\n", " 0.0.0.0:4005 (IPv4)\n", "\n", "PANTHER master listening on socket: 0.0.0.0:4005 (IPv4)\n", "\n", "using control file: \"pest.pst\"\n", "\n", "Note: only non-zero weighted observations contribute to\n", " the phi and group phi sensitivity metrics. Please\n", " make sure this is what you want...\n", "\n", "\n", "Building model run parameter sets...\n", "\n", "Performing model runs...\n", " running model 16 times\n", " starting at 05/22/25 10:43:54\n", "\n", " waiting for agents to appear...\n", "\n", "\n", "PANTHER progress\n", " avg = average model run time in minutes\n", " runs(C = completed | F = failed | T = timed out)\n", " agents(R = running | W = waiting | U = unavailable)\n", "--------------------------------------------------------------------------------\n", "2025-05-22 10:43:55.175610 : trying to connect to localhost:4005...\n", "2025-05-22 10:43:55.192563 : trying to connect to localhost:4005...\n", "2025-05-22 10:43:55.215713 : trying to connect to localhost:4005...\n", "2025-05-22 10:43:55.224188 : trying to connect to localhost:4005...\n", "2025-05-22 10:43:55.250394 : trying to connect to localhost:4005...\n", "2025-05-22 10:43:55.270683 : trying to connect to localhost:4005...\n", "2025-05-22 10:43:55.294529 : trying to connect to localhost:4005...\n", "2025-05-22 10:43:55.324636 : trying to connect to localhost:4005...\n", "2025-05-22 10:43:55.441099 : connected to localhost:4005\n", "2025-05-22 10:43:55.449900 : connected to localhost:4005\n", "2025-05-22 10:43:55.473354 : connected to localhost:4005\n", "2025-05-22 10:43:55.486236 : connected to localhost:4005\n", "2025-05-22 10:43:55.508256 : connected to localhost:4005\n", "2025-05-22 10:43:55.528148 : connected to localhost:4005\n", "2025-05-22 10:43:55.551553 : connected to localhost:4005\n", "2025-05-22 10:43:55.582018 : connected to localhost:4005\n", "05/22 10:43:57 remaining file transfers: 0 \n", "\n", " 16 runs complete : 0 runs failed\n", " 0.00534 avg run time (min) : 0.0446 run mgr time (min)\n", " 8 agents connected\n", "\n", "\n", "Calculating sensitivities...\n", "\n", "par_name:PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_G\n", "PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_G\n", "\n", "par_name:PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_A\n", "PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_A\n", "\n", "par_name:PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:CONSTANT_D\n", "PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:CONSTANT_D\n", "\n", "par_name:\n", "\n", "\n", "par_name:PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_G\n", "PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_G\n", "\n", "par_name:PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_A\n", "PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_A\n", "\n", "par_name:PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:CONSTANT_D\n", "PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:CONSTANT_D\n", "\n", "par_name:\n", "\n", "\n", "par_name:PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_G\n", "PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_G\n", "\n", "par_name:PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_A\n", "PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_A\n", "\n", "par_name:PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:CONSTANT_D\n", "PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:CONSTANT_D\n", "\n", "par_name:\n", "\n", "\n", "par_name:PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_G\n", "PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_G\n", "\n", "par_name:PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_A\n", "PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:PEX_A\n", "\n", "par_name:PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:CONSTANT_D\n", "PNAME:P_INST:0_PTYPE:GR_USECOL:OPTIMAL_PSTYLE:D_PARNAMES:CONSTANT_D\n", "\n", "\n", "writing output files\n", "\n", "\n", "pestpp-sen analysis complete...\n", "started at 05/22/25 10:43:54\n", "finished at 05/22/25 10:43:57\n", "took 0.0333333 minutes\n" ] } ], "source": [ "ml_sen = ml.copy()\n", "ml_sen.name = \"Pestsen\"\n", "solver = psp.PestSenSolver(\n", " exe_name=\"bin/pestpp-sen\",\n", " model_ws=Path(\"pestf_sen/model\"),\n", " temp_ws=Path(\"pestf_sen/temp\"),\n", " master_ws=Path(\"pestf_sen/master\"),\n", " noptmax=1,\n", " port_number=4005,\n", ")\n", "ml_sen.solver = solver\n", "ml_sen.solver.set_model(ml_sen)\n", "ml_sen.solver.start()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | n_samples | \n", "sen_mean | \n", "sen_mean_abs | \n", "sen_std_dev | \n", "
|---|---|---|---|---|
| pex_A | \n", "4 | \n", "-2125350.00 | \n", "2125350.00 | \n", "2686220.00 | \n", "
| pex_a | \n", "4 | \n", "-174948.00 | \n", "174948.00 | \n", "349897.00 | \n", "
| constant_d | \n", "4 | \n", "1158.04 | \n", "1369.95 | \n", "2072.42 | \n", "